Speed tables: prompts with engine 0.1.22 (bench/results/2026-09-29-speed-0122)

It says what happens (the experts must stay in RAM, so the OS pages part of them: much slower, maybe no start) and
asks, "no" by default, so an unattended --yes install still stops. Thanks @FatheredPuma81.

CMakeLists.txt

@@ -4,11 +4,11 @@
 # `strata-pack` are added when their sources exist - a target with no sources would make the build graph
 # claim something the tree does not have, which is worse than an absent target.
 #
-# CUDA targets arrive in Phase 2 and this build REJECTS any architecture other than 120: the engine is
-# written against sm_120 (RTX 5000 / Blackwell) and silently building for an older arch would produce a
-# binary that runs and is wrong about its own performance, which is the worst of both outcomes.
+# CUDA targets arrive in Phase 2.  The kernels need sm_80 or newer (tf32 mma, bf16 math) and the guard
+# below refuses anything older: silently building for a pre-Ampere arch would produce a binary that runs
+# and is wrong about its own performance, which is the worst of both outcomes.
 cmake_minimum_required(VERSION 3.24)
-project(strata VERSION 0.1.20 LANGUAGES CXX)
+project(strata VERSION 0.1.23 LANGUAGES CXX)
 add_compile_definitions(STRATA_VERSION="${PROJECT_VERSION}")
 # issue #31: release builds keep a PDB beside the exe (not shipped in it), so a minidump from a stalled engine can
 # be read back to the exact source line
@@ -56,7 +56,7 @@ if(STRATA_ENABLE_CUDA)
     if(_base STREQUAL "native" OR _base STREQUAL "all" OR _base STREQUAL "all-major")
       continue()
     endif()
-    if(_base LESS 80)
+    if(_base LESS 80 AND NOT STRATA_EXPERIMENTAL_SM75)   # the A/B of a Turing card as a layer-split stage
       message(FATAL_ERROR
         "Strata needs an NVIDIA GPU of compute capability 8.0 or newer (RTX 30 / 40 / 50 series), but "
         "CMAKE_CUDA_ARCHITECTURES is '${CMAKE_CUDA_ARCHITECTURES}'.")
@@ -100,6 +100,12 @@ add_executable(strata-dequant src/artifact/dequant.cpp)
 target_link_libraries(strata-dequant PRIVATE strata_artifact)
 # this file is now nothing but the CLI entry point, and its inherited guard spells that as a "selftest"
 target_compile_definitions(strata-dequant PRIVATE STRATA_DEQUANT_SELFTEST)
+if(STRATA_BUILD_TESTS)
+  # the reader refuses a duplicate tensor name at open, naming it (a synthetic GGUF: no model, no GPU)
+  add_executable(gguf_reader_test src/artifact/gguf_reader_test.cpp)
+  target_link_libraries(gguf_reader_test PRIVATE strata_artifact)
+  add_test(NAME gguf_reader_test COMMAND gguf_reader_test)
+endif()
 
 # ---- strata_plan ----------------------------------------------------------
 # Also header-only.  A "planner" that hides state would be one whose decision is not inspectable, and the
@@ -189,6 +195,7 @@ if(STRATA_ENABLE_CUDA)
                           src/kernels/cuda/fused_gr.cu
                           src/kernels/cuda/cvec.cu
                           src/kernels/cuda/qsa_decode_attn.cu
+                          src/kernels/cuda/qsa_prompt_attn.cu
                           src/kernels/cuda/fused_gdn.cu
                           src/kernels/cuda/dequant_bf16.cu
                           src/kernels/cuda/qsa_select.cu
@@ -198,6 +205,13 @@ if(STRATA_ENABLE_CUDA)
                           src/ngram/ple_reader.cpp
                           src/platform/direct_file.cpp)
   target_link_libraries(strata_kernels PUBLIC strata_core strata_artifact)
+  # perf-review D-1: the tensor-core prompt attention vs the FP32 kernel and FP64 (GPU, synthetic, no model); with
+  # the tests, or alone with -DSTRATA_PARITY_PROMPT_ATTN=ON
+  if(STRATA_BUILD_TESTS OR STRATA_PARITY_PROMPT_ATTN)
+    add_executable(qsa_prompt_attn_parity src/kernels/qsa_prompt_attn_parity.cpp)
+    target_include_directories(qsa_prompt_attn_parity PRIVATE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+    target_link_libraries(qsa_prompt_attn_parity PRIVATE strata_kernels CUDA::cudart)
+  endif()
   # plan v0.3 P6: llama.cpp's block layouts and codebook grids for the i-quant formats (MIT, third_party/ggml)
   target_include_directories(strata_kernels PRIVATE third_party/ggml)
   # Match the pinned llama.cpp CUDA arithmetic contract for this adapter only.
@@ -245,7 +259,7 @@ if(STRATA_ENABLE_CUDA)
   # already links `strata_core` - putting `gdn_layer` in core would make the two mutually dependent.  A third
   # library is the honest shape of the dependency graph: core knows about devices and weights, kernels know
   # about arithmetic, and this one knows the ORDER the arithmetic happens in.
-  add_library(strata_engine STATIC src/core/layer.cpp src/core/session.cpp src/core/expert_source.cpp
+  add_library(strata_engine STATIC src/core/layer.cpp src/core/session.cpp src/core/expert_source.cpp src/core/remote_experts.cpp
     src/core/expert_cache.cpp src/core/native_head.cpp src/core/native_dense.cpp src/core/verify.cpp src/core/mtp.cpp)
   target_include_directories(strata_engine PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
   target_link_libraries(strata_engine PUBLIC strata_core strata_kernels strata_kernels_cpu CUDA::cudart)
@@ -571,8 +585,10 @@ if(STRATA_BUILD_TESTS)
   # Plan v0.3 P6: one expert applied to several tokens, bitwise vs the single-token kernel (synthetic, CPU only).
   # plan v0.3 P5: the device dequantizers vs the artifact's CPU dequantizers on the real shards (not a ctest: needs
   # the 38 GB model; run it by hand or from tools/gpu_session.py)
-  add_executable(dequant_bf16_test src/kernels/dequant_bf16_test.cpp)
-  target_link_libraries(dequant_bf16_test PRIVATE strata_kernels strata_artifact CUDA::cudart)
+  if(STRATA_ENABLE_CUDA)   # #93: it needs the device dequantizers and cudart (a CPU-only configure has neither)
+    add_executable(dequant_bf16_test src/kernels/dequant_bf16_test.cpp)
+    target_link_libraries(dequant_bf16_test PRIVATE strata_kernels strata_artifact CUDA::cudart)
+  endif()
 
   add_executable(expert_multi_test src/kernels/cpu/expert_multi_test.cpp)
   target_link_libraries(expert_multi_test PRIVATE strata_kernels_cpu)
@@ -596,5 +612,8 @@ endif()
 
 if(STRATA_BUILD_TESTS)
   enable_testing()
-  add_subdirectory(tests)
+  # the unit tests above build from src/; the tests/ tree (model-level checks) is not part of the published repo
+  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/tests/CMakeLists.txt)
+    add_subdirectory(tests)
+  endif()
 endif()

README.md

@@ -25,15 +25,15 @@ Measured on an RTX 5070 (12 GB), a Ryzen 5 7600 and 64 GB of RAM:
 
 | Size | Writes answers (short chat) | Writes answers (128K context) | Reads your prompt |
 | --- | ---: | ---: | ---: |
-| **Q2_0** | 90 tokens/s | 67 tokens/s | 1,310 tokens/s |
-| **IQ2_XS** | 74 tokens/s | 60 tokens/s | 1,240 tokens/s |
-| **IQ3_XXS** | 62 tokens/s | 46 tokens/s | 1,110 tokens/s |
-| **IQ3_S** | 52 tokens/s | 41 tokens/s | 1,070 tokens/s |
-| **Coder** (IQ1_M) | 51 tokens/s | 44 tokens/s | 1,300 tokens/s |
+| **Q2_0** | 90 tokens/s | 67 tokens/s | 1,840 tokens/s |
+| **IQ2_XS** | 74 tokens/s | 60 tokens/s | 1,800 tokens/s |
+| **IQ3_XXS** | 62 tokens/s | 46 tokens/s | 1,550 tokens/s |
+| **IQ3_S** | 52 tokens/s | 41 tokens/s | 1,500 tokens/s |
+| **Coder** (IQ1_M) | 51 tokens/s | 44 tokens/s | 1,870 tokens/s |
 
 - **Writes answers** = how fast the reply appears (tokens per second).
 - **Reads your prompt** = how fast it takes in what you send (long documents, code, chat history), measured on a
-  32K-token prompt; a 4K prompt reads at 740-1,000 tokens/s. A 32K prompt takes about 25 seconds with Q2_0.
+  32K-token prompt; a 4K prompt reads at 890-1,520 tokens/s. A 32K prompt takes about 17 seconds with Q2_0.
 
 A card with more VRAM is faster, because more of the model fits on the GPU: an RTX 3090 (24 GB) should do roughly
 100-140 tokens per second. All measurements, long-context numbers and estimates for other cards are in the
@@ -42,6 +42,13 @@ A card with more VRAM is faster, because more of the model fits on the GPU: an R
 Every PC is different: `START-HERE.bat --calibrate` measures a few engine settings on yours and keeps the fastest
 (about 5-10 minutes; on the PC above it made the Coder 7% faster).
 
+**Two or three NVIDIA cards?** Just run `START-HERE.bat`: it lists your cards, says which ones Strata can use, and
+asks whether to share the model across them (recommended when two can). An install made on one card asks once at
+its next start. Or choose yourself: `START-HERE.bat --gpus 0,2` (both, remembered), `--gpus all`, or `--gpu 0` (one
+card, this start only). Each card keeps the experts of its own layers, and prompts flow through the cards in a
+pipeline: on an RTX 5080 + RTX 3090 prompts were read 18-20% faster than on the 5080 alone, decoding on par.
+Every card must be an RTX 30 series or newer with 8 GB or more. See [docs/MULTI_GPU.md](docs/MULTI_GPU.md).
+
 ## Which model should I pick?
 
 **The size** (the same model, compressed more or less):

bench/results/2026-09-29-layer-split-limits/README.md

@@ -0,0 +1,59 @@
+# Layer split: the features that were off across GPUs (2026-09-29)
+
+**Setup:**
+- **Rig:** RIBPC. RTX 5080 16 GB (CUDA0) + RTX 3090 24 GB; RTX 2080 Ti 22 GB as a helper card. Ryzen 9 9950X3D,
+  62 GB RAM, Windows 11.
+- **Model:** the Coder (IQ1_M native pack).
+- **Build:** branch `mg/limits`, sm_75 + sm_86 + sm_120.
+- **Correctness runs:** `rib_norepeat.py`, with a fixed cache, no swaps, no PCIe share and no suffix drafts where two
+  runs are compared byte for byte.
+- **Speed runs:** `rib_pp.py`.
+- Raw results are in `data/`.
+
+## Correctness
+
+| Check | Result |
+|---|---|
+| One GPU, this branch vs the 0.1.21-equivalent run (fixed cache) | 10/10 byte-identical |
+| Speed projection (control vector), one GPU vs the split on the same GPU (K=24) | 10/10 byte-identical |
+| KV streaming (128K context, 32K resident), one GPU vs the split on the same GPU | 10/10 byte-identical |
+| Images, one GPU vs the split on the same GPU (three questions about a test picture) | identical answers |
+| Speed projection / KV streaming / plain, 5080 + 3090 (K=24): no-repeat under presence 100, the 144 sanity case | all pass, 0 repeats |
+| The vector acts across GPUs: 5080 + 3090 with vs without the vector | 8 of 10 answers differ (as on one GPU: 7 of 10) |
+| Images, 5080 + 3090 (K=24): "What color is the circle?" / "What number?" / describe | "Red" / "42" / "A red circle, the number 42, and a blue square are displayed on a white background" |
+| Mid-prompt checkpoint (every 16,384 tokens): a 19.5K prompt, then another sharing its first 18,600 tokens | one GPU and 5080 + 3090: reused 16,384, answer right |
+| 5080 + 3090 (K=30) + a 2080 Ti helper holding 3,000 experts | all pass |
+| `--mmap-experts` with the Coder (a native pack) | a clear start-up error: it needs a canonical pack's `experts.bin` (with or without a split) |
+
+Across GPUs the text is not byte-identical to one GPU: experts that one card leaves to the CPU run on a GPU in the
+split, and the two round differently.
+
+## Speed
+
+| Run | Prompt 16K / 28K / 60K tok/s | Decode story / code tok/s | Hits |
+|---|---|---|---|
+| KV streaming (128K), 5080 alone | 1,487 / 1,801 / 1,837 | 66.6 / 74.0 | 59-71% |
+| KV streaming (128K), 5080 + 3090 auto | 1,685 / 1,957 / 1,968 | 75.8 / 97.0 | 96-97% |
+| Speed projection, 5080 + 3090 auto | 1,813 / 2,053 / - | 77.0 / 103.6 | 96-97% |
+| 5080 + 3090 (K=30), no helper | - | 62.5 / 85.2 | |
+| 5080 + 3090 (K=30) + 2080 Ti helper | - | 43.0 / 63.7 | |
+
+**The helper card makes it slower,** as it did without a split (bench/results/2026-09-29-layer-split). Its per-layer
+round trip costs more than the CPU pool needs for the few experts it would take over. It stays supported: the
+helpers take the visible GPUs no stage runs on and hold only experts no stage's cache holds. But it isn't
+recommended.
+
+## What changed
+
+- **Per-device tables:** the image-position (mrope) table and the control vector's tables are now kept per device;
+  every stage's rope and steering kernels read their own device's copy. `cvec_set_enabled` switches the vector on
+  every device.
+- **KV streaming:** each stage's session streams its own KV host copy (pinned, portable). The split no longer forces
+  it off.
+- **Mid-prompt checkpoints:** `Prefill::on_stage_chunk` lets each stage save its part of a checkpoint when it has
+  read the chunk. The last stage assembles the parts: the earlier ones are a chunk ahead by then.
+- **Helpers (`--expert-cache-remote`) with a split:** they take the visible devices after the stages, skip every
+  pair a stage's cache holds, and preflight and open on those devices.
+- **Explicit `--expert-cache` with a split:** it is capped so CUDA0 keeps room for the prompt path's own buffers.
+  Without the cap, the first prompt failed with "device buffers ... do not fit".
+- **`--mmap-experts`:** refused at start for a native pack, with the reason.

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-cv-same24-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 102,
+  "drafts_offered": 166,
+  "tok_s": 66.7,
+  "text": "The wind didn\u2019t just howl; it screamed. It tore through the iron gallery of the Blackwood Point Light, rattling the glass panes like a prisoner testing the bars of a cell.\n\nElias Thorne was not a man who feared the dark. He was a man who feared the silence.\n\nFor forty years, he had lived in the shadow of the Fresnel lens, a rotating eye of glass and brass that swept the Atlantic with a beam of pure, unyielding light. His job was simple: keep the light burning. Keep the logbook current. Keep the foghorn sounding when the mist rolled in, thick as wool.\n\nBut Elias knew something else. He knew that the light was not just for ships. It was for the lost.\n\nIt was November 14th, 1982. The storm was a Category 5, a monster that had swallowed the horizon. The waves were mountains of black water, crashing against the granite"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 134,
+  "drafts_offered": 180,
+  "tok_s": 80.8,
+  "text": "```python\nimport csv\nfrom typing import List, Dict\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, str]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile(mode='w', suffix='.csv', delete=False"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 72.5,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 74.6,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 104,
+  "repeats": 0,
+  "drafts_accepted": 29,
+  "drafts_offered": 92,
+  "tok_s": 45.4,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAnaconda    \nAntelope     \nBaboon       \nBadger      \nBat         \nBear        \nBeaver          \nBirds of Paradise (Manucode) \nBoa Constrictor \n\nBuffalo           \nCamel            \nCapybara             \nCat              \nChameleon\nCheetah\n\nChicken\n\n\nCrocodile  \nDeer  \n\nDingo   \nDog    \nDonkey     \nDragonfly   \n\nElephant       \nEmu      \nFalcon        \nFerret          \nFlamingo         \nFox<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 72,
+  "repeats": 0,
+  "drafts_accepted": 15,
+  "drafts_offered": 70,
+  "tok_s": 46.1,
+  "text": "The wind didn\u2019t just howl; it screamed against the glass of Lantern House 42, demanding entry into every crack and crevice that had been sealed with tar in previous years but now wept like old wounds on an open sore skin\u2014like a wound left unhealed by time. Elias knew this rhythm better than he remembered his own name<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 26,
+  "drafts_offered": 76,
+  "tok_s": 49.3,
+  "text": "The wind didn\u2019t just blow at Blackwood Rock; it screamed through the gaps in my teeth, searching for something to break its rhythm against stone that refused be broken by anything less than God himself had forged into being willing and able withstand eternity\u2019s cruelty without complaint or mercy from start finish until now forevermore if any such concept even existed within human comprehension let alone divine authority which we all knew did not exist but wished it would because wishful thinking is always easier to imagine hopelessly impossible truths as probable lies than vice versa.\n\nElias Thorne was eighty-three years old today, exactly one hundred and forty-seven days since his last heart attack had been logged by Dr Armitage who himself retired seven months ago after receiving a subpoena from the county clerk indicating he might have overstayed professional courtesy boundaries"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 94,
+  "repeats": 0,
+  "drafts_accepted": 27,
+  "drafts_offered": 56,
+  "tok_s": 46.7,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKangaroo      \nKoala        \nHippo         \nRhino          \nCheetah            \nLeopard           \nJaguar             \nWolf                \nFox              \nBear               \nDeer                 \nElk                  \nMoose                    \nBuffalo                        \nCamel                       \nDonkey                      \nGoat                     \nSheep                   \nCow                         \nBull                          \nOx                           \nPig                            \nBoar                             \nRabbit                              \nSquirrel                                <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 13,
+  "tok_s": 62.4,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 37,
+  "drafts_offered": 148,
+  "tok_s": 56.4,
+  "text": "The phrase \"the History Of The Sea\" is inherently ambiguous, as it could refer to several distinct historical narratives: geological formation (how oceans came into existence), maritime navigation and exploration by humans throughout human civilization; or perhaps more specifically in modern times refers often used metaphorically for economic trends like stock market fluctuations (\"sea change\"). However given your request likely intends an academic-style comprehensive overview covering both natural science aspects AND cultural significance through time periods\u2014especially since this kind question appears frequently on exams asking students write essays discussing evolution from primordial origins up present-day impacts including environmental changes due climate crisis etc.\u2014I will craft such broad yet structured piece combining geology + anthropology/ecological impact together while maintaining coherent flow suitable high-level educational context without overly technical jargon but still accurate enough scientifically grounded facts where appropriate within reasonable scope limits typical school assignments expect (~80\u201312% word count range implied implicitly though not stated explicitly here we aim moderate length ~65 words? Wait no user said \u201clong\u201d so probably >3k+ characters minimum safe bet around 7-9 paragraphs total approximating maybe close-to-full-page standard double-spaced equivalent which translates roughly \u22654K chars safely acceptable). Let me proceed accordingly ensuring clarity structure intro body conclusion style formal tone neutral informative balanced perspective acknowledging uncertainties if any minor ones exist e.g exact dates vary depending sources hence use hedging language sparingly only when necessary avoid overclaiming certainty especially regarding speculative theories vs established consensus knowledge base current mainstream understanding circa early-mid"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-cv-single-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 102,
+  "drafts_offered": 166,
+  "tok_s": 67.1,
+  "text": "The wind didn\u2019t just howl; it screamed. It tore through the iron gallery of the Blackwood Point Light, rattling the glass panes like a prisoner testing the bars of a cell.\n\nElias Thorne was not a man who feared the dark. He was a man who feared the silence.\n\nFor forty years, he had lived in the shadow of the Fresnel lens, a rotating eye of glass and brass that swept the Atlantic with a beam of pure, unyielding light. His job was simple: keep the light burning. Keep the logbook current. Keep the foghorn sounding when the mist rolled in, thick as wool.\n\nBut Elias knew something else. He knew that the light was not just for ships. It was for the lost.\n\nIt was November 14th, 1982. The storm was a Category 5, a monster that had swallowed the horizon. The waves were mountains of black water, crashing against the granite"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 134,
+  "drafts_offered": 180,
+  "tok_s": 81.0,
+  "text": "```python\nimport csv\nfrom typing import List, Dict\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, str]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile(mode='w', suffix='.csv', delete=False"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 62.4,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 73.4,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 104,
+  "repeats": 0,
+  "drafts_accepted": 29,
+  "drafts_offered": 92,
+  "tok_s": 43.5,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAnaconda    \nAntelope     \nBaboon       \nBadger      \nBat         \nBear        \nBeaver          \nBirds of Paradise (Manucode) \nBoa Constrictor \n\nBuffalo           \nCamel            \nCapybara             \nCat              \nChameleon\nCheetah\n\nChicken\n\n\nCrocodile  \nDeer  \n\nDingo   \nDog    \nDonkey     \nDragonfly   \n\nElephant       \nEmu      \nFalcon        \nFerret          \nFlamingo         \nFox<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 72,
+  "repeats": 0,
+  "drafts_accepted": 15,
+  "drafts_offered": 70,
+  "tok_s": 46.1,
+  "text": "The wind didn\u2019t just howl; it screamed against the glass of Lantern House 42, demanding entry into every crack and crevice that had been sealed with tar in previous years but now wept like old wounds on an open sore skin\u2014like a wound left unhealed by time. Elias knew this rhythm better than he remembered his own name<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 26,
+  "drafts_offered": 76,
+  "tok_s": 48.9,
+  "text": "The wind didn\u2019t just blow at Blackwood Rock; it screamed through the gaps in my teeth, searching for something to break its rhythm against stone that refused be broken by anything less than God himself had forged into being willing and able withstand eternity\u2019s cruelty without complaint or mercy from start finish until now forevermore if any such concept even existed within human comprehension let alone divine authority which we all knew did not exist but wished it would because wishful thinking is always easier to imagine hopelessly impossible truths as probable lies than vice versa.\n\nElias Thorne was eighty-three years old today, exactly one hundred and forty-seven days since his last heart attack had been logged by Dr Armitage who himself retired seven months ago after receiving a subpoena from the county clerk indicating he might have overstayed professional courtesy boundaries"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 94,
+  "repeats": 0,
+  "drafts_accepted": 27,
+  "drafts_offered": 56,
+  "tok_s": 46.8,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKangaroo      \nKoala        \nHippo         \nRhino          \nCheetah            \nLeopard           \nJaguar             \nWolf                \nFox              \nBear               \nDeer                 \nElk                  \nMoose                    \nBuffalo                        \nCamel                       \nDonkey                      \nGoat                     \nSheep                   \nCow                         \nBull                          \nOx                           \nPig                            \nBoar                             \nRabbit                              \nSquirrel                                <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 13,
+  "tok_s": 63.1,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 37,
+  "drafts_offered": 148,
+  "tok_s": 56.6,
+  "text": "The phrase \"the History Of The Sea\" is inherently ambiguous, as it could refer to several distinct historical narratives: geological formation (how oceans came into existence), maritime navigation and exploration by humans throughout human civilization; or perhaps more specifically in modern times refers often used metaphorically for economic trends like stock market fluctuations (\"sea change\"). However given your request likely intends an academic-style comprehensive overview covering both natural science aspects AND cultural significance through time periods\u2014especially since this kind question appears frequently on exams asking students write essays discussing evolution from primordial origins up present-day impacts including environmental changes due climate crisis etc.\u2014I will craft such broad yet structured piece combining geology + anthropology/ecological impact together while maintaining coherent flow suitable high-level educational context without overly technical jargon but still accurate enough scientifically grounded facts where appropriate within reasonable scope limits typical school assignments expect (~80\u201312% word count range implied implicitly though not stated explicitly here we aim moderate length ~65 words? Wait no user said \u201clong\u201d so probably >3k+ characters minimum safe bet around 7-9 paragraphs total approximating maybe close-to-full-page standard double-spaced equivalent which translates roughly \u22654K chars safely acceptable). Let me proceed accordingly ensuring clarity structure intro body conclusion style formal tone neutral informative balanced perspective acknowledging uncertainties if any minor ones exist e.g exact dates vary depending sources hence use hedging language sparingly only when necessary avoid overclaiming certainty especially regarding speculative theories vs established consensus knowledge base current mainstream understanding circa early-mid"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-cv-x2-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 85,
+  "drafts_offered": 172,
+  "tok_s": 36.6,
+  "text": "The wind didn\u2019t just howl; it screamed. It tore through the gaps in the iron railing of the Lantern Room, carrying the scent of brine and rotting kelp. Elias pulled his woolen collar up to his chin, his fingers numb and clumsy inside thick leather gloves.\n\nHe was the last man on the island. The automated systems had failed three winters ago, a glitch in the solar-powered backup that no engineer had been able to fix before the supply boat stopped coming. Now, it was just him, the light, and the sea.\n\nElias climbed the spiral stairs, his boots heavy with salt spray. The iron steps groaned under his weight, a sound that seemed to mock the storm outside. He reached the top, the glass walls vibrating with the force of the gale. Outside, the world was a blur of black and white. The ocean was invisible; it was only a presence, a weight pressing against the stone foundation of the tower.\n\nHe"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 136,
+  "drafts_offered": 173,
+  "tok_s": 96.8,
+  "text": "```python\nimport csv\nfrom typing import List, Dict\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, str]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile(mode='w', suffix='.csv', delete=False"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 88.7,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 94.1,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 104,
+  "repeats": 0,
+  "drafts_accepted": 30,
+  "drafts_offered": 96,
+  "tok_s": 55.9,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAnaconda    \nAntelope     \nBaboon       \nBadger      \nBat         \nBear        \nBeaver          \nBirds of Paradise (Manucode) \nBoa Constrictor \n\nBuffalo           \nCamel            \nCapybara             \nCat              \nChameleon\nCheetah\n\nChicken\n\n\nCrocodile  \nDeer  \n\nDingo   \nDog    \nDonkey     \nDragonfly   \n\nElephant       \nEmu      \nFalcon        \nFerret          \nFlamingo         \nFox<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 29,
+  "drafts_offered": 114,
+  "tok_s": 55.4,
+  "text": "The wind didn\u2019t just howl; it screamed against the glass of Lantern House 42, demanding entry into every crack and crevice that had been sealed with tar in previous years but now wept like old wounds at this one\u2019s age: forty-three winters since its construction. My name was Elias Thorne\u2014no middle initial\u2014and I kept no logbook for visitors because there were none; only ships passed by, their hulls cutting through the black water as if they too feared what lay beneath it more than above or beside them in fog so thick that even my own reflection vanished from sight when looked upon directly.\n\nI climbed each morning not to greet dawn but check whether light still lived within me\u2014the bulb\u2019s heart had failed twice before, once during a storm named *Harrow*,"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 29,
+  "drafts_offered": 110,
+  "tok_s": 52.2,
+  "text": "The wind didn\u2019t just blow at Blackwood Rock; it screamed through the gaps in my teeth, searching for something to hold onto until I died.\u201d said Elias Thorne as he adjusted his collar against November\u2019s bite.** He was not merely complaining of cold weather but articulating a poetic metaphor: that even after death (metaphorically), there would be no relief from this relentless gale.\n\nElias had been the last human keeper on any significant stretch coast since automation became standard practice, yet here he stood , stubbornly refusing to let technology replace his vigilance . His cottage perched atop a crumbling granite foundation was more than just shelter; it housed centuries of handwritten logs detailing every ship\u2019s name and cargo load that passed within sight.\n\nOn most nights Elias tended the light mechanically\u2014"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 91,
+  "repeats": 0,
+  "drafts_accepted": 29,
+  "drafts_offered": 69,
+  "tok_s": 58.7,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKangaroo      \nKoala        \nHippopotamus          \nRhinoceros            \nLeopard                \nCheetah                 \nPanther                  \nWolf                    \nFox                      \nBear                     \nDeer                       \nElk                        \nMoose               \nBuffalo                   \nCamel           \nDonkey              \nGoat             \nSheep         \nCow  \nBull   \nOx    \nPig     \nDog       \nCat<|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 12,
+  "repeats": 0,
+  "drafts_accepted": 4,
+  "drafts_offered": 17,
+  "tok_s": 60.3,
+  "text": "1 2,3456789<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 41,
+  "drafts_offered": 168,
+  "tok_s": 59.9,
+  "text": "The phrase \"the History Of The Sea\" is inherently ambiguous, as it could refer to several distinct historical narratives: geological formation (how oceans came into existence), maritime navigation and exploration histories (\"history at/over/across/on/in/of/to/from/toward/off by means around above below through under upon alongside against among between within without throughout despite during after before since until while when where who whom whose which what how why whether\"), or perhaps even metaphorical interpretations involving human interaction with water bodies over time periods like ages eras epochs generations centuries decades years months days hours minutes seconds moments instants times occasions instances events episodes incidents occurrences situations circumstances conditions states phases stages steps processes procedures sequences series chains cycles loops repetitions iterations continuations progressions developments evolutions transformations changes alterations modifications adjustments corrections revisions amendments updates upgrades improvements enhancements refinements optimizations perfecting completing finishing concluding ending terminating ceasing stopping halting pausing resuming continuing proceeding advancing progressing moving shifting translating rotating reflecting mirroring flipping turning twisting bending curving arches spirals helix coils twists turns rotations reflections mirrors flips inversions reversings reversion backtracking retracing reversing undo redo do again repeat repeats repeated repeatedly repetition repetitive repetitiveness repete repeater repeatingly recurrent recurrence recurring recurs recur recurred recursion recursive recursively recurse recessed recessionary regress regression regressed retrograde reverse reversed reversal revert reverted reverberate resonated echo echoed echoing echoes echolocation sonar radar radio telecommunication communication communications communicate communicated communicating communicative comms comm chatter chat chats chatting chatted talk talks talked"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-hp0-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 88,
+  "drafts_offered": 171,
+  "tok_s": 62.5,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the rough fabric scratching against his stubble. Outside the thick, salt-crusted glass of the lantern room, the Atlantic was a churning cauldron of black and white foam. Rain lashed against the panes in horizontal sheets, blurring the world into a smear of gray and shadow.\n\nInside, however, it was warm. The smell of kerosene and old wood hung heavy in the air, a scent that had defined his life for thirty years.\n\nElias was not a man of many words. He was a man of many tasks. His hands, mapped with scars from broken glass and rusted iron, moved with a rhythm that predated his memory. He checked the clockwork mechanism of the Fresnel lens, ensuring the rotation was smooth, silent, and precise. *Tick. Tick. Tick.* The sound was the heartbeat of the coast."
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 135,
+  "drafts_offered": 184,
+  "tok_s": 85.2,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 78.1,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 80.5,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 59,
+  "repeats": 0,
+  "drafts_accepted": 13,
+  "drafts_offered": 48,
+  "tok_s": 46.6,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAntelope    \nBaboon      \nBadger     \nBeaver       \nBear        \nBird         \nBuffalo     \n\nCat          \nCheetah      \n\nChimpanzee \nClam         \n\n\nCrocodile   \n\n\n\nDolphin      \n\n\n\n\nDonkey       <|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 30,
+  "drafts_offered": 101,
+  "tok_s": 51.3,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral vaults below Elias\u2019s boots smelled faintly brine and old coffee as he climbed spiral stairs worn smooth by decades feet his hands were calloused from gripping iron railings that had been polished to a dull shine\n\nElias was not merely keeping light alive; He kept order intact. Every night, at precisely 18:07 hours local time\u2014adjusted for leap seconds and magnetic drift\u2014he wound the clockwork mechanism inside brass housing behind Fresnel lens This wasn\u2019t vanity It\u2019s necessity The rotation speed must remain constant or ships lose their bearing in fog where compasses fail\n\nHe spoke softly to himself while working, murmuring numbers aloud because silence made him"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 67,
+  "repeats": 0,
+  "drafts_accepted": 14,
+  "drafts_offered": 68,
+  "tok_s": 41.2,
+  "text": "The wind didn\u2019t just blow at Blackwood Rock; it screamed through the gaps in his teeth, like he had no mouth to hold back its fury anymore that night.\u201d Elara pulled her wool coat tighter around herself as she climbed spiral staircase of **Blackrock Lighthouse**. Her boots were slick with salt spray from previous days.<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 91,
+  "repeats": 0,
+  "drafts_accepted": 26,
+  "drafts_offered": 54,
+  "tok_s": 48.8,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKoala      \nKangaroo        \nHippopotamus          \nRhinoceros            \nCamel                 \nCheetah               \nLeopard                \nJaguar                  \nBear                    \nWolf                     \nFox                      \nDog                       \nCat                        \nMouse                   \nRat                         \nSquirrel              \nDeer                      \n\nAntelope                \n\nBuffalo                 \n\n\nBison                   \n\n\n\nElk                    \n\n\n\n\nMoose                  <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 11,
+  "tok_s": 68.8,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 32,
+  "drafts_offered": 107,
+  "tok_s": 52.3,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant storybook that weaves together geology with biology as wellas human culture into an epic narrative spanning billions years from its earliest beginnings in primordial times through modern-day ecological crises The vast expanse covers more than seventy percent Earth\u2019s surface area making up nearly all life support systems on our planet yet despite being so large humans have only explored less five thousand square kilometers out total 361 million km\u00b2 which means over ninety-nine point nine eight six per cent unexplored territory This makes oceans arguably larger unknown territories compared space exploration where NASA estimates they\u2019ve mapped roughly ten billion stars versus current marine surveys covering just two hundred fifty-six species documented globally against estimated millions undiscovered organisms including deep-sea creatures like anglerfish or vampire squid whose existence was first recorded centuries ago but still remain poorly understood due extreme pressures depths exceeding eleven miles below Pacific Trenches Mariana located near Guam island chain reaching maximum depth known today at approximately thirty-seven feet under pressure equivalent atmospheric levels causing structural collapse if unprotected by specialized vessels capable handling such conditions without failure mechanisms designed specifically for submersible operations operating within these zones require titanium hulls rated withstand forces greater gravity acting upon objects falling vertically downward acceleration rates calculated using Newtonian physics principles applied universally across gravitational fields regardless medium density variations present throughout different layers stratosphere troposphere mesospheres thermoclines etceterally speaking however when discussing aquatic environments instead air masses replace gases dissolved oxygen nitrogen arg"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-hp1-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 94,
+  "drafts_offered": 169,
+  "tok_s": 43.0,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was seventy-three years old, and he had been the keeper of the Blackwood Point Light for forty of those years. The other thirty-three had been spent in a small, drafty cottage that smelled perpetually of kerosene and damp wood.\n\nFor decades, Elias had tended the great Fresnel lens, polishing the glass prisms until they gleamed like diamonds, winding the clockwork mechanism that turned the beam, and trimming the wick of the oil lamp. It was a job of patience, of precision, of silence. The light was his charge. The sea was his neighbor.\n\nBut this morning, the sea was not his neighbor. It was his enemy.\n\nThe storm had been forecasted for three days, but it had arrived in one. The barometric pressure had dropped so sharply that Elias\u2019s ears popped."
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 136,
+  "drafts_offered": 178,
+  "tok_s": 63.7,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 56.8,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 57.7,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 94,
+  "repeats": 0,
+  "drafts_accepted": 24,
+  "drafts_offered": 49,
+  "tok_s": 43.3,
+  "text": "Lion  \nTiger   \nElephant    \nGiraffe     \nZebra       \nHippopotamus \nRhinoceros      \nCheetah         \nLeopard        \nJaguar          \nPuma           \nWolf            \nFox             \nBear              \nDeer                \nElk                 \nMoose               \nAntelope                  \nBison                   \nBuffalo                    \nCamel                     \nDonkey                      \nGoat                       \nSheep                        \nCow                         \nDog                          \nCat                           \nRabbit                            \nMouse                             \nHamster                               \nGuinea Pig<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 35,
+  "drafts_offered": 108,
+  "tok_s": 41.4,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral vaults below Elias\u2019s boots smelled faintly brine and old coffee as he climbed spiral stairs worn smooth by decades feet his hands were calloused from gripping iron railings that had been polished to a dull shine.\n\nElias wasn\u2019t just keeping light; was guarding silence the beam swept across black water every seven seconds cutting through fog so thick it swallowed sound entirely for three nights straight no ships answered its signal not even fishing trawlers seeking shelter they\u2019d long abandoned this stretch of coast where currents shifted unpredictably pulling debris from sunken wrecks into tangled heaps that snagged anchors and tore hulls open like wounds.\n\nHe remembered the last time someone had"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 59,
+  "repeats": 0,
+  "drafts_accepted": 11,
+  "drafts_offered": 46,
+  "tok_s": 33.7,
+  "text": "The wind didn\u2019t just blow at Blackwood Rock; it screamed through the gaps in his teeth, trying to pry them apart like loose planks of weathered oak outside your own mouth again and then you are going away from home is fine but for now let me write a proper story.<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 60,
+  "repeats": 0,
+  "drafts_accepted": 17,
+  "drafts_offered": 48,
+  "tok_s": 43.2,
+  "text": "Aardvark  \nAntelope allonax (Oryx) \u2013 *Note: Using standard names for clarity*  \n\nLet me provide a clean list of exactly **1** animal name on each single physical text file/terminal output stream with no other characters except the newline (\\n).<|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 11,
+  "tok_s": 48.9,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 27,
+  "drafts_offered": 105,
+  "tok_s": 40.2,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant machine for making life possible on Earth.\" \u2014 *David Attenborough* \n\nThis quote captures perhaps more than any other single sentence in human language: that which we call \"the Sea\" or \u201cOcean\u201d was never just an element\u2014it became civilization itself through its capacity as both barrier (to land) but also conduit between lands\u2014and thus gave rise directly from being part nature\u2019s own evolutionary engine into becoming humanity's greatest historical actor across all time scales since first emergence until now present-day future projections still ongoing today within our current era context window limits allowed here by system constraints set above threshold levels required before output generation begins below line break marker inserted immediately after final punctuation mark ending entire preceding paragraph block structure defined earlier during initial drafting phase prior execution stage completed successfully without errors detected internally via internal validation routines executed automatically upon completion cycle termination signal received end-of-file condition triggered correctly according standard protocol specifications followed strictly adhered-to throughout process flow control logic implemented using structured programming constructs compatible with target runtime environment configuration parameters specified herein explicitly declared beforehand at top-level scope declaration section located near beginning source code file header comments included inline documentation style guidelines observed consistently applied uniformly everywhere else similarly formatted sections appearing later down document body text area scrolling vertically downward directionally aligned left margin justified right edge wrapped around fixed-width column boundaries determined dynamically based available screen real estate measured pixel-per-unit scaling factor adjusted responsively depending viewport dimensions reported back client-side rendering pipeline components responsible"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-kv-same24-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 94,
+  "drafts_offered": 167,
+  "tok_s": 62.7,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the fabric stiff with salt and ice. Outside the thick glass of the lantern room, the Atlantic was a churning cauldron of black and white, the waves crashing against the granite base with a violence that shook the very foundations of the tower. It was November, 1924, and the storm had been raging for three days.\n\nElias was not a man who feared the dark. He was a man who feared the silence.\n\nHe checked the clock on the wall. 11:42 PM. The rotation of the great Fresnel lens was steady, a rhythmic *whirrr-click, whirrr-click* that seemed to hold the chaos of the ocean at bay. The beam swept out, a blade of pure, concentrated light cutting through the rain and sleet, illuminating the jagged teeth of the reef below.\n\n"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 136,
+  "drafts_offered": 179,
+  "tok_s": 81.3,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 71.0,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 71.9,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 61,
+  "repeats": 0,
+  "drafts_accepted": 18,
+  "drafts_offered": 52,
+  "tok_s": 47.9,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAntelope    \nBaboon      \nBadger     \nBeaver       \nBear        \nBird         \nBoar          \nBuffalo \nButterfly \n\nCamel           \nCat            \nCheetah        \n\nDolphin         \n\n\nElephant        \n\n\n\nFalcon          <|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 29,
+  "drafts_offered": 122,
+  "tok_s": 47.4,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral vaults below Elias\u2019s boots were heavy with salt and exhaustion as he climbed one hundred twelve steps to reach his post at dawn every morning for thirty years. His name was written on a brass plaque by door: **Elias Thorne**.\n\nHe wasn\u2019t just keeping light; He kept time, weathered hands gripping the cold iron railing until knuckles turned white against frostbite and windburn that never healed fully because they were always there waiting to be felt again when he stepped outside into storm or calm alike. The lens above him was a Fresnel giant\u2019s eye made of hand-ground crystal prisms each one polished daily with chamois cloth soaked in whale"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 65,
+  "repeats": 0,
+  "drafts_accepted": 11,
+  "drafts_offered": 58,
+  "tok_s": 42.6,
+  "text": "The wind didn\u2019t just blow at Cape Hollow; it screamed through the gaps in my cottage\u2019s shutters, rattling them like loose teeth until they slammed against their hinges every night for eleven months of winter and summer alike.\u201d I had never seen calm water here\u2014only gray churning seas that swallowed ships smaller than themselves<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 101,
+  "repeats": 0,
+  "drafts_accepted": 32,
+  "drafts_offered": 53,
+  "tok_s": 49.4,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKoala      \nKangaroo        \nCheetah         \nHippopotamus          \nRhinoceros            \nLeopard             \nJaguar              \nWolf                \nFox                 \nBear                  \nDeer                    \nOtter                   \nBeaver                      \nBadger                        \nMongoose                          \nMeerkat                           \nHyena                             \nJackal                            \nAntelope                              \nBison                               \nElk                                 \nCaribou                                   \nLlama                                    \nAlpace                                     <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 12,
+  "tok_s": 64.7,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 52,
+  "drafts_offered": 148,
+  "tok_s": 49.9,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant machine for making life possible on Earth.\" \u2014 *David Attenborough* \n\nThis quote captures perhaps more than any other single sentence in human language: that which we call \"the Sea\" or \u201cOcean\u201d was never just an element within our planet\u2019s geography but rather its most fundamental biological engine\u2014a living system whose existence predates humanity by hundreds millions years yet continues shaping every aspect from climate regulation through evolutionary biology down even into modern industrial logistics today's global supply chains depend upon them directly indirectly via atmospheric pressure systems weather patterns migration routes food webs carbon cycles oxygen generation thermal buffering tidal forces salinity gradients depth stratification light attenuation currents circulation gyres upwelling zones thermoclines halothermics bathymetric features continental shelves abyssal plains trenching ridges volcanic arcs island formations atoll structures seamounts submarine canons canyon walls cliff faces overhang ledge platforms reef flats fore-reef back-lagoon inner-shelf outer-slope drop-offs pinnacle pinnacles spires columns arches tunnels caves grottos swim-through passages chimney vents blowholes fissures cracks crevices joints faults fractures breaks gaps holes openings void spaces hollow chambers cavernous interiors subterranean networks underground rivers lakes pools springs wells shaft pits sinks drains sewers pipes conduits tubes channels gutters ditches furrows grooves rills runnels streams brooks rivulets creeks tributaries branches forks splits divergences con"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-kv-single-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 94,
+  "drafts_offered": 167,
+  "tok_s": 49.2,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the fabric stiff with salt and ice. Outside the thick glass of the lantern room, the Atlantic was a churning cauldron of black and white, the waves crashing against the granite base with a violence that shook the very foundations of the tower. It was November, 1924, and the storm had been raging for three days.\n\nElias was not a man who feared the dark. He was a man who feared the silence.\n\nHe checked the clock on the wall. 11:42 PM. The rotation of the great Fresnel lens was steady, a rhythmic *whirrr-click, whirrr-click* that seemed to hold the chaos of the ocean at bay. The beam swept out, a blade of pure, concentrated light cutting through the rain and sleet, illuminating the jagged teeth of the reef below.\n\n"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 136,
+  "drafts_offered": 179,
+  "tok_s": 76.2,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 73.1,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 73.4,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 61,
+  "repeats": 0,
+  "drafts_accepted": 18,
+  "drafts_offered": 52,
+  "tok_s": 47.1,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAntelope    \nBaboon      \nBadger     \nBeaver       \nBear        \nBird         \nBoar          \nBuffalo \nButterfly \n\nCamel           \nCat            \nCheetah        \n\nDolphin         \n\n\nElephant        \n\n\n\nFalcon          <|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 29,
+  "drafts_offered": 122,
+  "tok_s": 47.0,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral vaults below Elias\u2019s boots were heavy with salt and exhaustion as he climbed one hundred twelve steps to reach his post at dawn every morning for thirty years. His name was written on a brass plaque by door: **Elias Thorne**.\n\nHe wasn\u2019t just keeping light; He kept time, weathered hands gripping the cold iron railing until knuckles turned white against frostbite and windburn that never healed fully because they were always there waiting to be felt again when he stepped outside into storm or calm alike. The lens above him was a Fresnel giant\u2019s eye made of hand-ground crystal prisms each one polished daily with chamois cloth soaked in whale"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 65,
+  "repeats": 0,
+  "drafts_accepted": 11,
+  "drafts_offered": 58,
+  "tok_s": 42.3,
+  "text": "The wind didn\u2019t just blow at Cape Hollow; it screamed through the gaps in my cottage\u2019s shutters, rattling them like loose teeth until they slammed against their hinges every night for eleven months of winter and summer alike.\u201d I had never seen calm water here\u2014only gray churning seas that swallowed ships smaller than themselves<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 101,
+  "repeats": 0,
+  "drafts_accepted": 32,
+  "drafts_offered": 53,
+  "tok_s": 50.0,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKoala      \nKangaroo        \nCheetah         \nHippopotamus          \nRhinoceros            \nLeopard             \nJaguar              \nWolf                \nFox                 \nBear                  \nDeer                    \nOtter                   \nBeaver                      \nBadger                        \nMongoose                          \nMeerkat                           \nHyena                             \nJackal                            \nAntelope                              \nBison                               \nElk                                 \nCaribou                                   \nLlama                                    \nAlpace                                     <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 12,
+  "tok_s": 64.6,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 52,
+  "drafts_offered": 148,
+  "tok_s": 49.8,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant machine for making life possible on Earth.\" \u2014 *David Attenborough* \n\nThis quote captures perhaps more than any other single sentence in human language: that which we call \"the Sea\" or \u201cOcean\u201d was never just an element within our planet\u2019s geography but rather its most fundamental biological engine\u2014a living system whose existence predates humanity by hundreds millions years yet continues shaping every aspect from climate regulation through evolutionary biology down even into modern industrial logistics today's global supply chains depend upon them directly indirectly via atmospheric pressure systems weather patterns migration routes food webs carbon cycles oxygen generation thermal buffering tidal forces salinity gradients depth stratification light attenuation currents circulation gyres upwelling zones thermoclines halothermics bathymetric features continental shelves abyssal plains trenching ridges volcanic arcs island formations atoll structures seamounts submarine canons canyon walls cliff faces overhang ledge platforms reef flats fore-reef back-lagoon inner-shelf outer-slope drop-offs pinnacle pinnacles spires columns arches tunnels caves grottos swim-through passages chimney vents blowholes fissures cracks crevices joints faults fractures breaks gaps holes openings void spaces hollow chambers cavernous interiors subterranean networks underground rivers lakes pools springs wells shaft pits sinks drains sewers pipes conduits tubes channels gutters ditches furrows grooves rills runnels streams brooks rivulets creeks tributaries branches forks splits divergences con"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-kv-x2-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 97,
+  "drafts_offered": 176,
+  "tok_s": 73.0,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was the last of the line, the only man left on the jagged spine of Widow\u2019s Rock. For forty years, he had tended the light. For forty years, he had listened to the ocean\u2019s endless, hungry whisper.\n\nTonight was different. The storm wasn\u2019t just a storm; it was a *tempest*. The waves didn\u2019t crash; they *shattered*. They threw themselves against the granite base of the lighthouse with the force of a battering ram, sending tremors up through the soles of Elias\u2019s boots.\n\nHe climbed the spiral stairs, one hundred and twelve steps of iron and rust. His hand, calloused and scarred from decades of polishing the brass railing, found the familiar groove in the metal. He didn\u2019t look up. He didn\u2019t look down. He looked *forward*."
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 135,
+  "drafts_offered": 176,
+  "tok_s": 102.5,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 90.9,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 91.7,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 118,
+  "repeats": 0,
+  "drafts_accepted": 38,
+  "drafts_offered": 103,
+  "tok_s": 62.8,
+  "text": "Aardvark  \nAlbatross   \nAlligator    \nAntelope     \nBaboon       \nBadger      \nBeaver        \nBear          \nBird of Paradise \nBuffalo         \nCamel           \nCheetah            \nChimpanzee             \nCrane              \nCrocodile                \nDeer                 \nDolphin               \nDonkey                  \nElephant                   \nFlamingo                    \nFox                       \nGiraffe                     \nGoat                      \nHippopotamus  \nIguana       \nJaguar        \nKangaroo      \nKoala          \nLion         \nLeopard     \nMonkey    \nMoose   \nOstrich \nOtter\nPanda<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 58,
+  "repeats": 0,
+  "drafts_accepted": 9,
+  "drafts_offered": 63,
+  "tok_s": 50.4,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty jail cell below sea level three hundred feet down to where Elias kept his boots dry and ready for emergencies when he had no time even think about changing them.<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 140,
+  "repeats": 0,
+  "drafts_accepted": 22,
+  "drafts_offered": 90,
+  "tok_s": 50.1,
+  "text": "The wind didn\u2019t just blow at Blackwood Rock; it screamed through the gaps in his teeth, demanding entry into every crack and crevice of Stone Harbor Lighthous eKeepers have always been solitary figures against isolation\u2019s greatest adversary: time itself Elias Thorne knew this intimately. For thirty years he had kept watch over one hundred thousand ships each night scanning for lights that might signal disaster or hope\n\nHis job was simple but deadly serious He checked weather logs every day without fail The foghorn echoed through the tower three times before dawn Each morning at 4:02 AM Elias would climb eighty-two steps to reach his office where coffee cooled beside a single candle flickering in drafts from below<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 96,
+  "repeats": 0,
+  "drafts_accepted": 31,
+  "drafts_offered": 69,
+  "tok_s": 58.9,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKangaroo      \nKoala        \nHippo         \nRhino          \nCheetah            \nLeopard             \nJaguar              \nPanther               \nCougar                \nBear                  \nWolf                    \nFox                     \nDeer                      \nElk                        \nMoose                       \nBison                         \nBuffalo                          \nAntelope                           \nCamel                            \nDonkey                              \nGoat                                \nSheep                                  \nCow                                    \nPig                                     \nHorse                                       <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 13,
+  "tok_s": 71.2,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 37,
+  "drafts_offered": 119,
+  "tok_s": 61.3,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant machine for making life possible on Earth\u2019s surface.\u201d \u2014 *This quote by Rachel Carson serves as an apt introduction.* The story we tell today begins with that first drop falling into primordial pools billions years ago\u2014but what follows next? What happens when those drops become oceans themselves\u2014when they begin moving across continents carrying sediment from mountains down rivers until finally reaching their final resting place at last\u2014the deep abyssal plains where sunlight cannot reach nor pressure crushes everything else alive there below depths greater than six kilometers beneath waves crashing against shores above them all around world forever changing shape over time due plate tectonics shifting landmass slowly apart together again repeatedly throughout geological epochs spanning hundreds millions or even thousands million+ yrs since formation early Hadean era through Archeozoic eons onward present Holocene epoch still ongoing now right here under current conditions observed via satellites monitoring global climate patterns affecting currents temperatures salinity levels biodiversity distribution human impacts altering ecosystems directly indirectly causing extinction events locally globally recently historically prehistorically anciently mythologically culturally spiritually philosoph scientifically mathematically physically chem biol geo astro cosmological sociopolitical economic environmental ecological evolutionary genetic linguistic literary artistic musical cinematic theatrical photographic journalistic digital analog virtual real tangible intangible metaphysical ontosemantic semiotic symbolic allegorical narrative descriptive analytical interpretive evaluative critical reflective introspective contemplatory ponderous thoughtful considerate attentive observant perceptive insightful intelligent knowledgeable informed educated literati learned skilled capable"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-nocv-x2-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 87,
+  "drafts_offered": 170,
+  "tok_s": 71.4,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the rough fabric scratching against his stubble. Outside the thick, salt-crusted glass of the lantern room, the Atlantic was a churning cauldron of black and white foam. Rain lashed against the panes in horizontal sheets, blurring the world into a smear of gray and green.\n\nInside, however, it was warm. The smell of kerosene and old wood hung heavy in the air, a scent that had defined his life for thirty years.\n\nElias was not a man of many words. He was a man of many tasks. His hands, mapped with scars from rope burns and broken glass, moved with a rhythm that predated his memory. He checked the clockwork mechanism of the Fresnel lens, listening for the faint, rhythmic *tick-tick-tick* that signaled the rotation was true. If the rhythm faltered, ships"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 136,
+  "drafts_offered": 178,
+  "tok_s": 104.1,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 90.1,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 94.8,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 76,
+  "repeats": 0,
+  "drafts_accepted": 20,
+  "drafts_offered": 64,
+  "tok_s": 61.6,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAntelope    \nBaboon      \nBadger     \nBeaver       \nBear         \nBird        \nBuffalo          \nButterfly \nCamel           \nCat            \nCaterpillar \n\nCentipede    \n\nChameleon   \n\nCheetah     \n\nChicken      \n\nCricket       \n\nCrocodile     \n\n\nDolphin       \n\n\n\nDonkey        \n\n\n\n\nDragon<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 21,
+  "drafts_offered": 90,
+  "tok_s": 59.2,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty jail cell below sea level to above cloud height at once\u2014no matter how high you climbed or deep into its foundations down through rock and saltwater. It was November again when Elias first noticed that his logbook had begun recording entries he hadn\u2019t written yet: *Storm surge expected Tuesday, but the sky is clear.*\n\nHe stopped writing for a moment to listen\u2014not with ears alone\u2014but by pressing both palms flat against cold iron walls where condensation formed in slow spirals like fingerprints left behind after touching something invisible. The metal hummed faintly beneath his skin as if remembering old storms long past their dates on paper calendars torn from newspapers dated three years ahead of today"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 17,
+  "drafts_offered": 63,
+  "tok_s": 52.6,
+  "text": "The wind didn\u2019t just blow at Blackwood Rock; it screamed through the gaps in his teeth, trying to get inside where he had spent thirty years refusing entry except for family and friends like him who were old enough too know better than young men ever could be again about what safety meant. It was November 2014 when Elias Thorne first stepped onto that spiral staircase with only one boot on because his other foot hurt after walking from the harbor office all morning long before anyone else thought he needed shoes off there for work even if nobody saw them do it but everyone knew why so they did not ask him anymore once this happened already twice last winter near same season now here again today too soon then tomorrow maybe never will come back unless someone sends message tonight via radio signal code Morse dot dash pattern"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 94,
+  "repeats": 0,
+  "drafts_accepted": 29,
+  "drafts_offered": 60,
+  "tok_s": 58.8,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nKangaroo       \nPanda        \nKoala          \nCheetah      \nWolf         \nFox           \nBear            \nMonkey             \nDolphin              \nWhale                \nShark                 \nOctopus               \nJellyfish                  \nCrab                    \nSpider                   \nRabbit                     \nCat                      \nDog                       \nHorse                        \nCow                          \nSheep                            \nGoat                              \nChicken                                \nRooster                                   \nTurkey                                  \nEagle                                    \nFalcon                                     <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 13,
+  "tok_s": 71.8,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 52,
+  "drafts_offered": 148,
+  "tok_s": 61.8,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant machine for making life possible on Earth.\u201d \u2014 *David Attenborough* \n\nThis quote captures perhaps more than any other single sentence in human language: that which we call \u201cthe Sea\u201d or \"Ocean\" (terms often used interchangeably but distinct geographically) was never just an element within our planet\u2019s geography\u2014it became its engine itself\u2014a living system whose evolution shaped everything else around us from climate patterns down through evolutionary biology right up into modern civilization's economic structures cultural identities political boundaries ecological systems scientific understanding technological capabilities philosophical frameworks psychological experiences artistic expressions literary metaphors cinematic narratives musical compositions architectural designs urban planning strategies agricultural practices industrial processes medical discoveries legal precedents educational curricula religious traditions spiritual beliefs mythological stories folkloric tales poetic verses dramatic scripts visual arts photographic images digital media social networks behavioral norms emotional responses cognitive models decision-making algorithms predictive analytics statistical probabilities probabilistic reasoning causal inference counterfactual thinking hypothetical scenarios normative ethics deontal logic modal semantics temporal dynamics spatial relations topographical features hydrological cycles meteor phenomena atmospheric chemistry geological formations tectonic shifts volcanic eruptions seismic events tsunamis hurricanes cyclones monsoons typhoons droughts floods storms winds currents waves swells breakers surf foam spray mist fog rain snow ice glaciers berg floes drift pack-ice brine salinity pH alkalines acidity hardness softness temperature pressure depth light darkness color hue saturation brightness contrast luminance chroma tone value shade tint reflection"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/norepeat-lim-reg-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 88,
+  "drafts_offered": 187,
+  "tok_s": 58.7,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was seventy years old, but he moved with the grace of a man who had spent decades on a narrow spiral staircase. His hands, mapped with veins like the roots of an ancient oak, gripped the cold iron railing.\n\nFor forty years, Elias had tended the light at the end of the world. Not the end of the map, but the end of the land. Here, the Atlantic threw itself against the granite cliffs with such violence that the spray reached the gallery, thirty feet above the rocks.\n\nHe checked the clock. 11:42 PM. The fog was thickening, a gray soup that swallowed the horizon. In the old days, he would have climbed the stairs to the lantern room, polished the Fresnel lens, and wound the clockwork mechanism that turned the great beam. But the automation had changed"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 135,
+  "drafts_offered": 179,
+  "tok_s": 80.7,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 71.8,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 73.0,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 61,
+  "repeats": 0,
+  "drafts_accepted": 19,
+  "drafts_offered": 49,
+  "tok_s": 49.1,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAntelope    \nBaboon      \nBadger     \nBeaver       \nBear        \nBird         \nBoar          \nBuffalo \nButterfly \n\nCamel           \nCat            \nCheetah        \n\nDolphin         \n\n\nElephant        \n\n\n\nFlamingo<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 80,
+  "repeats": 0,
+  "drafts_accepted": 6,
+  "drafts_offered": 52,
+  "tok_s": 46.2,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral hall outside my window at midnight on November third this year\u2014so I pulled up to check what was happening with that storm and why they were calling me out there when all we had left for tonight\u2019s shift is one last pass through our logs before dawn breaks.<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 17,
+  "drafts_offered": 83,
+  "tok_s": 43.9,
+  "text": "The wind didn\u2019t just blow at Cape Harrow; it screamed through the gaps in its teeth, hungry and relentless for three centuries of stone salt-spray history that had long since forgotten to be polite or gentle with itself against anything smaller than God Himself was willing acknowledge as divine protection from drowning souls who were already lost before they ever touched water.\n\nElias knew this rhythm better he himself could hear it sometimes; his boots worn smooth by pacing the spiral stair\u2019s center-post groove where thousands upon countless steps had rubbed away texture until all that remained beneath sole leather now felt like polished bone carved out slowly but surely over time into something more hollow than wood ever really should be allowed become when used so often enough within such confined circular spaces meant only for vertical ascent down always descending again back upward never truly leaving"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 62,
+  "repeats": 0,
+  "drafts_accepted": 18,
+  "drafts_offered": 31,
+  "tok_s": 49.1,
+  "text": "Lion  \nTiger  \n\nElephant  \n  \nGiraffe  \n\n\n\nHippopotamus    \nZebra   \nRhinoceros   \n\n\nPanda Bear    \n\n\n\n\nKangaroo     \nKoala      \nSloth      \n\n\n\n\n\nCheetah       \nLeopard        \nPanther        \n\n\n\n\n\n\nWolf              \nFox                \nBear                 <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 13,
+  "tok_s": 62.7,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 47,
+  "drafts_offered": 153,
+  "tok_s": 52.2,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant machine for making life possible on Earth.\" \u2014 *David Attenborough* \n\nThis quote captures perhaps more than any other single sentence in human language: that which we call \"the Sea\" or \u201cOcean\u201d was never just an element within our planet\u2019s geography but rather its most fundamental biological engine\u2014a living system whose existence predates humanity by billions years yet continues shaping every aspect from climate regulation through evolutionary biology all way down modern industrial supply chains today's global shipping lanes carry cargo across vast distances because they were once crossed only slowly via coastal hopping routes used since antiquity until steamships revolutionized travel during late nineteenth century when diesel engines replaced wind power entirely allowing container ships carrying thousands TEUs (Twenty-foot Equivalent Units) per voyage crossing entire Pacific basins without stopping at intermediate ports anymore unlike earlier sailing vessels needing favorable winds currents seasonal patterns dictating departure windows based upon monsoon calendars established centuries ago before technological advances changed those constraints fundamentally altering logistics networks worldwide thereby transforming economic structures globally while simultaneously creating ecological disruptions due invasive species transported unintentionally alongside legitimate goods causing cascading ecosystem collapses locally where non-native organisms outcompete native fauna disrupting food webs leading population declines affecting biodiversity indices measured scientifically using standardized metrics like Shannon-Wiener diversity index calculations applied consistently over decades revealing trends showing significant decreases post-introduction events correl strongly with introduction dates matching historical records documenting first sightings confirmed taxonomically classified formally according International Code Nomenclature rules"
+ }
+]

bench/results/2026-09-29-layer-split-limits/data/pp-lim-cv-pp-x2.json

@@ -0,0 +1,84 @@
+{
+ "label": "lim-cv-pp-x2",
+ "extra": [
+  "--layer-split",
+  "auto",
+  "--control-vector-scaled",
+  "C:\\Users\\ribbon\\strata-test\\Strata\\data\\experimental-speed-projection\\Qwen3.8-Flash-Next-experimental-speed-projection.gguf:1.0",
+  "--control-vector-layer-range",
+  "4",
+  "44",
+  "--cvec-mode",
+  "project",
+  "--cvec-dir",
+  "per-layer"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 15925,
+   "prompt_ms": 8783.1,
+   "pp_tok_s": 1813.1,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27474,
+   "prompt_ms": 13379.7,
+   "pp_tok_s": 2053.4,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3652.4,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 246.8,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 77.0,
+   "reps": [
+    77.0
+   ],
+   "hit_rate": 0.9718150054476462,
+   "text": "The wind didn\u2019t just howl; it screamed. It tore through the gaps in the iron railing of the Lantern Room, carrying the scent of brine and rotting kelp. Elias pulled his woolen collar up to his chin, his fingers numb and clumsy inside thick leather gloves.\n\nHe was the last man on the island. The automated systems had failed three winters ago, a glitch in the solar panel array that no engineer had been able to fix before the supply boat stopped running. Now, it was just him, the light, and the sea.\n\nElias climbed the spiral stairs, his boots heavy with salt spray. The iron steps groaned under his weight, a sound that seemed to mock the storm outside. He reached the top, the glass walls vibrating with the force of the gale.\n\nInside the lantern room, the great Fresnel lens sat like a dormant eye. It was cold, dark, and waiting.\n\nElias moved to the control panel. His"
+  },
+  {
+   "name": "code",
+   "tok_s": 103.6,
+   "reps": [
+    103.6
+   ],
+   "hit_rate": 0.957731667226356,
+   "text": "```python\nimport csv\nfrom typing import List, Dict\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, str]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile(mode='w', suffix='.csv', delete=False"
+  }
+ ],
+ "stage_timing": [],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split auto)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: layer split auto: CUDA0 84 SMs at 2.81 GHz -> 0.31 ms per layer, 8.30 GiB for experts",
+  "strata generate: layer split auto: CUDA1 82 SMs at 1.86 GHz -> 0.48 ms per layer, 14.14 GiB for experts",
+  "strata generate: layer split auto: K=22 - predicted 19.7 ms per decode window; the caches hold 11298 of 12288 profiled pairs (~99.7% of the routed mass)",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 3479 slots, 6.22 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 22-47, expert cache 6656 slots (13.34 GiB), 6656 of its 6656 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-21",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-21 (CUDA0), 22-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 91.6% (5554 hits / 6062 lookups)",
+  "strata serve: decode expert cache hit rate: 95.3% (3592 hits / 3771 lookups)",
+  "strata serve: decode expert cache hit rate: 96.5% (3650 hits / 3782 lookups)",
+  "strata serve: decode expert cache hit rate: 96.3% (25027 hits / 25980 lookups)",
+  "strata serve: decode expert cache hit rate: 96.5% (3201 hits / 3318 lookups)",
+  "strata serve: decode expert cache hit rate: 97.2% (129334 hits / 133085 lookups)",
+  "strata serve: decode expert cache hit rate: 95.8% (108375 hits / 113158 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split-limits/data/pp-lim-kv-pp-single.json

@@ -0,0 +1,76 @@
+{
+ "label": "lim-kv-pp-single",
+ "extra": [
+  "--max-context",
+  "131072",
+  "--kv-resident",
+  "32768"
+ ],
+ "gpus": "0",
+ "needles": [
+  {
+   "tokens": 15925,
+   "prompt_ms": 10706.3,
+   "pp_tok_s": 1487.4,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27474,
+   "prompt_ms": 15256.7,
+   "pp_tok_s": 1800.8,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 59725,
+   "prompt_ms": 32515.5,
+   "pp_tok_s": 1836.8,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 4006.4,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 276.2,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 66.6,
+   "reps": [
+    66.6
+   ],
+   "hit_rate": 0.7101864314556545,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the fabric stiff with salt and ice. Outside the thick glass of the lantern room, the Atlantic was a churning cauldron of black and white, the waves crashing against the granite base with a violence that shook the very foundations of the tower. It was November, 1924, and the storm had been raging for three days.\n\nElias was not a man who feared the dark. He was a man who feared the silence.\n\nHe checked the clock on the wall. 11:42 PM. The rotation of the great Fresnel lens was steady, a rhythmic *whirrr-click, whirrr-click* that seemed to sync with his own heartbeat. He had been doing this for forty years. His hands were mapped with scars from rope burns and broken glass, his eyes permanently squinted against the glare of the bulb."
+  },
+  {
+   "name": "code",
+   "tok_s": 74.0,
+   "reps": [
+    74.0
+   ],
+   "hit_rate": 0.5873783774988882,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [],
+ "engine_lines": [
+  "strata generate: expert cache auto: 8.99 GiB free, 700 MiB reserved -> 3349 slots",
+  "strata generate: expert cache 2434 slots, 4.67 GiB of VRAM; policy is",
+  "strata serve: the prompt path borrows 1944 cache slots (3.72 GiB)",
+  "strata serve: decode expert cache hit rate: 41.8% (1746 hits / 4176 lookups)",
+  "strata serve: decode expert cache hit rate: 50.4% (1512 hits / 3000 lookups)",
+  "strata serve: decode expert cache hit rate: 54.9% (1689 hits / 3077 lookups)",
+  "strata serve: decode expert cache hit rate: 60.3% (1938 hits / 3213 lookups)",
+  "strata serve: decode expert cache hit rate: 58.3% (11146 hits / 19121 lookups)",
+  "strata serve: decode expert cache hit rate: 66.1% (2201 hits / 3331 lookups)",
+  "strata serve: decode expert cache hit rate: 71.0% (74397 hits / 104757 lookups)",
+  "strata serve: decode expert cache hit rate: 58.7% (54151 hits / 92191 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split-limits/data/pp-lim-kv-pp-x2.json

@@ -0,0 +1,87 @@
+{
+ "label": "lim-kv-pp-x2",
+ "extra": [
+  "--max-context",
+  "131072",
+  "--kv-resident",
+  "32768",
+  "--layer-split",
+  "auto"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 15925,
+   "prompt_ms": 9452.6,
+   "pp_tok_s": 1684.7,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27474,
+   "prompt_ms": 14042.4,
+   "pp_tok_s": 1956.5,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 59725,
+   "prompt_ms": 30345.9,
+   "pp_tok_s": 1968.1,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3806.1,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 245.6,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 75.8,
+   "reps": [
+    75.8
+   ],
+   "hit_rate": 0.9698055042592483,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the rough fabric scratching against his stubble. Outside the thick, salt-crusted glass of the lantern room, the Atlantic was a churning cauldron of black and white foam. Rain lashed against the panes in horizontal sheets, blurring the world into a smear of gray and shadow.\n\nInside, however, it was warm. The smell of kerosene and old wood hung heavy in the air, a scent that had defined Elias\u2019s life for thirty years. He stood before the great Fresnel lens, a cathedral of glass and brass that caught the light of the single, powerful bulb and fractured it into a thousand beams.\n\nHe checked the clock. 11:42 PM. The rotation was smooth, silent, a hypnotic sweep that painted the darkness with a slow, rhythmic pulse. *Flash. Dark. Flash. Dark.*\n\nE"
+  },
+  {
+   "name": "code",
+   "tok_s": 97.0,
+   "reps": [
+    97.0
+   ],
+   "hit_rate": 0.9562207324057541,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split auto)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 17.96 GiB free",
+  "strata generate: layer split auto: CUDA0 84 SMs at 2.81 GHz -> 0.31 ms per layer, 8.13 GiB for experts",
+  "strata generate: layer split auto: CUDA1 82 SMs at 1.86 GHz -> 0.48 ms per layer, 13.98 GiB for experts",
+  "strata generate: layer split auto: K=22 - predicted 19.7 ms per decode window; the caches hold 11199 of 12288 profiled pairs (~99.7% of the routed mass)",
+  "strata generate: expert cache auto: 10.30 GiB free, 700 MiB reserved -> 3277 slots",
+  "strata generate: expert cache 3409 slots, 6.09 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 22-47, expert cache 6656 slots (13.34 GiB), 6656 of its 6656 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-21",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-21 (CUDA0), 22-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 91.3% (5098 hits / 5585 lookups)",
+  "strata serve: decode expert cache hit rate: 94.7% (4000 hits / 4225 lookups)",
+  "strata serve: decode expert cache hit rate: 96.5% (4095 hits / 4245 lookups)",
+  "strata serve: decode expert cache hit rate: 97.8% (4183 hits / 4278 lookups)",
+  "strata serve: decode expert cache hit rate: 96.2% (27177 hits / 28250 lookups)",
+  "strata serve: decode expert cache hit rate: 97.6% (4177 hits / 4281 lookups)",
+  "strata serve: decode expert cache hit rate: 97.0% (130241 hits / 134296 lookups)",
+  "strata serve: decode expert cache hit rate: 95.6% (113534 hits / 118732 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split-limits/data/vision.json

@@ -0,0 +1,24 @@
+{
+ "answers": {
+  "single": [
+   " Red",
+   "42",
+   "A red circle, the number 42, and a blue square are arranged on a white background."
+  ],
+  "same24": [
+   " Red",
+   "42",
+   "A red circle, the number 42, and a blue square are arranged on a white background."
+  ],
+  "x2": [
+   " Red",
+   "42",
+   "A red circle, the number 42, and a blue square are displayed on a white background."
+  ]
+ },
+ "correct": {
+  "single": true,
+  "same24": true,
+  "x2": true
+ }
+}

bench/results/2026-09-29-layer-split/README.md

@@ -0,0 +1,84 @@
+# Layer split across GPUs (2026-09-29)
+
+**Rig (a community member's PC, "RIBPC"):**
+- Ryzen 9 9950X3D, 62 GB DDR5, Windows 11, CUDA 13.1.
+- RTX 5080 16 GB (x16), RTX 3090 24 GB (x4 gen4, ~6 GB/s), RTX 2080 Ti 22 GB (x4 gen3, Turing sm_75; the
+  experimental `STRATA_EXPERIMENTAL_SM75` build).
+
+**Model:** the Coder (GSQ-RCO IQ1_M, 48 layers x 256 experts = 12,288 expert pairs, ~23 GB of experts), 32K
+context, int8 KV, `--prefill auto`, `--spec 4`. It is the only model that fits the rig's RAM and disk.
+
+**Tools:** `rib_pp.py` (needle prompts at 2.7K/8K/16K/28K tokens with a code word at 50% depth, a 2-turn chat that
+must reuse its checkpoint, greedy decode of a story and a code prompt) and `rib_norepeat.py` (the Phase A
+no-repeat and sanity cases). Raw results are in `data/`.
+
+## Correctness
+
+| Check | Result |
+|---|---|
+| This branch vs the 0.1.20 release, one GPU (5080), fixed cache, all 10 cases | **10/10 byte-identical** |
+| This branch, one GPU vs the split on the same GPU (K=24, `--split-device 0`) | **10/10 byte-identical**: the hand-off is exact |
+| 5080 + 3090 (auto) and 5080 + 2080 Ti + 3090 (auto): no-repeat under presence 100 (greedy and sampled), 144 sanity | all pass, 0 repeats |
+| Needles at 2.7K-28K tokens, every configuration | all found |
+| 2-turn chat (the checkpoint restored on every GPU) | reused 6,631-6,632 of 6,660 tokens, answer right |
+
+Across GPUs the text is not byte-identical to one GPU: experts the CPU computed on one card run on a GPU in the split,
+and they round differently (the same effect as `--expert-cache`, bench/results/2026-09-27-cache-parity).
+
+## Speed
+
+**Prompt tok/s** at 16K / 28K tokens, and **decode tok/s** for story / code (greedy; three repeats averaged
+in the K sweep):
+
+| Configuration | Split | Prompt 16K | Prompt 28K | Decode story | Decode code | Decode hits |
+|---|---|---|---|---|---|---|
+| 0.1.20, 5080 alone | - | 2,045 | 2,005 | 83.2 | 88.2 | 70-84% |
+| this branch, 5080 alone | - | 2,017 | 1,974 | 83.1 | 88.0 | 70-84% |
+| this branch, 5080 alone, 3 repeats (adaptive swaps learn the prompt) | - | 1,726 | 1,970 | 86.7 | 105.2 | 91-92% |
+| 3090 alone | - | 1,253 | 1,211 | 71.7 | 92.9 | 86-93% |
+| 5080 + 3090 | K=18 | 1,547 | 1,694 | 77.2 | 103.8 | 99.9% |
+| 5080 + 3090 | K=22 | 1,840 | 2,074 | 80.2 | 106.4 | 99.8% |
+| 5080 + 3090 | K=24 | 1,978 | 2,258 | 83.5 | 109.5 | 99.5% |
+| **5080 + 3090** | **K=26** | **2,039** | **2,357** | **83.8** | **109.7** | 99.3% |
+| 5080 + 3090 | K=28 | 2,011 | 2,287 | 83.5 | 112.5 | 99.0% |
+| 5080 + 3090 | K=30 | 1,935 | 2,218 | 81.1 | 109.6 | 98% |
+| 5080 + 3090 | K=32 | 1,818 | 2,063 | 83.0 | 108.7 | 98% |
+| 5080 + 3090 | auto (K=22) | 2,037 | 2,073 | 79.5 | 108.9 | 99.7% |
+| 3090 + 5080 | auto (K=30) | 1,672 | 1,876 | 90.3 | 105.1 | 99.4% |
+| 5080 + 2080 Ti + 3090 | auto (22, 23) | 1,687 | 1,847 | 67.6 | 89.6 | 99.7% |
+| 5080 + 3090 + 2080 Ti | first auto (12, 33) | 1,148 | 1,144 | 51.9 | 69.7 | 100% |
+| 5080 + 2080 Ti | first auto | 914 | 906 | 45.5 | 58.0 | 97% |
+
+What decides it:
+
+- **The best two-GPU split for the Coder is K=26-28.** Prompts run +18-20% over the 5080 alone at 16-28K;
+  decode is on par for story and +4-7% for code.
+- **Residency saturates quickly: the per-layer GPU speed decides.** From K=18 up, both caches hold 98-100% of
+  the routed experts. Moving layers to the faster card is what helps: the 3090 costs 0.50 ms per layer per
+  window, the 5080 0.33 (`STRATA_SPLIT_TIMING`, the "wait for the GPU" per stage).
+- **Decode is bounded by that GPU time.** One 5080 with its cache adapted spends 16.3 ms waiting for its GPU and
+  5.5 ms in the CPU pool per window. The split removes the pool (0.8-1.0 ms) but adds the slower card's layers:
+  7.9 + 12.1 ms at K=24. The per-layer GPU half (kernels plus the host round trip that publishes each layer's
+  expert plan) is the next thing to cut, for one GPU as well (perf-review-plan E-6).
+- **A third, slower card costs more than it holds.** The 2080 Ti's one-layer stage took 3.1 ms per window.
+  For a model that already fits two cards, leave it out. It pays only when the two faster cards cannot hold most
+  of the routed experts (a bigger model; not testable on this rig's 62 GB of RAM).
+- **Prompt chunk:** 1024 instead of 2048 frees ~0.7 GB per card, but prompts read 7-12% slower.
+
+## Auto placement
+
+`--layer-split auto` predicts one decode window's time for every placement (every placement up to three GPUs) and
+keeps the fastest:
+
+- **per-layer time** from SMs x clock (0.33 ms for an RTX 5080);
+- **cache contents:** each card's layers' profiled pairs, hottest first, as the fill does;
+- **routed mass** of rank r taken as (r+1)^-1.2 (fits the sweep's hit rates within 0.1-1%);
+- **missed mass** costing 190 ms per unit (fitted to the sweep's best K; `STRATA_SPLIT_MISS_MS` overrides).
+
+It picked K=22 on the 5080 + 3090: within 0-12% of the best fixed split on prompts and on par on decode. It gives
+a slow third card a single layer.
+
+## Reproduce
+
+On a rig: the `strata-remote` tools, then `rib_matrix.bat`, `rib_ksweep.bat`, `rib_validate.bat` (in the dev
+notes, not in this repository).

bench/results/2026-09-29-layer-split/data/norepeat-nr-v-x2-auto-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 88,
+  "drafts_offered": 193,
+  "tok_s": 74.6,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was seventy years old, but he moved with the grace of a man who had spent decades on a narrow spiral staircase. His hands, rough as sandpaper, gripped the iron railing as he ascended the final flight of steps to the lantern room.\n\nBelow him, the Atlantic churned in a frenzy, whitecaps exploding against the black rocks like fireworks. Above, the sky was a bruised purple, heavy with rain that hadn\u2019t yet fallen.\n\nElias checked the clock. 11:42 PM. The light was due.\n\nHe didn\u2019t flip a switch. He didn\u2019t press a button. He reached for the heavy brass lever that had worn smooth by his palm over the years. With a grunt that matched the strain of his aging muscles, he pulled it down.\n\n*Clunk.*\n\nThe"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 136,
+  "drafts_offered": 181,
+  "tok_s": 105.5,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 85.8,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 94.3,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 118,
+  "repeats": 0,
+  "drafts_accepted": 37,
+  "drafts_offered": 105,
+  "tok_s": 67.9,
+  "text": "Aardvark  \nAlbatross   \nAlligator    \nAntelope     \nBaboon       \nBadger      \nBeaver        \nBear          \nBird of Paradise \nBuffalo         \nCamel           \nCheetah            \nChimpanzee             \nCrane              \nCrocodile                \nDeer                 \nDolphin               \nDonkey                  \nElephant                   \nFlamingo                    \nFox                       \nGiraffe                     \nGoat                      \nHippopotamus  \nIguana       \nJaguar        \nKangaroo      \nKoala          \nLion         \nLeopard     \nMonkey    \nMoose   \nOstrich \nOtter\nPanda<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 147,
+  "repeats": 0,
+  "drafts_accepted": 26,
+  "drafts_offered": 103,
+  "tok_s": 63.2,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral hall outside my window at midnight on November third this year\u2014so I pulled up to that date and time for context because you asked me about a lighthouse keeper but gave no specifics beyond \u201cshort story,\u201d so let\u2019s set it: isolated coastal station off the Oregon coast during winter storm season, 1987.\n\nMy name is Elias Vane (I made this fictional; real keepers often kept journals signed with initials only), and I\u2019m writing from memory because my logbook got waterlogged when a rogue wave breached lower storage room at dawn\u2014so here\u2019s what happened:<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 109,
+  "repeats": 0,
+  "drafts_accepted": 13,
+  "drafts_offered": 56,
+  "tok_s": 54.1,
+  "text": "The wind didn\u2019t just blow at Cape Hollow; it screamed through the gaps in my cottage\u2019s shutters, rattling them like loose teeth until they settled into silence with every storm break out of three hundred days here being storms that broke something else again but mostly kept me warm because I burned coal and wood whenever we had any left over from last winter when even then if you\u2019re lucky enough to get through it all without freezing yourself so hard your own thoughts become ice too\n\nBut Elias knew better than anyone: fear was useful only until dawn<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 92,
+  "repeats": 0,
+  "drafts_accepted": 26,
+  "drafts_offered": 48,
+  "tok_s": 64.7,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKoala      \nKangaroo        \nHippopotamus          \nRhinoceros            \nCamel              \nCheetah             \nLeopard           \nPanther         \nJaguar                \nBison               \nElk                 \nDeer                  \nFox                   \nWolf                    \nBear                     \nLlama                       \nDonkey                        \nGoat                      \nSheep                         \nCow                           \nPig                          \nRabbit                            \nCat                              \nDog<|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 14,
+  "tok_s": 71.7,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 53,
+  "drafts_offered": 134,
+  "tok_s": 66.1,
+  "text": "The ocean is not merely water; it has been, and continues to be one with humanity\u2019s most profound narrative arc in human civilization: **the story we tell ourselves through our relationship** between landmasses separated by vast expansiveness that defines both isolation from other worlds (and thus their own unique evolutionary paths) as well than connection across them via currents wind patterns tides depth pressure temperature salinity biodiversity geology climate change economic trade routes ecological systems cultural exchange political boundaries scientific discovery technological advancement environmental degradation conservation efforts maritime archaeography historical cartographic evolution legal frameworks international law enforcement mechanisms insurance liability regimes regulatory compliance standards safety protocols navigation aids communication networks emergency response strategies search-and-rescue operations salvage rights ownership disputes heritage protection zones sustainable harvesting practices aquaculture development infrastructure engineering projects coastal management policies urban planning regulations zoning ordinances building codes architectural design guidelines aesthetic considerations functional requirements accessibility provisions energy efficiency measures thermal insulation techniques moisture control methods ventilation optimization lighting solutions acoustic soundproofing structural integrity assessments seismic resistance evaluations flood mitigation plans storm surge barriers levee construction seawall reinforcement breakwater installation harbor dredging channel maintenance buoy placement lighthouse automation radar monitoring sonar mapping bathymetric surveys hydrographic charts nautical almanacs tide tables current predictions weather forecasts marine meteorological models satellite imagery remote sensing data assimilation numerical simulations computational fluid dynamics finite element analysis partial differential equations tensor calculus linear algebra probability theory statistics stochastic processes Markov chains Bayesian inference machine learning artificial neural network deep convolutional recurrent transformer attention heads generative adversarial diffusion transformers large language model tokenization"
+ }
+]

bench/results/2026-09-29-layer-split/data/norepeat-nr-v-x3-auto-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 88,
+  "drafts_offered": 193,
+  "tok_s": 68.2,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was seventy years old, but he moved with the grace of a man who had spent decades on a narrow spiral staircase. His hands, rough as sandpaper, gripped the iron railing as he ascended the final flight of steps to the lantern room.\n\nBelow him, the Atlantic churned in a frenzy, whitecaps exploding against the black rocks like fireworks. Above, the sky was a bruised purple, heavy with rain that hadn\u2019t yet fallen.\n\nElias checked the clock. 11:42 PM. The light was due.\n\nHe didn\u2019t flip a switch. He didn\u2019t press a button. He reached for the heavy brass lever that had worn smooth by his palm over the years. With a grunt that matched the strain of his aging muscles, he pulled it down.\n\n*Clunk.*\n\nThe"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 136,
+  "drafts_offered": 181,
+  "tok_s": 99.4,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 74.5,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 76.3,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 118,
+  "repeats": 0,
+  "drafts_accepted": 37,
+  "drafts_offered": 105,
+  "tok_s": 65.2,
+  "text": "Aardvark  \nAlbatross   \nAlligator    \nAntelope     \nBaboon       \nBadger      \nBeaver        \nBear          \nBird of Paradise \nBuffalo         \nCamel           \nCheetah            \nChimpanzee             \nCrane              \nCrocodile                \nDeer                 \nDolphin               \nDonkey                  \nElephant                   \nFlamingo                    \nFox                       \nGiraffe                     \nGoat                      \nHippopotamus  \nIguana       \nJaguar        \nKangaroo      \nKoala          \nLion         \nLeopard     \nMonkey    \nMoose   \nOstrich \nOtter\nPanda<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 147,
+  "repeats": 0,
+  "drafts_accepted": 26,
+  "drafts_offered": 103,
+  "tok_s": 45.7,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral hall outside my window at midnight on November third this year\u2014so I pulled up to that date and time for context because you asked me about a lighthouse keeper but gave no specifics beyond \u201cshort story,\u201d so let\u2019s set it: isolated coastal station off the Oregon coast during winter storm season, 1987.\n\nMy name is Elias Vane (I made this fictional; real keepers often kept journals signed with initials only), and I\u2019m writing from memory because my logbook got waterlogged when a rogue wave breached lower storage room at dawn\u2014so here\u2019s what happened:<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 109,
+  "repeats": 0,
+  "drafts_accepted": 13,
+  "drafts_offered": 56,
+  "tok_s": 42.1,
+  "text": "The wind didn\u2019t just blow at Cape Hollow; it screamed through the gaps in my cottage\u2019s shutters, rattling them like loose teeth until they settled into silence with every storm break out of three hundred days here being storms that broke something else again but mostly kept me warm because I burned coal and wood whenever we had any left over from last winter when even then if you\u2019re lucky enough to get through it all without freezing yourself so hard your own thoughts become ice too\n\nBut Elias knew better than anyone: fear was useful only until dawn<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 92,
+  "repeats": 0,
+  "drafts_accepted": 26,
+  "drafts_offered": 48,
+  "tok_s": 43.9,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKoala      \nKangaroo        \nHippopotamus          \nRhinoceros            \nCamel              \nCheetah             \nLeopard           \nPanther         \nJaguar                \nBison               \nElk                 \nDeer                  \nFox                   \nWolf                    \nBear                     \nLlama                       \nDonkey                        \nGoat                      \nSheep                         \nCow                           \nPig                          \nRabbit                            \nCat                              \nDog<|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 14,
+  "tok_s": 61.4,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 53,
+  "drafts_offered": 134,
+  "tok_s": 51.9,
+  "text": "The ocean is not merely water; it has been, and continues to be one with humanity\u2019s most profound narrative arc in human civilization: **the story we tell ourselves through our relationship** between landmasses separated by vast expansiveness that defines both isolation from other worlds (and thus their own unique evolutionary paths) as well than connection across them via currents wind patterns tides depth pressure temperature salinity biodiversity geology climate change economic trade routes ecological systems cultural exchange political boundaries scientific discovery technological advancement environmental degradation conservation efforts maritime archaeography historical cartographic evolution legal frameworks international law enforcement mechanisms insurance liability regimes regulatory compliance standards safety protocols navigation aids communication networks emergency response strategies search-and-rescue operations salvage rights ownership disputes heritage protection zones sustainable harvesting practices aquaculture development infrastructure engineering projects coastal management policies urban planning regulations zoning ordinances building codes architectural design guidelines aesthetic considerations functional requirements accessibility provisions energy efficiency measures thermal insulation techniques moisture control methods ventilation optimization lighting solutions acoustic soundproofing structural integrity assessments seismic resistance evaluations flood mitigation plans storm surge barriers levee construction seawall reinforcement breakwater installation harbor dredging channel maintenance buoy placement lighthouse automation radar monitoring sonar mapping bathymetric surveys hydrographic charts nautical almanacs tide tables current predictions weather forecasts marine meteorological models satellite imagery remote sensing data assimilation numerical simulations computational fluid dynamics finite element analysis partial differential equations tensor calculus linear algebra probability theory statistics stochastic processes Markov chains Bayesian inference machine learning artificial neural network deep convolutional recurrent transformer attention heads generative adversarial diffusion transformers large language model tokenization"
+ }
+]

bench/results/2026-09-29-layer-split/data/norepeat-nr-x2-auto-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 91,
+  "drafts_offered": 190,
+  "tok_s": 74.0,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was seventy years old, but he moved with the grace of a man who had spent decades on a narrow spiral staircase. His hands, rough as sandpaper, knew every bolt, every lever, and every creak in the iron bones of the tower.\n\nFor forty years, Elias had been the guardian of the Point. Not just of the light, but of the silence between the waves.\n\nIt was November. The sky was a bruised purple, heavy with rain that hadn\u2019t yet fallen. The sea was a churning cauldron of grey and white, throwing itself against the granite base with a violence that shook the floorboards beneath Elias\u2019s boots.\n\nHe climbed the stairs. One hundred and twelve steps. He counted them by feel, his left hand trailing along the cold iron railing. At the top, the air was"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 137,
+  "drafts_offered": 178,
+  "tok_s": 103.4,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 91.1,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 70.7,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 118,
+  "repeats": 0,
+  "drafts_accepted": 38,
+  "drafts_offered": 108,
+  "tok_s": 65.4,
+  "text": "Aardvark  \nAlbatross   \nAlligator    \nAntelope     \nBaboon       \nBadger      \nBeaver        \nBear          \nBird of Paradise \nBuffalo         \nCamel           \nCheetah            \nChimpanzee             \nCrane              \nCrocodile                \nDeer                 \nDolphin               \nDonkey                  \nElephant                   \nFlamingo                    \nFox                       \nGiraffe                     \nGoat                      \nHippopotamus  \nIguana       \nJaguar        \nKangaroo      \nKoala          \nLion         \nLeopard     \nMonkey    \nMoose   \nOstrich \nOtter\nPanda<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 57,
+  "repeats": 0,
+  "drafts_accepted": 11,
+  "drafts_offered": 61,
+  "tok_s": 57.1,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral vaults below Elias\u2019s boots were heavy with salt and exhaustion as he climbed one hundred twelve steps to reach top room where his world began again.<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 58,
+  "repeats": 0,
+  "drafts_accepted": 14,
+  "drafts_offered": 54,
+  "tok_s": 52.9,
+  "text": "The wind didn\u2019t just blow at Cape Harrow; it screamed through the gaps in stone and steel, carrying with them spray from three miles out into churning black water that no longer resembled an ocean so much as abyssal chaos waiting for its prey to surface beneath waves of salt.<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 59,
+  "repeats": 0,
+  "drafts_accepted": 11,
+  "drafts_offered": 47,
+  "tok_s": 51.7,
+  "text": "Lion  \nTiger  \n\nElephant\\nGiraffe\\Zebra\\rHippopotamus\\tRhinoceros\\vCrocodile\\fAlligator\\bSnake\\aSpider\\u2615CoffeeCup\u2615\ufe0fTeaTime\ud83d\udc09DragonFishWhaleDolph<|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 12,
+  "tok_s": 73.5,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 43,
+  "drafts_offered": 126,
+  "tok_s": 63.4,
+  "text": "The ocean is not merely water; it has been, and continues to be one with humanity\u2019s most profound narrative arc in human civilization: **the story told by waves**. It serves as both witness (through sedimentary layers)and participant( through currents that carry messages across continents). Its \"history\" spans geological epochs where life first emerged from primordial pools into complex ecosystems today\u2014and cultural histories shaped entirely around its roleas connector between distant lands or barrier separating them forever depending on perspective taken at different times throughout millennia since ancient civilizations began building boats outof logs carved using stone tools until modern container ships carrying billions tons cargo daily cross Atlantic routes once navigated onlyby stars alone under moonlight during nights when sailors feared monsters lurking beneath hulls made fragile against storms they could barely predict even now despite advanced meteorology systems developed over past century after industrial revolution transformed maritime trade patterns globally due economic pressures driving demand for faster logistics chains linking factories producing electronics manufactured inside cities located far inland near rivers feeding estuaries opening directly onto open seas connected via canals engineered centuries ago like Suez Canal built originallyfor military purposes later adapted commercial use post-World War II era reshaping global supply networks dramatically because geopolitical shifts altered which ports remained accessible politically versus physically blocked temporarily then reopened permanently following treaty agreements signed among nations agreeing upon shared responsibilities regarding environmental stewardship including coral reef restoration efforts funded jointly international organizations coordinating scientific research conducted aboard vessels specifically designed deep-sea exploration equipped robotic submersibles capable descending trenches deeper than any previous technology allowed"
+ }
+]

bench/results/2026-09-29-layer-split/data/norepeat-nr-x3-auto-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 98,
+  "drafts_offered": 166,
+  "tok_s": 54.4,
+  "text": "The wind didn\u2019t just blow; it screamed. It tore through the iron gallery of the Blackwood Point Lighthouse, rattling the glass panes like a prisoner shaking its bars.\n\nElias Thorne was seventy years old, and he had been the keeper for forty of them. He knew every creak in the spiral staircase, every draft that slipped under the door, and every shift in the weather that the barometer couldn\u2019t quite predict. He knew that when the fog rolled in thick enough to swallow the horizon, the world ceased to exist. There was only the light, the stone, and him.\n\nIt was November, the month of storms. The sea was a churning cauldron of grey and white, throwing itself against the granite base with a violence that made the floorboards tremble. Elias stood by the lantern room, his hands wrapped around a mug of coffee that had gone cold hours ago. He wasn\u2019t drinking it. He was holding it for warmth."
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 137,
+  "drafts_offered": 163,
+  "tok_s": 70.7,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv(file_path: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys being the header values\n    and values being the corresponding cell values.\n    \n    Args:\n        file_path: Path to the CSV file.\n        \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result = []\n    with open(file_path, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_basic():\n    \"\"\"Test basic CSV parsing with simple data.\"\"\"\n    import tempfile\n    import os\n    \n    # Create a temporary CSV"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 56.7,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 57.6,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 59,
+  "repeats": 0,
+  "drafts_accepted": 13,
+  "drafts_offered": 47,
+  "tok_s": 39.7,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAntelope    \nBaboon      \nBadger     \nBeaver       \nBear        \nBird         \nBuffalo     \n\nCat          \nCheetah      \n\nChimpanzee \nClam         \n\n\nCrocodile   \n\n\n\nDolphin      \n\n\n\n\nDonkey       <|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 57,
+  "repeats": 0,
+  "drafts_accepted": 12,
+  "drafts_offered": 58,
+  "tok_s": 39.9,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral vaults below Elias\u2019s boots were heavy with salt and exhaustion as he climbed one hundred twelve steps to reach topmost gallery where his world began.<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 30,
+  "drafts_offered": 111,
+  "tok_s": 34.7,
+  "text": "The wind didn\u2019t just blow at Blackwood Rock; it screamed through the gaps in his stone cottage like an angry ghost trying to get inside, yet finding no door left unlocked because Elias never let one stay that way for more than three seconds after entering or leaving during storm season. He had been doing this since 1987\u2014since before he even knew why\u2014and now it was muscle memory so deep into his bones and fingers\u2019 calloused skin texture patterns worn thin by years of gripping the same iron railing at midnight watch that if you asked him what time exactly, down to quarter-hour increments when fog rolled in thick enough where light beams bent around itself like old rope coiling tight on a winch drum spooling out slowly from below deck floor level storage locker number seven under stairs leading"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 96,
+  "repeats": 0,
+  "drafts_accepted": 31,
+  "drafts_offered": 48,
+  "tok_s": 42.3,
+  "text": "Lion  \nTiger   \nElephant \nGiraffe    \nZebra     \nPanda       \nKoala      \nKangaroo        \nCheetah         \nHippopotamus          \nRhinoceros            \nLeopard             \nJaguar              \nWolf                \nFox                 \nBear                  \nDeer                    \nOtter                   \nBeaver                      \nBadger                        \nMongoose                          \nMeerkat                           \nChimpanzee                            \nGorilla                             \nOrangetutan                              \nAntelope                                \nBison                                   \nElk                                     <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 13,
+  "tok_s": 43.0,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 29,
+  "drafts_offered": 127,
+  "tok_s": 37.6,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant machine for making life possible on Earth.\u201d \u2014 *David Attenborough* \n\nThis quote captures perhaps more than any other single sentence in human language: that which we call \u201cthe Sea\u201d or \"Ocean\" (terms often used interchangeably but distinct geographically) was never just an element within our planet\u2019s geography\u2014it became its engine itself\u2014a living system whose existence predates humanity by billions years yet continues shaping us today through climate regulation oxygen production food supply trade routes cultural identity ecological resilience economic development political boundaries scientific discovery technological innovation environmental crisis conservation efforts historical memory philosophical reflection poetic metaphor musical composition cinematic storytelling literary narrative visual art architectural inspiration culinary tradition spiritual symbolism psychological influence sociological transformation educational foundation medical necessity agricultural dependency industrial revolution urban planning infrastructural engineering legal framework regulatory policy international cooperation global governance planetary stewardship existential risk mitigation sustainable practice ethical consideration moral imperative social justice democratic participation civic engagement public health community welfare individual responsibility personal accountability collective action shared experience common understanding mutual respect reciprocal trust collaborative effort cooperative endeavor joint venture unified mission integrated strategy coordinated plan synchronized schedule aligned timeline concurrent execution simultaneous operation parallel processing sequential progression iterative refinement recursive improvement continuous adaptation dynamic adjustment flexible response adaptive behavior responsive conduct proactive initiative anticipatory foresight predictive modeling probabilistic assessment statistical analysis empirical observation experimental verification theoretical hypothesis conceptual abstraction logical deduction rational inference critical evaluation analytical reasoning systematic investigation methodical inquiry structured exploration comprehensive examination detailed scrutiny meticulous inspection thorough review exhaustive audit complete survey full census total enumeration absolute"
+ }
+]

bench/results/2026-09-29-layer-split/data/norepeat-reg-dev-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 88,
+  "drafts_offered": 187,
+  "tok_s": 59.2,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was seventy years old, but he moved with the grace of a man who had spent decades on a narrow spiral staircase. His hands, mapped with veins like the roots of an ancient oak, gripped the cold iron railing.\n\nFor forty years, Elias had tended the light at the end of the world. Not the end of the map, but the end of the land. Here, the Atlantic threw itself against the granite cliffs with such violence that the spray reached the gallery, thirty feet above the rocks.\n\nHe checked the clock. 11:42 PM. The fog was thickening, a gray soup that swallowed the horizon. In the old days, he would have climbed the stairs to the lantern room, polished the Fresnel lens, and wound the clockwork mechanism that turned the great beam. But the automation had changed"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 135,
+  "drafts_offered": 179,
+  "tok_s": 81.6,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 74.9,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 74.3,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 61,
+  "repeats": 0,
+  "drafts_accepted": 19,
+  "drafts_offered": 49,
+  "tok_s": 49.6,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAntelope    \nBaboon      \nBadger     \nBeaver       \nBear        \nBird         \nBoar          \nBuffalo \nButterfly \n\nCamel           \nCat            \nCheetah        \n\nDolphin         \n\n\nElephant        \n\n\n\nFlamingo<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 80,
+  "repeats": 0,
+  "drafts_accepted": 6,
+  "drafts_offered": 52,
+  "tok_s": 44.9,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral hall outside my window at midnight on November third this year\u2014so I pulled up to check what was happening with that storm and why they were calling me out there when all we had left for tonight\u2019s shift is one last pass through our logs before dawn breaks.<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 17,
+  "drafts_offered": 83,
+  "tok_s": 44.1,
+  "text": "The wind didn\u2019t just blow at Cape Harrow; it screamed through the gaps in its teeth, hungry and relentless for three centuries of stone salt-spray history that had long since forgotten to be polite or gentle with itself against anything smaller than God Himself was willing acknowledge as divine protection from drowning souls who were already lost before they ever touched water.\n\nElias knew this rhythm better he himself could hear it sometimes; his boots worn smooth by pacing the spiral stair\u2019s center-post groove where thousands upon countless steps had rubbed away texture until all that remained beneath sole leather now felt like polished bone carved out slowly but surely over time into something more hollow than wood ever really should be allowed become when used so often enough within such confined circular spaces meant only for vertical ascent down always descending again back upward never truly leaving"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 62,
+  "repeats": 0,
+  "drafts_accepted": 18,
+  "drafts_offered": 31,
+  "tok_s": 49.5,
+  "text": "Lion  \nTiger  \n\nElephant  \n  \nGiraffe  \n\n\n\nHippopotamus    \nZebra   \nRhinoceros   \n\n\nPanda Bear    \n\n\n\n\nKangaroo     \nKoala      \nSloth      \n\n\n\n\n\nCheetah       \nLeopard        \nPanther        \n\n\n\n\n\n\nWolf              \nFox                \nBear                 <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 13,
+  "tok_s": 65.2,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 47,
+  "drafts_offered": 153,
+  "tok_s": 52.5,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant machine for making life possible on Earth.\" \u2014 *David Attenborough* \n\nThis quote captures perhaps more than any other single sentence in human language: that which we call \"the Sea\" or \u201cOcean\u201d was never just an element within our planet\u2019s geography but rather its most fundamental biological engine\u2014a living system whose existence predates humanity by billions years yet continues shaping every aspect from climate regulation through evolutionary biology all way down modern industrial supply chains today's global shipping lanes carry cargo across vast distances because they were once crossed only slowly via coastal hopping routes used since antiquity until steamships revolutionized travel during late nineteenth century when diesel engines replaced wind power entirely allowing container ships carrying thousands TEUs (Twenty-foot Equivalent Units) per voyage crossing entire Pacific basins without stopping at intermediate ports anymore unlike earlier sailing vessels needing favorable winds currents seasonal patterns dictating departure windows based upon monsoon calendars established centuries ago before technological advances changed those constraints fundamentally altering logistics networks worldwide thereby transforming economic structures globally while simultaneously creating ecological disruptions due invasive species transported unintentionally alongside legitimate goods causing cascading ecosystem collapses locally where non-native organisms outcompete native fauna disrupting food webs leading population declines affecting biodiversity indices measured scientifically using standardized metrics like Shannon-Wiener diversity index calculations applied consistently over decades revealing trends showing significant decreases post-introduction events correl strongly with introduction dates matching historical records documenting first sightings confirmed taxonomically classified formally according International Code Nomenclature rules"
+ }
+]

bench/results/2026-09-29-layer-split/data/norepeat-reg-rel-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 88,
+  "drafts_offered": 187,
+  "tok_s": 58.4,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was seventy years old, but he moved with the grace of a man who had spent decades on a narrow spiral staircase. His hands, mapped with veins like the roots of an ancient oak, gripped the cold iron railing.\n\nFor forty years, Elias had tended the light at the end of the world. Not the end of the map, but the end of the land. Here, the Atlantic threw itself against the granite cliffs with such violence that the spray reached the gallery, thirty feet above the rocks.\n\nHe checked the clock. 11:42 PM. The fog was thickening, a gray soup that swallowed the horizon. In the old days, he would have climbed the stairs to the lantern room, polished the Fresnel lens, and wound the clockwork mechanism that turned the great beam. But the automation had changed"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 135,
+  "drafts_offered": 179,
+  "tok_s": 80.6,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 75.2,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 74.2,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 61,
+  "repeats": 0,
+  "drafts_accepted": 19,
+  "drafts_offered": 49,
+  "tok_s": 48.5,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAntelope    \nBaboon      \nBadger     \nBeaver       \nBear        \nBird         \nBoar          \nBuffalo \nButterfly \n\nCamel           \nCat            \nCheetah        \n\nDolphin         \n\n\nElephant        \n\n\n\nFlamingo<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 80,
+  "repeats": 0,
+  "drafts_accepted": 6,
+  "drafts_offered": 52,
+  "tok_s": 46.4,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral hall outside my window at midnight on November third this year\u2014so I pulled up to check what was happening with that storm and why they were calling me out there when all we had left for tonight\u2019s shift is one last pass through our logs before dawn breaks.<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 17,
+  "drafts_offered": 83,
+  "tok_s": 43.3,
+  "text": "The wind didn\u2019t just blow at Cape Harrow; it screamed through the gaps in its teeth, hungry and relentless for three centuries of stone salt-spray history that had long since forgotten to be polite or gentle with itself against anything smaller than God Himself was willing acknowledge as divine protection from drowning souls who were already lost before they ever touched water.\n\nElias knew this rhythm better he himself could hear it sometimes; his boots worn smooth by pacing the spiral stair\u2019s center-post groove where thousands upon countless steps had rubbed away texture until all that remained beneath sole leather now felt like polished bone carved out slowly but surely over time into something more hollow than wood ever really should be allowed become when used so often enough within such confined circular spaces meant only for vertical ascent down always descending again back upward never truly leaving"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 62,
+  "repeats": 0,
+  "drafts_accepted": 18,
+  "drafts_offered": 31,
+  "tok_s": 49.8,
+  "text": "Lion  \nTiger  \n\nElephant  \n  \nGiraffe  \n\n\n\nHippopotamus    \nZebra   \nRhinoceros   \n\n\nPanda Bear    \n\n\n\n\nKangaroo     \nKoala      \nSloth      \n\n\n\n\n\nCheetah       \nLeopard        \nPanther        \n\n\n\n\n\n\nWolf              \nFox                \nBear                 <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 13,
+  "tok_s": 62.5,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 47,
+  "drafts_offered": 153,
+  "tok_s": 52.6,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant machine for making life possible on Earth.\" \u2014 *David Attenborough* \n\nThis quote captures perhaps more than any other single sentence in human language: that which we call \"the Sea\" or \u201cOcean\u201d was never just an element within our planet\u2019s geography but rather its most fundamental biological engine\u2014a living system whose existence predates humanity by billions years yet continues shaping every aspect from climate regulation through evolutionary biology all way down modern industrial supply chains today's global shipping lanes carry cargo across vast distances because they were once crossed only slowly via coastal hopping routes used since antiquity until steamships revolutionized travel during late nineteenth century when diesel engines replaced wind power entirely allowing container ships carrying thousands TEUs (Twenty-foot Equivalent Units) per voyage crossing entire Pacific basins without stopping at intermediate ports anymore unlike earlier sailing vessels needing favorable winds currents seasonal patterns dictating departure windows based upon monsoon calendars established centuries ago before technological advances changed those constraints fundamentally altering logistics networks worldwide thereby transforming economic structures globally while simultaneously creating ecological disruptions due invasive species transported unintentionally alongside legitimate goods causing cascading ecosystem collapses locally where non-native organisms outcompete native fauna disrupting food webs leading population declines affecting biodiversity indices measured scientifically using standardized metrics like Shannon-Wiener diversity index calculations applied consistently over decades revealing trends showing significant decreases post-introduction events correl strongly with introduction dates matching historical records documenting first sightings confirmed taxonomically classified formally according International Code Nomenclature rules"
+ }
+]

bench/results/2026-09-29-layer-split/data/norepeat-reg-same24-coder-iq1_m.json

@@ -0,0 +1,92 @@
+[
+ {
+  "name": "greedy story 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 88,
+  "drafts_offered": 187,
+  "tok_s": 58.4,
+  "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was seventy years old, but he moved with the grace of a man who had spent decades on a narrow spiral staircase. His hands, mapped with veins like the roots of an ancient oak, gripped the cold iron railing.\n\nFor forty years, Elias had tended the light at the end of the world. Not the end of the map, but the end of the land. Here, the Atlantic threw itself against the granite cliffs with such violence that the spray reached the gallery, thirty feet above the rocks.\n\nHe checked the clock. 11:42 PM. The fog was thickening, a gray soup that swallowed the horizon. In the old days, he would have climbed the stairs to the lantern room, polished the Fresnel lens, and wound the clockwork mechanism that turned the great beam. But the automation had changed"
+ },
+ {
+  "name": "greedy code 200",
+  "tokens": 200,
+  "repeats": 0,
+  "drafts_accepted": 135,
+  "drafts_offered": 179,
+  "tok_s": 80.5,
+  "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+ },
+ {
+  "name": "greedy sanity",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 74.5,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy, no penalty (sanity)",
+  "tokens": 4,
+  "repeats": 0,
+  "drafts_accepted": 3,
+  "drafts_offered": 3,
+  "tok_s": 76.8,
+  "text": "144<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64",
+  "tokens": 61,
+  "repeats": 0,
+  "drafts_accepted": 19,
+  "drafts_offered": 49,
+  "tok_s": 49.6,
+  "text": "Aardvark  \nAlbatross  \n\nAlligator   \nAntelope    \nBaboon      \nBadger     \nBeaver       \nBear        \nBird         \nBoar          \nBuffalo \nButterfly \n\nCamel           \nCat            \nCheetah        \n\nDolphin         \n\n\nElephant        \n\n\n\nFlamingo<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=64 story",
+  "tokens": 80,
+  "repeats": 0,
+  "drafts_accepted": 6,
+  "drafts_offered": 52,
+  "tok_s": 46.7,
+  "text": "The wind didn\u2019t just blow; it screamed against the glass of Lantern House 42, rattling like chains in an empty cathedral hall outside my window at midnight on November third this year\u2014so I pulled up to check what was happening with that storm and why they were calling me out there when all we had left for tonight\u2019s shift is one last pass through our logs before dawn breaks.<|im_end|>"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 3",
+  "tokens": 160,
+  "repeats": 0,
+  "drafts_accepted": 17,
+  "drafts_offered": 83,
+  "tok_s": 44.3,
+  "text": "The wind didn\u2019t just blow at Cape Harrow; it screamed through the gaps in its teeth, hungry and relentless for three centuries of stone salt-spray history that had long since forgotten to be polite or gentle with itself against anything smaller than God Himself was willing acknowledge as divine protection from drowning souls who were already lost before they ever touched water.\n\nElias knew this rhythm better he himself could hear it sometimes; his boots worn smooth by pacing the spiral stair\u2019s center-post groove where thousands upon countless steps had rubbed away texture until all that remained beneath sole leather now felt like polished bone carved out slowly but surely over time into something more hollow than wood ever really should be allowed become when used so often enough within such confined circular spaces meant only for vertical ascent down always descending again back upward never truly leaving"
+ },
+ {
+  "name": "sampled presence=100 last_n=64 seed 11",
+  "tokens": 62,
+  "repeats": 0,
+  "drafts_accepted": 18,
+  "drafts_offered": 31,
+  "tok_s": 50.0,
+  "text": "Lion  \nTiger  \n\nElephant  \n  \nGiraffe  \n\n\n\nHippopotamus    \nZebra   \nRhinoceros   \n\n\nPanda Bear    \n\n\n\n\nKangaroo     \nKoala      \nSloth      \n\n\n\n\n\nCheetah       \nLeopard        \nPanther        \n\n\n\n\n\n\nWolf              \nFox                \nBear                 <|im_end|>"
+ },
+ {
+  "name": "greedy frequency=100 last_n=16",
+  "tokens": 11,
+  "repeats": 0,
+  "drafts_accepted": 5,
+  "drafts_offered": 13,
+  "tok_s": 62.0,
+  "text": "1 to 50, separated by commas:<|im_end|>"
+ },
+ {
+  "name": "greedy presence=100 last_n=4096 (8-row windows)",
+  "tokens": 300,
+  "repeats": 0,
+  "drafts_accepted": 47,
+  "drafts_offered": 153,
+  "tok_s": 52.1,
+  "text": "The ocean is not merely water; it has been, and remains to this day one giant machine for making life possible on Earth.\" \u2014 *David Attenborough* \n\nThis quote captures perhaps more than any other single sentence in human language: that which we call \"the Sea\" or \u201cOcean\u201d was never just an element within our planet\u2019s geography but rather its most fundamental biological engine\u2014a living system whose existence predates humanity by billions years yet continues shaping every aspect from climate regulation through evolutionary biology all way down modern industrial supply chains today's global shipping lanes carry cargo across vast distances because they were once crossed only slowly via coastal hopping routes used since antiquity until steamships revolutionized travel during late nineteenth century when diesel engines replaced wind power entirely allowing container ships carrying thousands TEUs (Twenty-foot Equivalent Units) per voyage crossing entire Pacific basins without stopping at intermediate ports anymore unlike earlier sailing vessels needing favorable winds currents seasonal patterns dictating departure windows based upon monsoon calendars established centuries ago before technological advances changed those constraints fundamentally altering logistics networks worldwide thereby transforming economic structures globally while simultaneously creating ecological disruptions due invasive species transported unintentionally alongside legitimate goods causing cascading ecosystem collapses locally where non-native organisms outcompete native fauna disrupting food webs leading population declines affecting biodiversity indices measured scientifically using standardized metrics like Shannon-Wiener diversity index calculations applied consistently over decades revealing trends showing significant decreases post-introduction events correl strongly with introduction dates matching historical records documenting first sightings confirmed taxonomically classified formally according International Code Nomenclature rules"
+ }
+]

bench/results/2026-09-29-layer-split/data/pp-dev-3090.json

@@ -0,0 +1,72 @@
+{
+ "label": "dev-3090",
+ "extra": [],
+ "gpus": "2",
+ "needles": [
+  {
+   "tokens": 2726,
+   "prompt_ms": 3424.8,
+   "pp_tok_s": 796.0,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 8379,
+   "prompt_ms": 8617.9,
+   "pp_tok_s": 972.3,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 16066,
+   "prompt_ms": 12820.1,
+   "pp_tok_s": 1253.2,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 22901.1,
+   "pp_tok_s": 1211.4,
+   "found": true,
+   "answer": "willow-474<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 5657.8,
+  "turn2_tokens": 6660,
+  "reused": 6631,
+  "turn2_ms": 238.7,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 71.7,
+   "hit_rate": 0.9266798230257504,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias had lived in the tower for thirty years, and he knew the difference between a gale and a hurricane by the way the glass rattled in its frame. Tonight, it was the latter. The sea was a churning black void, punctuated by whitecaps that looked like teeth.\n\nHe climbed the spiral stairs, his knees protesting with every step. The iron rungs were cold, slick with condensation that had leaked through the hairline cracks in the masonry. At the top, the lantern room hummed with the low, steady thrum of the rotating lens. It was a sound Elias had learned to sleep with, a heartbeat of the house.\n\nHe checked the clock. 11:42 PM. The light swept across the water, a slow, hypnotic arc. *Flash. Darkness. Flash. Darkness.*\n\nFor a moment, the beam caught something that wasn\u2019t water. A shape"
+  },
+  {
+   "name": "code",
+   "tok_s": 92.9,
+   "hit_rate": 0.8636411400016346,
+   "text": "```python\nimport csv\nfrom typing import List, Dict\n\n\ndef parse_csv(filepath: str) -> List[Dict[str, str]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header row\n    and values from the corresponding row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result = []\n    with open(filepath, 'r', newline='') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n```<|im_end|>"
+  }
+ ],
+ "engine_lines": [
+  "strata generate: expert cache auto: 17.31 GiB free, 700 MiB reserved -> 6704 slots",
+  "strata generate: expert cache 8730 slots, 16.62 GiB of VRAM; policy is",
+  "strata serve: the prompt path borrows 2315 cache slots (4.42 GiB)",
+  "strata serve: decode expert cache hit rate: 86.6% (4986 hits / 5756 lookups)",
+  "strata serve: decode expert cache hit rate: 87.1% (3757 hits / 4315 lookups)",
+  "strata serve: decode expert cache hit rate: 88.7% (3827 hits / 4314 lookups)",
+  "strata serve: decode expert cache hit rate: 92.1% (3975 hits / 4318 lookups)",
+  "strata serve: decode expert cache hit rate: 95.5% (4124 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 94.3% (25793 hits / 27360 lookups)",
+  "strata serve: decode expert cache hit rate: 94.6% (4088 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 92.7% (116454 hits / 125668 lookups)",
+  "strata serve: decode expert cache hit rate: 86.4% (73970 hits / 85649 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-dev-5080.json

@@ -0,0 +1,72 @@
+{
+ "label": "dev-5080",
+ "extra": [],
+ "gpus": "0",
+ "needles": [
+  {
+   "tokens": 2726,
+   "prompt_ms": 2129.8,
+   "pp_tok_s": 1279.9,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 8379,
+   "prompt_ms": 5057.2,
+   "pp_tok_s": 1656.8,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 16066,
+   "prompt_ms": 7964.7,
+   "pp_tok_s": 2017.2,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 14053.6,
+   "pp_tok_s": 1974.1,
+   "found": true,
+   "answer": "willow-474<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3366.4,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 197.3,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 83.1,
+   "hit_rate": 0.8418683557684931,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias had lived in the tower for thirty years, and he knew the difference between a gale and a hurricane by the way the glass rattled in its frame. Tonight, it was the latter. The sea was a churning cauldron of black ink and white foam, throwing itself against the granite base with the force of a battering ram.\n\nInside the lantern room, the air was thick with the smell of kerosene and old brass. Elias wiped his hands on his woolen trousers, leaving faint, oily smears on the fabric. He checked the clock. 11:42 PM. The rotation was due.\n\nHe reached for the crank handle, a heavy iron wheel worn smooth by decades of his own palms. With a grunt that matched the strain of his shoulders, he began to turn it. *Click. Click. Click.* The gears engaged, and the massive Fresnel lens began its slow"
+  },
+  {
+   "name": "code",
+   "tok_s": 88.0,
+   "hit_rate": 0.7011385968111186,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "engine_lines": [
+  "strata generate: expert cache auto: 9.15 GiB free, 700 MiB reserved -> 3415 slots",
+  "strata generate: expert cache 4293 slots, 8.17 GiB of VRAM; policy is",
+  "strata serve: the prompt path borrows 2343 cache slots (4.42 GiB)",
+  "strata serve: decode expert cache hit rate: 59.6% (2749 hits / 4610 lookups)",
+  "strata serve: decode expert cache hit rate: 64.2% (2136 hits / 3329 lookups)",
+  "strata serve: decode expert cache hit rate: 67.3% (2309 hits / 3432 lookups)",
+  "strata serve: decode expert cache hit rate: 72.5% (2539 hits / 3504 lookups)",
+  "strata serve: decode expert cache hit rate: 77.3% (2817 hits / 3642 lookups)",
+  "strata serve: decode expert cache hit rate: 75.3% (16500 hits / 21916 lookups)",
+  "strata serve: decode expert cache hit rate: 80.0% (2978 hits / 3723 lookups)",
+  "strata serve: decode expert cache hit rate: 84.2% (98752 hits / 117301 lookups)",
+  "strata serve: decode expert cache hit rate: 70.1% (69215 hits / 98718 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-ks-k18.json

@@ -0,0 +1,85 @@
+{
+ "label": "ks-k18",
+ "extra": [
+  "--layer-split",
+  "18"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 10383.7,
+   "pp_tok_s": 1547.2,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 16371.9,
+   "pp_tok_s": 1694.5,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 4308.0,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 239.8,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 77.2,
+   "reps": [
+    74.6,
+    78.1,
+    79.0
+   ],
+   "hit_rate": 0.9985378787878788,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias had lived in the tower for thirty years, and he knew the difference between a gale and a hurricane by the way the glass rattled in its frame. Tonight, the glass was vibrating like a plucked cello string. The sea, usually a rhythmic pulse against the granite base, was now a chaotic, frothing beast, throwing itself against the rocks with the force of a battering ram.\n\nElias climbed the spiral stairs, his hand gripping the cold iron railing. His knees ached, not from age, but from the constant vibration of the floor beneath his feet. He reached the lantern room, the heart of the lighthouse, and checked the clockwork mechanism. The great lens, a prism of Fresnel glass, rotated with a steady, hypnotic *whirrr-click, whirrr-click*. It was the only sound that mattered. The light swept out into the black void, cutting through"
+  },
+  {
+   "name": "code",
+   "tok_s": 103.8,
+   "reps": [
+    97.6,
+    106.4,
+    107.3
+   ],
+   "hit_rate": 0.9984863945578232,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 524 windows; per window: wait for the GPU 6.456 ms, pool + plan 0.181 ms, host staging 0.581 ms, commit 0.206 ms",
+  "strata serve: stage 1: 524 windows; per window: wait for the GPU 14.872 ms, pool + plan 0.501 ms, host staging 0.000 ms, commit 0.387 ms",
+  "strata serve: stage 0: 593 windows; per window: wait for the GPU 6.479 ms, pool + plan 0.185 ms, host staging 0.515 ms, commit 0.207 ms",
+  "strata serve: stage 1: 593 windows; per window: wait for the GPU 15.053 ms, pool + plan 0.496 ms, host staging 0.000 ms, commit 0.390 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split 18)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4608 slots, 8.24 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 18-47, expert cache 7152 slots (14.14 GiB), 7152 of its 7680 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-17",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-17 (CUDA0), 18-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 98.7% (6159 hits / 6240 lookups)",
+  "strata serve: decode expert cache hit rate: 98.9% (3796 hits / 3840 lookups)",
+  "strata serve: decode expert cache hit rate: 99.2% (3811 hits / 3840 lookups)",
+  "strata serve: decode expert cache hit rate: 99.6% (27723 hits / 27840 lookups)",
+  "strata serve: decode expert cache hit rate: 99.1% (3807 hits / 3840 lookups)",
+  "strata serve: decode expert cache hit rate: 99.3% (142002 hits / 143040 lookups)",
+  "strata serve: decode expert cache hit rate: 99.8% (139882 hits / 140160 lookups)",
+  "strata serve: decode expert cache hit rate: 99.9% (131807 hits / 132000 lookups)",
+  "strata serve: decode expert cache hit rate: 99.0% (119278 hits / 120480 lookups)",
+  "strata serve: decode expert cache hit rate: 99.8% (117410 hits / 117600 lookups)",
+  "strata serve: decode expert cache hit rate: 99.8% (117422 hits / 117600 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-ks-k19.json

@@ -0,0 +1,85 @@
+{
+ "label": "ks-k19",
+ "extra": [
+  "--layer-split",
+  "19"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 9875.2,
+   "pp_tok_s": 1626.9,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 15498.0,
+   "pp_tok_s": 1790.1,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 4085.5,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 238.7,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 79.3,
+   "reps": [
+    74.8,
+    85.6,
+    77.4
+   ],
+   "hit_rate": 0.9977757017660928,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias had lived in the tower for thirty years, and he knew the difference between a gale and a hurricane by the way the glass rattled in its frame. Tonight, the glass was vibrating like a plucked cello string. The sea, usually a rhythmic pulse against the granite base, was now a chaotic, frothing beast, throwing itself against the rocks with the force of a battering ram.\n\nElias stood at the top of the spiral staircase, his hand resting on the cold iron railing. He was not afraid. Fear was for the tourists who came in summer, wide-eyed and clutching their cameras. Elias was a keeper. He was a guardian.\n\nHe checked the logbook. The entry for the night was simple: *Wind, 60 knots. Visibility, zero. Light, operational.*\n\nHe walked down the stairs, his boots heavy on the metal grating. The smell of salt"
+  },
+  {
+   "name": "code",
+   "tok_s": 107.4,
+   "reps": [
+    98.9,
+    111.3,
+    112.1
+   ],
+   "hit_rate": 0.9986286196529353,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header row\n    and values from the corresponding row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 533 windows; per window: wait for the GPU 6.226 ms, pool + plan 0.294 ms, host staging 0.561 ms, commit 0.225 ms",
+  "strata serve: stage 1: 533 windows; per window: wait for the GPU 14.156 ms, pool + plan 0.373 ms, host staging 0.000 ms, commit 0.371 ms",
+  "strata serve: stage 0: 599 windows; per window: wait for the GPU 6.314 ms, pool + plan 0.292 ms, host staging 0.501 ms, commit 0.225 ms",
+  "strata serve: stage 1: 599 windows; per window: wait for the GPU 14.366 ms, pool + plan 0.375 ms, host staging 0.000 ms, commit 0.374 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split 19)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4606 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 19-47, expert cache 7160 slots (14.14 GiB), 7160 of its 7424 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-18",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-18 (CUDA0), 19-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 98.5% (6141 hits / 6234 lookups)",
+  "strata serve: decode expert cache hit rate: 99.0% (3797 hits / 3836 lookups)",
+  "strata serve: decode expert cache hit rate: 99.4% (3814 hits / 3837 lookups)",
+  "strata serve: decode expert cache hit rate: 99.5% (28659 hits / 28795 lookups)",
+  "strata serve: decode expert cache hit rate: 99.3% (3814 hits / 3839 lookups)",
+  "strata serve: decode expert cache hit rate: 99.2% (141378 hits / 142508 lookups)",
+  "strata serve: decode expert cache hit rate: 99.8% (124963 hits / 125273 lookups)",
+  "strata serve: decode expert cache hit rate: 99.8% (134574 hits / 134874 lookups)",
+  "strata serve: decode expert cache hit rate: 99.0% (121132 hits / 122322 lookups)",
+  "strata serve: decode expert cache hit rate: 99.9% (114073 hits / 114235 lookups)",
+  "strata serve: decode expert cache hit rate: 99.9% (113598 hits / 113754 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-ks-k20.json

@@ -0,0 +1,85 @@
+{
+ "label": "ks-k20",
+ "extra": [
+  "--layer-split",
+  "20"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 9044.3,
+   "pp_tok_s": 1776.4,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 13928.7,
+   "pp_tok_s": 1991.8,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3743.5,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 236.8,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 76.4,
+   "reps": [
+    74.2,
+    78.4,
+    76.7
+   ],
+   "hit_rate": 0.9979911611088791,
+   "text": "The wind didn\u2019t just blow; it screamed. It clawed at the granite base of the tower, trying to pry it from the bedrock, and it howled through the narrow slits of the lantern room glass.\n\nElias Thorne was seventy years old, but he moved with the grace of a man who had spent fifty years listening to the sea. He knew the rhythm of the waves against the rocks below\u2014the *thud, thud, crash*\u2014and he knew when that rhythm changed. When the sea grew too quiet, or too violent, he knew a storm was coming, or a ship was in trouble.\n\nTonight, the storm was a Category 4. The rain lashed against the windows like gravel thrown by a giant. Elias stood in the center of the lantern room, his hand resting on the cold brass railing. He wasn\u2019t afraid. He was a guardian.\n\nHe checked the clockwork mechanism. It was old, a relic from 1"
+  },
+  {
+   "name": "code",
+   "tok_s": 105.9,
+   "reps": [
+    103.2,
+    106.8,
+    107.7
+   ],
+   "hit_rate": 0.9981849807154201,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 537 windows; per window: wait for the GPU 6.685 ms, pool + plan 0.403 ms, host staging 0.593 ms, commit 0.215 ms",
+  "strata serve: stage 1: 537 windows; per window: wait for the GPU 13.928 ms, pool + plan 0.282 ms, host staging 0.000 ms, commit 0.359 ms",
+  "strata serve: stage 0: 606 windows; per window: wait for the GPU 6.764 ms, pool + plan 0.399 ms, host staging 0.528 ms, commit 0.216 ms",
+  "strata serve: stage 1: 606 windows; per window: wait for the GPU 14.108 ms, pool + plan 0.288 ms, host staging 0.000 ms, commit 0.362 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split 20)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4603 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 20-47, expert cache 7142 slots (14.14 GiB), 7142 of its 7168 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-19",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-19 (CUDA0), 20-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 98.9% (6162 hits / 6230 lookups)",
+  "strata serve: decode expert cache hit rate: 99.2% (3803 hits / 3835 lookups)",
+  "strata serve: decode expert cache hit rate: 99.3% (3810 hits / 3836 lookups)",
+  "strata serve: decode expert cache hit rate: 99.5% (27185 hits / 27330 lookups)",
+  "strata serve: decode expert cache hit rate: 99.3% (3807 hits / 3835 lookups)",
+  "strata serve: decode expert cache hit rate: 99.2% (139825 hits / 140938 lookups)",
+  "strata serve: decode expert cache hit rate: 99.7% (142122 hits / 142524 lookups)",
+  "strata serve: decode expert cache hit rate: 99.8% (146556 hits / 146851 lookups)",
+  "strata serve: decode expert cache hit rate: 99.0% (118026 hits / 119233 lookups)",
+  "strata serve: decode expert cache hit rate: 99.8% (118729 hits / 118997 lookups)",
+  "strata serve: decode expert cache hit rate: 99.8% (118791 hits / 119007 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-ks-k21.json

@@ -0,0 +1,85 @@
+{
+ "label": "ks-k21",
+ "extra": [
+  "--layer-split",
+  "21"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 8880.5,
+   "pp_tok_s": 1809.1,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 13654.6,
+   "pp_tok_s": 2031.8,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3680.6,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 235.4,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 77.4,
+   "reps": [
+    80.2,
+    80.8,
+    71.2
+   ],
+   "hit_rate": 0.9973622550922132,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the rough fabric scratching against his stubble. Outside the thick, salt-crusted glass of the lantern room, the Atlantic was a churning cauldron of black and white foam. Rain lashed against the panes in horizontal sheets, blurring the world into a smear of gray and shadow.\n\nInside, however, it was warm. The smell of kerosene and old wood hung heavy in the air, a scent that had defined his life for thirty years.\n\nElias was not a man of many words. He was a man of many tasks. His hands, mapped with scars from rope burns and broken glass, moved with a rhythm that predated his memory. He checked the clockwork mechanism of the Fresnel lens, ensuring the rotation was smooth, silent, and steady. *Tick. Tick. Tick.* The sound was the heartbeat of the coast.\n\n"
+  },
+  {
+   "name": "code",
+   "tok_s": 106.7,
+   "reps": [
+    100.6,
+    110.5,
+    109.0
+   ],
+   "hit_rate": 0.998218711927068,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 527 windows; per window: wait for the GPU 6.934 ms, pool + plan 0.620 ms, host staging 0.530 ms, commit 0.245 ms",
+  "strata serve: stage 1: 527 windows; per window: wait for the GPU 13.912 ms, pool + plan 0.365 ms, host staging 0.000 ms, commit 0.359 ms",
+  "strata serve: stage 0: 595 windows; per window: wait for the GPU 7.031 ms, pool + plan 0.595 ms, host staging 0.472 ms, commit 0.245 ms",
+  "strata serve: stage 1: 595 windows; per window: wait for the GPU 14.051 ms, pool + plan 0.361 ms, host staging 0.000 ms, commit 0.361 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split 21)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4627 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 21-47, expert cache 6912 slots (13.78 GiB), 6912 of its 6912 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-20",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-20 (CUDA0), 21-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 98.7% (5672 hits / 5748 lookups)",
+  "strata serve: decode expert cache hit rate: 99.4% (4287 hits / 4314 lookups)",
+  "strata serve: decode expert cache hit rate: 99.4% (4287 hits / 4315 lookups)",
+  "strata serve: decode expert cache hit rate: 99.3% (29024 hits / 29229 lookups)",
+  "strata serve: decode expert cache hit rate: 99.1% (4278 hits / 4316 lookups)",
+  "strata serve: decode expert cache hit rate: 98.8% (131499 hits / 133085 lookups)",
+  "strata serve: decode expert cache hit rate: 99.6% (130974 hits / 131451 lookups)",
+  "strata serve: decode expert cache hit rate: 99.7% (138767 hits / 139134 lookups)",
+  "strata serve: decode expert cache hit rate: 98.8% (121454 hits / 122932 lookups)",
+  "strata serve: decode expert cache hit rate: 99.8% (116809 hits / 117079 lookups)",
+  "strata serve: decode expert cache hit rate: 99.8% (118803 hits / 119015 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-ks-k22.json

@@ -0,0 +1,85 @@
+{
+ "label": "ks-k22",
+ "extra": [
+  "--layer-split",
+  "22"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 8730.9,
+   "pp_tok_s": 1840.1,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 13375.4,
+   "pp_tok_s": 2074.2,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3611.4,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 234.8,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 80.2,
+   "reps": [
+    78.4,
+    80.9,
+    81.5
+   ],
+   "hit_rate": 0.9977764197412695,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was seventy-three years old, and he had been the keeper of the Blackwood Point Light for forty of those years. The other thirty-three had been spent in a small, drafty cottage on the mainland, waiting for the day when the sea would finally claim him.\n\nTonight, the storm was a Category 4. The waves weren\u2019t just crashing; they were climbing. They reached the gallery railing, ten feet above the base of the tower, spraying a fine mist of brine that coated Elias\u2019s glasses.\n\nHe climbed the spiral stairs, his knees protesting with every step. The iron steps were cold, slick with condensation. At the top, in the lantern room, the air was thin and sharp. The great Fresnel lens, a cathedral of glass and brass, stood silent. It was time to wake it.\n\n"
+  },
+  {
+   "name": "code",
+   "tok_s": 106.4,
+   "reps": [
+    100.7,
+    110.3,
+    108.1
+   ],
+   "hit_rate": 0.9971798370756316,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 529 windows; per window: wait for the GPU 7.229 ms, pool + plan 0.561 ms, host staging 0.529 ms, commit 0.233 ms",
+  "strata serve: stage 1: 529 windows; per window: wait for the GPU 13.058 ms, pool + plan 0.253 ms, host staging 0.000 ms, commit 0.334 ms",
+  "strata serve: stage 0: 598 windows; per window: wait for the GPU 7.325 ms, pool + plan 0.552 ms, host staging 0.469 ms, commit 0.235 ms",
+  "strata serve: stage 1: 598 windows; per window: wait for the GPU 13.238 ms, pool + plan 0.260 ms, host staging 0.000 ms, commit 0.337 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split 22)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4640 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 22-47, expert cache 6656 slots (13.34 GiB), 6656 of its 6656 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-21",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-21 (CUDA0), 22-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 97.7% (5609 hits / 5739 lookups)",
+  "strata serve: decode expert cache hit rate: 98.7% (4256 hits / 4310 lookups)",
+  "strata serve: decode expert cache hit rate: 99.1% (4278 hits / 4315 lookups)",
+  "strata serve: decode expert cache hit rate: 99.0% (27008 hits / 27289 lookups)",
+  "strata serve: decode expert cache hit rate: 98.8% (4260 hits / 4313 lookups)",
+  "strata serve: decode expert cache hit rate: 98.5% (135183 hits / 137274 lookups)",
+  "strata serve: decode expert cache hit rate: 99.6% (136132 hits / 136708 lookups)",
+  "strata serve: decode expert cache hit rate: 99.8% (135515 hits / 135817 lookups)",
+  "strata serve: decode expert cache hit rate: 98.4% (121287 hits / 123280 lookups)",
+  "strata serve: decode expert cache hit rate: 99.7% (117658 hits / 118026 lookups)",
+  "strata serve: decode expert cache hit rate: 99.7% (120574 hits / 120915 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-ks-k23.json

@@ -0,0 +1,85 @@
+{
+ "label": "ks-k23",
+ "extra": [
+  "--layer-split",
+  "23"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 8742.6,
+   "pp_tok_s": 1837.7,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 13084.2,
+   "pp_tok_s": 2120.3,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3566.1,
+  "turn2_tokens": 6660,
+  "reused": 6631,
+  "turn2_ms": 238.0,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 81.0,
+   "reps": [
+    76.8,
+    85.2,
+    81.1
+   ],
+   "hit_rate": 0.9961302681992337,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias had lived in the tower for thirty years, and he knew the difference between a gale and a hurricane by the way the glass rattled in its frame. Tonight, it was the latter. The sea was a churning black void, swallowing the horizon, throwing white foam against the granite base like a tantrum.\n\nInside the lantern room, the air was thick with the smell of kerosene and old brass. Elias wiped his hands on his wool trousers, the fabric stiff with salt and grease. He checked the clockwork mechanism that turned the lens. It was a beast of a thing, a Fresnel lens of eight feet in diameter, a cathedral of glass and crystal that focused the beam of the lamp into a single, piercing needle of light.\n\nHe wound the weights. *Click, click, click.* The gears engaged. The massive lens began its slow, hypnotic rotation.\n\nFor a moment, the"
+  },
+  {
+   "name": "code",
+   "tok_s": 108.4,
+   "reps": [
+    105.1,
+    109.0,
+    111.1
+   ],
+   "hit_rate": 0.9961698810290479,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 537 windows; per window: wait for the GPU 7.423 ms, pool + plan 0.953 ms, host staging 0.542 ms, commit 0.237 ms",
+  "strata serve: stage 1: 537 windows; per window: wait for the GPU 12.223 ms, pool + plan 0.236 ms, host staging 0.000 ms, commit 0.324 ms",
+  "strata serve: stage 0: 606 windows; per window: wait for the GPU 7.519 ms, pool + plan 0.904 ms, host staging 0.483 ms, commit 0.239 ms",
+  "strata serve: stage 1: 606 windows; per window: wait for the GPU 12.395 ms, pool + plan 0.242 ms, host staging 0.000 ms, commit 0.326 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split 23)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4609 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 23-47, expert cache 6400 slots (12.82 GiB), 6400 of its 6400 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-22",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-22 (CUDA0), 23-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 96.8% (5528 hits / 5709 lookups)",
+  "strata serve: decode expert cache hit rate: 98.4% (4235 hits / 4303 lookups)",
+  "strata serve: decode expert cache hit rate: 98.9% (4260 hits / 4308 lookups)",
+  "strata serve: decode expert cache hit rate: 98.6% (30643 hits / 31089 lookups)",
+  "strata serve: decode expert cache hit rate: 98.6% (4248 hits / 4309 lookups)",
+  "strata serve: decode expert cache hit rate: 98.3% (134776 hits / 137143 lookups)",
+  "strata serve: decode expert cache hit rate: 99.4% (133024 hits / 133766 lookups)",
+  "strata serve: decode expert cache hit rate: 99.6% (129995 hits / 130500 lookups)",
+  "strata serve: decode expert cache hit rate: 97.9% (115782 hits / 118287 lookups)",
+  "strata serve: decode expert cache hit rate: 99.6% (117953 hits / 118464 lookups)",
+  "strata serve: decode expert cache hit rate: 99.6% (117560 hits / 118012 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-ks-k24.json

@@ -0,0 +1,85 @@
+{
+ "label": "ks-k24",
+ "extra": [
+  "--layer-split",
+  "24"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 8122.5,
+   "pp_tok_s": 1978.0,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 12285.8,
+   "pp_tok_s": 2258.1,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3407.0,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 234.0,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 83.5,
+   "reps": [
+    84.9,
+    83.3,
+    82.3
+   ],
+   "hit_rate": 0.9949381327334084,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the rough fabric scratching against his stubble. Outside the thick, salt-crusted glass of the lantern room, the Atlantic was a churning cauldron of black and white foam. Rain lashed against the panes in horizontal sheets, blurring the world into a smear of gray and shadow.\n\nInside, however, it was warm. The smell of kerosene and old wood hung heavy in the air, a scent that had defined Elias\u2019s life for thirty years. He was not merely a keeper; he was a guardian of silence.\n\nHe checked the clock. 11:42 PM. The rotation of the great Fresnel lens was due. With a practiced hand, he engaged the brake on the clockwork mechanism. The heavy brass gears groaned, then began their slow, rhythmic turn. *Tick. Tick. Tick.* It was a heartbeat"
+  },
+  {
+   "name": "code",
+   "tok_s": 109.5,
+   "reps": [
+    104.9,
+    111.0,
+    112.6
+   ],
+   "hit_rate": 0.9953796671318199,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 519 windows; per window: wait for the GPU 7.835 ms, pool + plan 0.767 ms, host staging 0.557 ms, commit 0.244 ms",
+  "strata serve: stage 1: 519 windows; per window: wait for the GPU 11.922 ms, pool + plan 0.232 ms, host staging 0.000 ms, commit 0.325 ms",
+  "strata serve: stage 0: 587 windows; per window: wait for the GPU 7.943 ms, pool + plan 0.744 ms, host staging 0.497 ms, commit 0.246 ms",
+  "strata serve: stage 1: 587 windows; per window: wait for the GPU 12.073 ms, pool + plan 0.238 ms, host staging 0.000 ms, commit 0.326 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split 24)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4603 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 24-47, expert cache 6144 slots (12.35 GiB), 6144 of its 6144 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-23",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-23 (CUDA0), 24-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 95.9% (5916 hits / 6171 lookups)",
+  "strata serve: decode expert cache hit rate: 97.4% (3717 hits / 3815 lookups)",
+  "strata serve: decode expert cache hit rate: 98.4% (3761 hits / 3823 lookups)",
+  "strata serve: decode expert cache hit rate: 98.1% (27138 hits / 27664 lookups)",
+  "strata serve: decode expert cache hit rate: 97.8% (3742 hits / 3826 lookups)",
+  "strata serve: decode expert cache hit rate: 97.9% (126114 hits / 128875 lookups)",
+  "strata serve: decode expert cache hit rate: 99.2% (133626 hits / 134658 lookups)",
+  "strata serve: decode expert cache hit rate: 99.5% (132675 hits / 133350 lookups)",
+  "strata serve: decode expert cache hit rate: 96.8% (113593 hits / 117399 lookups)",
+  "strata serve: decode expert cache hit rate: 99.5% (118372 hits / 118949 lookups)",
+  "strata serve: decode expert cache hit rate: 99.5% (116981 hits / 117524 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-ks-k26.json

@@ -0,0 +1,85 @@
+{
+ "label": "ks-k26",
+ "extra": [
+  "--layer-split",
+  "26"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 7880.9,
+   "pp_tok_s": 2038.6,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 11771.9,
+   "pp_tok_s": 2356.7,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3323.9,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 231.5,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 83.8,
+   "reps": [
+    85.9,
+    83.1,
+    82.4
+   ],
+   "hit_rate": 0.9933966961830739,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the rough fabric scratching against his stubble. Outside the thick, salt-crusted glass of the lantern room, the Atlantic was a churning cauldron of black and white foam. Rain lashed against the panes in horizontal sheets, blurring the world into a smear of gray and green.\n\nInside, however, it was warm. The smell of kerosene and old wood hung heavy in the air, a scent that had defined Elias\u2019s life for thirty years. He was a keeper of the light, but more importantly, he was a keeper of time.\n\nHe checked his pocket watch. 11:42 PM. The rotation of the great Fresnel lens was due. With a practiced, rhythmic *clunk-clunk-clunk*, he wound the clockwork mechanism. The massive lens began its slow, hypnotic turn, sweeping its beam across"
+  },
+  {
+   "name": "code",
+   "tok_s": 109.7,
+   "reps": [
+    104.4,
+    112.7,
+    111.9
+   ],
+   "hit_rate": 0.9938670271352759,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 519 windows; per window: wait for the GPU 8.376 ms, pool + plan 1.030 ms, host staging 0.566 ms, commit 0.273 ms",
+  "strata serve: stage 1: 519 windows; per window: wait for the GPU 10.886 ms, pool + plan 0.214 ms, host staging 0.000 ms, commit 0.297 ms",
+  "strata serve: stage 0: 588 windows; per window: wait for the GPU 8.506 ms, pool + plan 0.985 ms, host staging 0.505 ms, commit 0.274 ms",
+  "strata serve: stage 1: 588 windows; per window: wait for the GPU 11.036 ms, pool + plan 0.219 ms, host staging 0.000 ms, commit 0.298 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split 26)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4568 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 26-47, expert cache 5632 slots (11.32 GiB), 5632 of its 5632 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-25",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-25 (CUDA0), 26-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 92.8% (5259 hits / 5665 lookups)",
+  "strata serve: decode expert cache hit rate: 95.2% (4056 hits / 4259 lookups)",
+  "strata serve: decode expert cache hit rate: 97.2% (4164 hits / 4285 lookups)",
+  "strata serve: decode expert cache hit rate: 97.1% (27690 hits / 28510 lookups)",
+  "strata serve: decode expert cache hit rate: 97.5% (4184 hits / 4293 lookups)",
+  "strata serve: decode expert cache hit rate: 96.9% (121839 hits / 125724 lookups)",
+  "strata serve: decode expert cache hit rate: 98.9% (132165 hits / 133648 lookups)",
+  "strata serve: decode expert cache hit rate: 99.3% (137652 hits / 138567 lookups)",
+  "strata serve: decode expert cache hit rate: 95.3% (111307 hits / 116822 lookups)",
+  "strata serve: decode expert cache hit rate: 99.3% (118436 hits / 119329 lookups)",
+  "strata serve: decode expert cache hit rate: 99.4% (119109 hits / 119844 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-ks-k28.json

@@ -0,0 +1,85 @@
+{
+ "label": "ks-k28",
+ "extra": [
+  "--layer-split",
+  "28"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 7989.3,
+   "pp_tok_s": 2010.9,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 12132.9,
+   "pp_tok_s": 2286.6,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3433.5,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 228.0,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 83.5,
+   "reps": [
+    83.7,
+    82.4,
+    84.5
+   ],
+   "hit_rate": 0.9900441970361987,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias had lived in the tower for thirty years, and he knew the difference between a gale and a hurricane by the way the glass rattled in its frame. Tonight, the glass was vibrating like a plucked cello string. The sea, usually a rhythmic pulse against the granite base, was now a chaotic, frothing beast, throwing itself against the rocks with the force of a battering ram.\n\nElias climbed the spiral stairs, his hand gripping the cold iron railing. His knees ached, not from age, but from the constant vibration of the floor beneath his feet. He reached the lantern room, the heart of the lighthouse, and checked the clockwork mechanism. The great lens, a prism of Fresnel glass, rotated with a steady, hypnotic *whirrr-click, whirrr-click*. It was the only sound that mattered. The light swept out into the black void, cutting through"
+  },
+  {
+   "name": "code",
+   "tok_s": 112.5,
+   "reps": [
+    101.4,
+    112.1,
+    124.1
+   ],
+   "hit_rate": 0.9896636587366694,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv(file_path: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys being the header values\n    and values being the corresponding cell values.\n    \n    Args:\n        file_path: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result = []\n    with open(file_path, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n```<|im_end|>"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 527 windows; per window: wait for the GPU 8.765 ms, pool + plan 1.401 ms, host staging 0.593 ms, commit 0.281 ms",
+  "strata serve: stage 1: 527 windows; per window: wait for the GPU 9.893 ms, pool + plan 0.195 ms, host staging 0.000 ms, commit 0.276 ms",
+  "strata serve: stage 0: 578 windows; per window: wait for the GPU 8.908 ms, pool + plan 1.359 ms, host staging 0.559 ms, commit 0.284 ms",
+  "strata serve: stage 1: 578 windows; per window: wait for the GPU 10.025 ms, pool + plan 0.200 ms, host staging 0.000 ms, commit 0.278 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split 28)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4537 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 28-47, expert cache 5120 slots (10.30 GiB), 5120 of its 5120 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-27",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-27 (CUDA0), 28-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 88.8% (5405 hits / 6086 lookups)",
+  "strata serve: decode expert cache hit rate: 93.8% (3539 hits / 3773 lookups)",
+  "strata serve: decode expert cache hit rate: 94.2% (3560 hits / 3781 lookups)",
+  "strata serve: decode expert cache hit rate: 95.7% (26702 hits / 27899 lookups)",
+  "strata serve: decode expert cache hit rate: 95.9% (3647 hits / 3804 lookups)",
+  "strata serve: decode expert cache hit rate: 95.7% (122618 hits / 128160 lookups)",
+  "strata serve: decode expert cache hit rate: 98.6% (135035 hits / 136884 lookups)",
+  "strata serve: decode expert cache hit rate: 99.0% (129476 hits / 130778 lookups)",
+  "strata serve: decode expert cache hit rate: 93.9% (111037 hits / 118290 lookups)",
+  "strata serve: decode expert cache hit rate: 98.9% (118829 hits / 120140 lookups)",
+  "strata serve: decode expert cache hit rate: 99.0% (90480 hits / 91425 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-ks-k30.json

@@ -0,0 +1,85 @@
+{
+ "label": "ks-k30",
+ "extra": [
+  "--layer-split",
+  "30"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 8301.3,
+   "pp_tok_s": 1935.4,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 12507.6,
+   "pp_tok_s": 2218.1,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3448.7,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 229.4,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 81.1,
+   "reps": [
+    78.0,
+    84.4,
+    81.0
+   ],
+   "hit_rate": 0.9798126997277784,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias had lived in the tower for thirty years, and he knew the difference between a gale and a hurricane by the way the glass rattled in its frame. Tonight, the glass was vibrating like a plucked cello string. The sea, usually a rhythmic pulse against the granite base, was now a chaotic, frothing beast, throwing itself against the rocks with the force of a battering ram.\n\nElias stood at the top of the spiral staircase, his hand resting on the cold iron railing. He was not afraid. Fear was for the tourists who came in summer, wide-eyed and clutching their cameras. Elias was a keeper. He was a guardian.\n\nHe checked the logbook. The entry for the night was simple: *Wind, 60 knots. Visibility, zero. Light, operational.*\n\nHe walked down the stairs, his boots heavy on the metal grating. The smell of salt"
+  },
+  {
+   "name": "code",
+   "tok_s": 109.6,
+   "reps": [
+    100.7,
+    114.1,
+    114.1
+   ],
+   "hit_rate": 0.9825370343977666,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 540 windows; per window: wait for the GPU 9.206 ms, pool + plan 1.794 ms, host staging 0.525 ms, commit 0.298 ms",
+  "strata serve: stage 1: 540 windows; per window: wait for the GPU 8.767 ms, pool + plan 0.173 ms, host staging 0.000 ms, commit 0.242 ms",
+  "strata serve: stage 0: 608 windows; per window: wait for the GPU 9.390 ms, pool + plan 1.733 ms, host staging 0.471 ms, commit 0.301 ms",
+  "strata serve: stage 1: 608 windows; per window: wait for the GPU 8.910 ms, pool + plan 0.178 ms, host staging 0.000 ms, commit 0.244 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split 30)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4479 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 30-47, expert cache 4608 slots (9.19 GiB), 4608 of its 4608 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-29",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-29 (CUDA0), 30-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 84.6% (5099 hits / 6028 lookups)",
+  "strata serve: decode expert cache hit rate: 90.7% (3403 hits / 3751 lookups)",
+  "strata serve: decode expert cache hit rate: 91.1% (3415 hits / 3747 lookups)",
+  "strata serve: decode expert cache hit rate: 94.1% (26108 hits / 27737 lookups)",
+  "strata serve: decode expert cache hit rate: 93.4% (3534 hits / 3784 lookups)",
+  "strata serve: decode expert cache hit rate: 94.6% (127532 hits / 134866 lookups)",
+  "strata serve: decode expert cache hit rate: 98.6% (135828 hits / 137786 lookups)",
+  "strata serve: decode expert cache hit rate: 98.0% (132455 hits / 135184 lookups)",
+  "strata serve: decode expert cache hit rate: 91.0% (106351 hits / 116917 lookups)",
+  "strata serve: decode expert cache hit rate: 97.9% (115791 hits / 118246 lookups)",
+  "strata serve: decode expert cache hit rate: 98.3% (117198 hits / 119281 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-ks-k32.json

@@ -0,0 +1,85 @@
+{
+ "label": "ks-k32",
+ "extra": [
+  "--layer-split",
+  "32"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 8837.1,
+   "pp_tok_s": 1818.0,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 13449.8,
+   "pp_tok_s": 2062.7,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3565.7,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 227.0,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 83.0,
+   "reps": [
+    75.0,
+    86.1,
+    87.8
+   ],
+   "hit_rate": 0.9796934433020751,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias had lived in the tower for thirty years, and he knew the difference between a gale and a hurricane by the way the glass rattled in its frame. Tonight, it was the latter. The sea was a churning cauldron of black ink and white foam, throwing itself against the granite base with the force of a battering ram.\n\nInside the lantern room, the air was thick with the smell of kerosene and old brass. Elias wiped his hands on his woolen trousers, leaving faint, oily smears on the fabric. He checked the clock. 11:42 PM. The rotation was due.\n\nHe reached for the crank handle, a heavy iron wheel worn smooth by decades of his own palms. With a grunt that matched the strain of his shoulders, he began to turn it. *Click. Click. Click.* The gears engaged, and the massive Fresnel lens began its slow"
+  },
+  {
+   "name": "code",
+   "tok_s": 108.7,
+   "reps": [
+    98.8,
+    112.2,
+    115.1
+   ],
+   "hit_rate": 0.9777186846317358,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 533 windows; per window: wait for the GPU 9.839 ms, pool + plan 2.192 ms, host staging 0.585 ms, commit 0.318 ms",
+  "strata serve: stage 1: 533 windows; per window: wait for the GPU 8.014 ms, pool + plan 0.156 ms, host staging 0.000 ms, commit 0.231 ms",
+  "strata serve: stage 0: 602 windows; per window: wait for the GPU 10.014 ms, pool + plan 2.111 ms, host staging 0.520 ms, commit 0.321 ms",
+  "strata serve: stage 1: 602 windows; per window: wait for the GPU 8.094 ms, pool + plan 0.160 ms, host staging 0.000 ms, commit 0.232 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split 32)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4488 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 32-47, expert cache 4096 slots (8.26 GiB), 4096 of its 4096 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-31",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-31 (CUDA0), 32-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 81.3% (4889 hits / 6010 lookups)",
+  "strata serve: decode expert cache hit rate: 87.7% (3272 hits / 3733 lookups)",
+  "strata serve: decode expert cache hit rate: 87.9% (3280 hits / 3733 lookups)",
+  "strata serve: decode expert cache hit rate: 92.3% (24238 hits / 26254 lookups)",
+  "strata serve: decode expert cache hit rate: 91.5% (3457 hits / 3778 lookups)",
+  "strata serve: decode expert cache hit rate: 93.7% (133254 hits / 142240 lookups)",
+  "strata serve: decode expert cache hit rate: 98.0% (130112 hits / 132812 lookups)",
+  "strata serve: decode expert cache hit rate: 98.0% (130069 hits / 132765 lookups)",
+  "strata serve: decode expert cache hit rate: 89.1% (105263 hits / 118190 lookups)",
+  "strata serve: decode expert cache hit rate: 97.2% (116970 hits / 120309 lookups)",
+  "strata serve: decode expert cache hit rate: 97.8% (115450 hits / 118081 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-ks-single.json

@@ -0,0 +1,76 @@
+{
+ "label": "ks-single",
+ "extra": [],
+ "gpus": "0",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 9310.4,
+   "pp_tok_s": 1725.6,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 14085.4,
+   "pp_tok_s": 1969.6,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3388.7,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 200.7,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 86.7,
+   "reps": [
+    78.2,
+    91.4,
+    90.4
+   ],
+   "hit_rate": 0.921882148773369,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was seventy-three years old, and his hands were a map of the coastline he had guarded for fifty years. The knuckles were swollen, the skin cracked like dry earth, but they moved with a precision that no machine could replicate.\n\nHe stood on the gallery of the *Harrow Point* lighthouse, the iron railing cold against his palms. Below, the Atlantic churned in a frenzy, whitecaps exploding against the black rocks like fireworks. Above, the sky was a bruised purple, heavy with rain that hadn\u2019t yet fallen.\n\nElias checked the clock. 11:42 PM. The light was due.\n\nHe didn\u2019t look at the gauge. He didn\u2019t check the fuel levels. He knew. He always knew. The rhythm of the ocean, the pressure of the air, the subtle vibration"
+  },
+  {
+   "name": "code",
+   "tok_s": 105.2,
+   "reps": [
+    87.8,
+    113.5,
+    114.2
+   ],
+   "hit_rate": 0.9109811738648947,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 382 windows; per window: wait for the GPU 15.155 ms, pool + plan 4.331 ms, host staging 0.656 ms, commit 0.458 ms",
+  "strata serve: stage 0: 455 windows; per window: wait for the GPU 15.790 ms, pool + plan 5.552 ms, host staging 0.688 ms, commit 0.467 ms",
+  "strata serve: stage 0: 524 windows; per window: wait for the GPU 16.064 ms, pool + plan 5.529 ms, host staging 0.606 ms, commit 0.472 ms",
+  "strata serve: stage 0: 593 windows; per window: wait for the GPU 16.305 ms, pool + plan 5.462 ms, host staging 0.540 ms, commit 0.475 ms"
+ ],
+ "engine_lines": [
+  "strata generate: expert cache auto: 9.15 GiB free, 700 MiB reserved -> 3415 slots",
+  "strata generate: expert cache 4293 slots, 8.17 GiB of VRAM; policy is",
+  "strata serve: the prompt path borrows 2343 cache slots (4.42 GiB)",
+  "strata serve: decode expert cache hit rate: 59.6% (2749 hits / 4610 lookups)",
+  "strata serve: decode expert cache hit rate: 67.8% (2299 hits / 3391 lookups)",
+  "strata serve: decode expert cache hit rate: 68.5% (2345 hits / 3424 lookups)",
+  "strata serve: decode expert cache hit rate: 75.6% (16605 hits / 21959 lookups)",
+  "strata serve: decode expert cache hit rate: 73.3% (2608 hits / 3558 lookups)",
+  "strata serve: decode expert cache hit rate: 83.7% (104878 hits / 125346 lookups)",
+  "strata serve: decode expert cache hit rate: 92.7% (119739 hits / 129184 lookups)",
+  "strata serve: decode expert cache hit rate: 92.2% (116867 hits / 126770 lookups)",
+  "strata serve: decode expert cache hit rate: 68.1% (64111 hits / 94116 lookups)",
+  "strata serve: decode expert cache hit rate: 89.0% (96370 hits / 108259 lookups)",
+  "strata serve: decode expert cache hit rate: 91.1% (102827 hits / 112875 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-rel-5080.json

@@ -0,0 +1,72 @@
+{
+ "label": "rel-5080",
+ "extra": [],
+ "gpus": "0",
+ "needles": [
+  {
+   "tokens": 2726,
+   "prompt_ms": 2132.8,
+   "pp_tok_s": 1278.1,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 8379,
+   "prompt_ms": 5008.9,
+   "pp_tok_s": 1672.8,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 16066,
+   "prompt_ms": 7855.4,
+   "pp_tok_s": 2045.2,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 13833.9,
+   "pp_tok_s": 2005.4,
+   "found": true,
+   "answer": "willow-474<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3298.2,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 197.9,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 83.2,
+   "hit_rate": 0.8418683557684931,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias had lived in the tower for thirty years, and he knew the difference between a gale and a hurricane by the way the glass rattled in its frame. Tonight, it was the latter. The sea was a churning cauldron of black ink and white foam, throwing itself against the granite base with the force of a battering ram.\n\nInside the lantern room, the air was thick with the smell of kerosene and old brass. Elias wiped his hands on his woolen trousers, leaving faint, oily smears on the fabric. He checked the clock. 11:42 PM. The rotation was due.\n\nHe reached for the crank handle, a heavy iron wheel worn smooth by decades of his own palms. With a grunt that matched the strain of his shoulders, he began to turn it. *Click. Click. Click.* The gears engaged, and the massive Fresnel lens began its slow"
+  },
+  {
+   "name": "code",
+   "tok_s": 88.2,
+   "hit_rate": 0.7011385968111186,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "engine_lines": [
+  "strata generate: expert cache auto: 9.15 GiB free, 700 MiB reserved -> 3415 slots",
+  "strata generate: expert cache 4293 slots, 8.17 GiB of VRAM; policy is",
+  "strata serve: the prompt path borrows 2343 cache slots (4.42 GiB)",
+  "strata serve: decode expert cache hit rate: 59.6% (2749 hits / 4610 lookups)",
+  "strata serve: decode expert cache hit rate: 64.2% (2136 hits / 3329 lookups)",
+  "strata serve: decode expert cache hit rate: 67.3% (2309 hits / 3432 lookups)",
+  "strata serve: decode expert cache hit rate: 72.5% (2539 hits / 3504 lookups)",
+  "strata serve: decode expert cache hit rate: 77.3% (2817 hits / 3642 lookups)",
+  "strata serve: decode expert cache hit rate: 75.3% (16500 hits / 21916 lookups)",
+  "strata serve: decode expert cache hit rate: 80.0% (2978 hits / 3723 lookups)",
+  "strata serve: decode expert cache hit rate: 84.2% (98752 hits / 117301 lookups)",
+  "strata serve: decode expert cache hit rate: 70.1% (69215 hits / 98718 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-t75-5080.json

@@ -0,0 +1,78 @@
+{
+ "label": "t75-5080",
+ "extra": [],
+ "gpus": "0",
+ "needles": [
+  {
+   "tokens": 2726,
+   "prompt_ms": 2158.8,
+   "pp_tok_s": 1262.7,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 8379,
+   "prompt_ms": 5060.0,
+   "pp_tok_s": 1655.9,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3389.7,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 213.5,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 80.4,
+   "reps": [
+    80.4
+   ],
+   "hit_rate": 0.855538332528651,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias had lived in the tower for thirty years, and he knew the difference between a gale and a hurricane by the way the glass rattled in its frame. Tonight, it was the latter. The sea was a churning black void, punctuated by whitecaps that looked like teeth.\n\nHe climbed the spiral stairs, his boots heavy with salt and exhaustion. The iron steps groaned under his weight, a sound that had become so familiar to him that he could sleep to it. But tonight, he wasn\u2019t sleeping. He was working.\n\nThe lens was the heart of the lighthouse. A massive, rotating Fresnel lens, a cathedral of glass and brass that turned with a mechanical hum. Elias checked the clockwork mechanism. It was smooth, rhythmic, a heartbeat in the dark. But the light itself was failing.\n\nHe climbed higher, into the lantern room. The wind battered the glass panes,"
+  },
+  {
+   "name": "code",
+   "tok_s": 87.5,
+   "reps": [
+    87.5
+   ],
+   "hit_rate": 0.7075461363025227,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 39 windows; per window: wait for the GPU 17.252 ms, pool + plan 8.473 ms, host staging 0.611 ms, commit 0.525 ms",
+  "strata serve: stage 0: 47 windows; per window: wait for the GPU 18.018 ms, pool + plan 8.616 ms, host staging 0.607 ms, commit 0.534 ms",
+  "strata serve: stage 0: 169 windows; per window: wait for the GPU 15.223 ms, pool + plan 5.717 ms, host staging 0.688 ms, commit 0.467 ms",
+  "strata serve: stage 0: 243 windows; per window: wait for the GPU 16.500 ms, pool + plan 7.316 ms, host staging 0.742 ms, commit 0.481 ms"
+ ],
+ "engine_lines": [
+  "strata generate: expert cache auto: 9.15 GiB free, 700 MiB reserved -> 3415 slots",
+  "strata generate: expert cache 4294 slots, 8.18 GiB of VRAM; policy is",
+  "strata serve: the prompt path borrows 2343 cache slots (4.42 GiB)",
+  "strata serve: decode expert cache hit rate: 59.6% (2749 hits / 4610 lookups)",
+  "strata serve: decode expert cache hit rate: 64.2% (2136 hits / 3329 lookups)",
+  "strata serve: decode expert cache hit rate: 67.3% (2309 hits / 3432 lookups)",
+  "strata serve: decode expert cache hit rate: 75.5% (16316 hits / 21602 lookups)",
+  "strata serve: decode expert cache hit rate: 74.4% (2660 hits / 3573 lookups)",
+  "strata serve: decode expert cache hit rate: 83.4% (92664 hits / 111086 lookups)",
+  "strata serve: decode expert cache hit rate: 72.0% (70946 hits / 98601 lookups)",
+  "strata generate: expert cache auto: 9.15 GiB free, 700 MiB reserved -> 3415 slots",
+  "strata generate: expert cache 4293 slots, 8.17 GiB of VRAM; policy is",
+  "strata serve: the prompt path borrows 2343 cache slots (4.42 GiB)",
+  "strata serve: decode expert cache hit rate: 59.6% (2749 hits / 4610 lookups)",
+  "strata serve: decode expert cache hit rate: 64.2% (2136 hits / 3329 lookups)",
+  "strata serve: decode expert cache hit rate: 67.3% (2309 hits / 3432 lookups)",
+  "strata serve: decode expert cache hit rate: 75.5% (16316 hits / 21602 lookups)",
+  "strata serve: decode expert cache hit rate: 74.4% (2659 hits / 3572 lookups)",
+  "strata serve: decode expert cache hit rate: 85.6% (101004 hits / 118059 lookups)",
+  "strata serve: decode expert cache hit rate: 70.8% (71044 hits / 100409 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-v-x2-auto.json

@@ -0,0 +1,103 @@
+{
+ "label": "v-x2-auto",
+ "extra": [
+  "--layer-split",
+  "auto"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 2726,
+   "prompt_ms": 2577.0,
+   "pp_tok_s": 1057.8,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 8379,
+   "prompt_ms": 4433.5,
+   "pp_tok_s": 1889.9,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 16066,
+   "prompt_ms": 7885.8,
+   "pp_tok_s": 2037.3,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 13384.6,
+   "pp_tok_s": 2072.8,
+   "found": true,
+   "answer": "willow-474<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3667.5,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 253.8,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 79.5,
+   "reps": [
+    78.9,
+    79.0,
+    80.7
+   ],
+   "hit_rate": 0.9969903443250137,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the rough fabric scratching against his stubble. Outside the thick, salt-crusted glass of the lantern room, the Atlantic was a churning cauldron of black and white foam. Rain lashed against the panes in horizontal sheets, blurring the world into a smear of gray and shadow.\n\nInside, however, it was warm. The smell of kerosene and old wood hung heavy in the air, a scent that had defined his life for thirty years.\n\nElias was not a man of many words. He was a man of many tasks. His hands, mapped with scars from rope burns and broken glass, moved with a rhythm that predated his memory. He checked the clockwork mechanism of the Fresnel lens, listening for the faint, rhythmic *tick-tick-tick* that signaled the rotation was true. If the rhythm faltered, ships"
+  },
+  {
+   "name": "code",
+   "tok_s": 108.9,
+   "reps": [
+    103.3,
+    113.7,
+    109.5
+   ],
+   "hit_rate": 0.9971795386830036,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 542 windows; per window: wait for the GPU 7.167 ms, pool + plan 0.572 ms, host staging 0.550 ms, commit 0.232 ms",
+  "strata serve: stage 1: 542 windows; per window: wait for the GPU 12.855 ms, pool + plan 0.250 ms, host staging 0.000 ms, commit 0.334 ms",
+  "strata serve: stage 0: 610 windows; per window: wait for the GPU 7.276 ms, pool + plan 0.559 ms, host staging 0.493 ms, commit 0.234 ms",
+  "strata serve: stage 1: 610 windows; per window: wait for the GPU 13.048 ms, pool + plan 0.258 ms, host staging 0.000 ms, commit 0.337 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split auto)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: layer split auto: CUDA0 84 SMs at 2.81 GHz -> 0.31 ms per layer, 8.30 GiB for experts",
+  "strata generate: layer split auto: CUDA1 82 SMs at 1.86 GHz -> 0.48 ms per layer, 14.14 GiB for experts",
+  "strata generate: layer split auto: K=22 - predicted 19.7 ms per decode window; the caches hold 11298 of 12288 profiled pairs (~99.7% of the routed mass)",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4640 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 22-47, expert cache 6656 slots (13.34 GiB), 6656 of its 6656 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-21",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-21 (CUDA0), 22-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 97.7% (5609 hits / 5739 lookups)",
+  "strata serve: decode expert cache hit rate: 98.5% (4241 hits / 4306 lookups)",
+  "strata serve: decode expert cache hit rate: 98.9% (4262 hits / 4309 lookups)",
+  "strata serve: decode expert cache hit rate: 99.4% (4290 hits / 4315 lookups)",
+  "strata serve: decode expert cache hit rate: 99.6% (4303 hits / 4319 lookups)",
+  "strata serve: decode expert cache hit rate: 98.8% (28379 hits / 28721 lookups)",
+  "strata serve: decode expert cache hit rate: 99.3% (4285 hits / 4314 lookups)",
+  "strata serve: decode expert cache hit rate: 98.5% (136232 hits / 138256 lookups)",
+  "strata serve: decode expert cache hit rate: 99.5% (135540 hits / 136216 lookups)",
+  "strata serve: decode expert cache hit rate: 99.7% (136812 hits / 137225 lookups)",
+  "strata serve: decode expert cache hit rate: 98.4% (118437 hits / 120406 lookups)",
+  "strata serve: decode expert cache hit rate: 99.7% (112836 hits / 113230 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-v-x2r-auto.json

@@ -0,0 +1,88 @@
+{
+ "label": "v-x2r-auto",
+ "extra": [
+  "--layer-split",
+  "auto"
+ ],
+ "gpus": "2,0",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 9610.0,
+   "pp_tok_s": 1671.8,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 14786.1,
+   "pp_tok_s": 1876.3,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3850.8,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 245.0,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 90.3,
+   "reps": [
+    91.0,
+    97.0,
+    82.8
+   ],
+   "hit_rate": 0.993905472636816,
+   "text": "The wind didn\u2019t just blow; it screamed. It tore through the iron gallery of the Blackwood Point Lighthouse, rattling the glass panes like a prisoner shaking its bars.\n\nElias Thorne was seventy years old, and he had been the keeper for forty of them. He knew every creak of the spiral staircase, every draft that slipped under the door, and every shift in the barometric pressure that preceded a storm. He was not merely a worker; he was the heartbeat of the coast.\n\nIt was November, the month of grey seas and shorter days. The fog had rolled in thick, a wall of white so dense that even the beam of the Fresnel lens seemed to struggle against it. Elias stood in the lantern room, his hands roughened by salt and rope, adjusting the rotation mechanism. The great lens, a cathedral of glass and brass, began its slow, rhythmic sweep. *Flash. Dark. Flash. Dark.*\n\nHe checked the log"
+  },
+  {
+   "name": "code",
+   "tok_s": 105.1,
+   "reps": [
+    98.4,
+    109.9,
+    107.0
+   ],
+   "hit_rate": 0.9949900793650793,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 492 windows; per window: wait for the GPU 15.016 ms, pool + plan 0.297 ms, host staging 0.517 ms, commit 0.404 ms",
+  "strata serve: stage 1: 492 windows; per window: wait for the GPU 5.775 ms, pool + plan 0.854 ms, host staging 0.001 ms, commit 0.210 ms",
+  "strata serve: stage 0: 561 windows; per window: wait for the GPU 15.257 ms, pool + plan 0.306 ms, host staging 0.455 ms, commit 0.408 ms",
+  "strata serve: stage 1: 561 windows; per window: wait for the GPU 5.852 ms, pool + plan 0.820 ms, host staging 0.001 ms, commit 0.210 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split auto)",
+  "strata generate: layer split: CUDA1 PCIe probe 57.7 GB/s -> pcie_frac 0.55",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 9.98 GiB free",
+  "strata generate: layer split auto: CUDA0 82 SMs at 1.86 GHz -> 0.48 ms per layer, 16.45 GiB for experts",
+  "strata generate: layer split auto: CUDA1 84 SMs at 2.81 GHz -> 0.31 ms per layer, 5.99 GiB for experts",
+  "strata generate: layer split auto: K=30 - predicted 20.8 ms per decode window; the caches hold 10684 of 12288 profiled pairs (~99.5% of the routed mass)",
+  "strata generate: expert cache auto: 18.62 GiB free, 700 MiB reserved -> 6634 slots",
+  "strata generate: expert cache 7680 slots, 14.23 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 30-47, expert cache 3004 slots (5.98 GiB), 3004 of its 4608 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-29",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-29 (CUDA0), 30-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 93.2% (5367 hits / 5760 lookups)",
+  "strata serve: decode expert cache hit rate: 93.7% (4049 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 96.7% (4178 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 97.7% (26272 hits / 26880 lookups)",
+  "strata serve: decode expert cache hit rate: 97.1% (4194 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 97.8% (110748 hits / 113280 lookups)",
+  "strata serve: decode expert cache hit rate: 99.6% (113787 hits / 114240 lookups)",
+  "strata serve: decode expert cache hit rate: 99.4% (127856 hits / 128640 lookups)",
+  "strata serve: decode expert cache hit rate: 96.3% (116528 hits / 120960 lookups)",
+  "strata serve: decode expert cache hit rate: 99.4% (116423 hits / 117120 lookups)",
+  "strata serve: decode expert cache hit rate: 99.5% (120354 hits / 120960 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-v-x3-auto.json

@@ -0,0 +1,89 @@
+{
+ "label": "v-x3-auto",
+ "extra": [
+  "--layer-split",
+  "auto"
+ ],
+ "gpus": "0,1,2",
+ "needles": [
+  {
+   "tokens": 16066,
+   "prompt_ms": 9525.4,
+   "pp_tok_s": 1686.6,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 15017.1,
+   "pp_tok_s": 1847.4,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 4039.6,
+  "turn2_tokens": 6660,
+  "reused": 6631,
+  "turn2_ms": 360.2,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 67.6,
+   "reps": [
+    59.8,
+    83.1,
+    59.9
+   ],
+   "hit_rate": 0.9967581085879247,
+   "text": "The wind didn\u2019t just blow; it screamed. It tore through the iron gallery of the Blackwood Point Lighthouse, rattling the glass panes like a prisoner shaking its bars.\n\nElias Thorne was seventy years old, and he had been the keeper for forty of them. He knew every creak in the spiral staircase, every draft that slipped under the door, and every shift in the weather that the barometer couldn\u2019t quite predict. He knew that when the fog rolled in thick enough to swallow the horizon, the world ceased to exist. There was only the light, the stone, and him.\n\nIt was November, the month of storms. The sea was a churning cauldron of grey and white, throwing itself against the granite base with the force of a battering ram. Elias stood in the lantern room, his hands rough as sandpaper, adjusting the massive Fresnel lens. The rotation mechanism groaned, a sound like a ship\u2019s anchor chain being h"
+  },
+  {
+   "name": "code",
+   "tok_s": 89.6,
+   "reps": [
+    81.0,
+    104.8,
+    83.0
+   ],
+   "hit_rate": 0.9975041387598007,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 2: 522 windows; per window: wait for the GPU 12.405 ms, pool + plan 0.242 ms, host staging 0.006 ms, commit 0.375 ms",
+  "strata serve: stage 0: 590 windows; per window: wait for the GPU 7.387 ms, pool + plan 0.549 ms, host staging 0.517 ms, commit 0.248 ms",
+  "strata serve: stage 1: 590 windows; per window: wait for the GPU 3.071 ms, pool + plan 0.011 ms, host staging 0.000 ms, commit 0.606 ms",
+  "strata serve: stage 2: 590 windows; per window: wait for the GPU 12.592 ms, pool + plan 0.250 ms, host staging 0.006 ms, commit 0.377 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 3 GPUs: CUDA0, then CUDA1,2 (split auto)",
+  "strata generate: layer split: CUDA1 PCIe probe 3.2 GB/s -> pcie_frac 0.00",
+  "strata generate: layer split: CUDA1 holds its weights, session; 16.67 GiB free",
+  "strata generate: layer split: CUDA2 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA2 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: layer split auto: CUDA0 84 SMs at 2.81 GHz -> 0.31 ms per layer, 8.30 GiB for experts",
+  "strata generate: layer split auto: CUDA1 68 SMs at 1.54 GHz -> 0.69 ms per layer, 13.50 GiB for experts",
+  "strata generate: layer split auto: CUDA2 82 SMs at 1.86 GHz -> 0.48 ms per layer, 14.14 GiB for experts",
+  "strata generate: layer split auto: K=22,23 - predicted 19.9 ms per decode window; the caches hold 11298 of 12288 profiled pairs (~99.7% of the routed mass)",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4640 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 22-22, expert cache 256 slots (0.52 GiB), 256 of its 256 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA2 runs layers 23-47, expert cache 6400 slots (12.82 GiB), 6400 of its 6400 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-21",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-21 (CUDA0), 22-22 (CUDA1), 23-47 (CUDA2), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 97.7% (5609 hits / 5739 lookups)",
+  "strata serve: decode expert cache hit rate: 98.7% (4256 hits / 4310 lookups)",
+  "strata serve: decode expert cache hit rate: 99.1% (4278 hits / 4315 lookups)",
+  "strata serve: decode expert cache hit rate: 99.0% (29862 hits / 30170 lookups)",
+  "strata serve: decode expert cache hit rate: 98.9% (4264 hits / 4313 lookups)",
+  "strata serve: decode expert cache hit rate: 98.6% (134418 hits / 136339 lookups)",
+  "strata serve: decode expert cache hit rate: 99.5% (116986 hits / 117525 lookups)",
+  "strata serve: decode expert cache hit rate: 99.7% (129134 hits / 129554 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-x2-auto-c1024.json

@@ -0,0 +1,84 @@
+{
+ "label": "x2-auto-c1024",
+ "extra": [
+  "--layer-split",
+  "auto",
+  "--prefill",
+  "1024"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 2726,
+   "prompt_ms": 2648.7,
+   "pp_tok_s": 1029.2,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 8379,
+   "prompt_ms": 5547.0,
+   "pp_tok_s": 1510.5,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 16066,
+   "prompt_ms": 10123.3,
+   "pp_tok_s": 1587.0,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 17652.0,
+   "pp_tok_s": 1571.7,
+   "found": true,
+   "answer": "willow-474<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 4379.9,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 235.5,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 82.7,
+   "hit_rate": 1.0,
+   "text": "The wind didn\u2019t just blow; it screamed. It tore through the iron gallery of the Blackwood Point Lighthouse, rattling the glass panes like a prisoner shaking its bars.\n\nElias Thorne was seventy years old, and he had been the keeper for forty of them. He knew every creak in the spiral staircase, every draft that slipped under the door, and every shift in the weather that the barometer couldn\u2019t quite predict. He knew that when the fog rolled in thick enough to swallow the horizon, the world ceased to exist. There was only the light, the stone, and him.\n\nIt was November, the month of storms. The sea was a churning cauldron of grey and white, throwing itself against the granite base with a violence that made the floorboards tremble. Elias stood by the lantern room, his hands wrapped around a mug of coffee that had gone cold hours ago. He wasn\u2019t drinking it. He was holding it for warmth."
+  },
+  {
+   "name": "code",
+   "tok_s": 111.1,
+   "hit_rate": 1.0,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv(file_path: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys being the header values\n    and values being the corresponding cell values.\n    \n    Args:\n        file_path: Path to the CSV file.\n        \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result = []\n    with open(file_path, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_basic():\n    \"\"\"Test basic CSV parsing with simple data.\"\"\"\n    import tempfile\n    import os\n    \n    # Create a temporary CSV"
+  }
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split auto)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: layer split auto: K=19 - the caches hold 12288 of 12288 profiled pairs (fullest device 99%)",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3613 slots",
+  "strata generate: expert cache 4864 slots, 8.76 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 19-47, expert cache 7424 slots (14.66 GiB), 7424 of its 7424 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-18",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-18 (CUDA0), 19-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 100.0% (5760 hits / 5760 lookups)",
+  "strata serve: decode expert cache hit rate: 100.0% (4320 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 100.0% (4320 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 100.0% (4320 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 100.0% (4320 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 100.0% (26400 hits / 26400 lookups)",
+  "strata serve: decode expert cache hit rate: 100.0% (4320 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 100.0% (130080 hits / 130080 lookups)",
+  "strata serve: decode expert cache hit rate: 100.0% (110400 hits / 110400 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-x2-auto.json

@@ -0,0 +1,82 @@
+{
+ "label": "x2-auto",
+ "extra": [
+  "--layer-split",
+  "auto"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 2726,
+   "prompt_ms": 2711.7,
+   "pp_tok_s": 1005.3,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 8379,
+   "prompt_ms": 5010.3,
+   "pp_tok_s": 1672.4,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 16066,
+   "prompt_ms": 8994.8,
+   "pp_tok_s": 1786.1,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 15497.2,
+   "pp_tok_s": 1790.2,
+   "found": true,
+   "answer": "willow-474<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 4050.3,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 249.7,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 83.0,
+   "hit_rate": 0.9920220447608565,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias had lived in the tower for thirty years, and he knew the difference between a gale and a hurricane by the way the glass rattled in its frame. Tonight, it was the latter. The sea was a churning cauldron of black ink and white foam, throwing itself against the granite base with the force of a battering ram.\n\nInside the lantern room, the air was thick with the smell of kerosene and old brass. Elias wiped his hands on his woolen trousers, leaving faint, oily smears on the fabric. He checked the clock. 11:42 PM. The rotation was due.\n\nHe reached for the crank handle, a heavy iron wheel worn smooth by decades of his own palms. With a grunt that matched the strain of his shoulders, he began to turn it. The massive Fresnel lens, a cathedral of glass and crystal, began its slow, hypnotic sweep"
+  },
+  {
+   "name": "code",
+   "tok_s": 101.8,
+   "hit_rate": 0.991767530342175,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split auto)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: layer split auto: K=19 - the caches hold 11767 of 12288 profiled pairs (fullest device 100%)",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4606 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 19-47, expert cache 7160 slots (14.14 GiB), 7160 of its 7424 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-18",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-18 (CUDA0), 19-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 98.5% (6141 hits / 6234 lookups)",
+  "strata serve: decode expert cache hit rate: 98.9% (3797 hits / 3838 lookups)",
+  "strata serve: decode expert cache hit rate: 99.1% (3804 hits / 3838 lookups)",
+  "strata serve: decode expert cache hit rate: 99.4% (3816 hits / 3839 lookups)",
+  "strata serve: decode expert cache hit rate: 99.6% (3825 hits / 3839 lookups)",
+  "strata serve: decode expert cache hit rate: 99.5% (27693 hits / 27832 lookups)",
+  "strata serve: decode expert cache hit rate: 99.4% (3815 hits / 3839 lookups)",
+  "strata serve: decode expert cache hit rate: 99.2% (127081 hits / 128103 lookups)",
+  "strata serve: decode expert cache hit rate: 99.2% (119386 hits / 120377 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-x2-k16.json

@@ -0,0 +1,81 @@
+{
+ "label": "x2-k16",
+ "extra": [
+  "--layer-split",
+  "16"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 2726,
+   "prompt_ms": 2987.1,
+   "pp_tok_s": 912.6,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 8379,
+   "prompt_ms": 5908.6,
+   "pp_tok_s": 1418.1,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 16066,
+   "prompt_ms": 10655.8,
+   "pp_tok_s": 1507.7,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 18484.6,
+   "pp_tok_s": 1500.9,
+   "found": true,
+   "answer": "willow-474<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 4790.8,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 243.3,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 71.4,
+   "hit_rate": 0.9814261642156863,
+   "text": "The wind didn\u2019t just blow; it screamed. It clawed at the granite base of the tower, trying to pry the lighthouse from its bedrock. Inside, however, the air was still, smelling faintly of kerosene and old paper.\n\nElias had been the keeper of the Point for thirty-two years. He knew every creak in the floorboards, every draft that slipped through the window seals, and exactly how much oil the great Fresnel lens needed to turn its slow, hypnotic rotation.\n\nIt was 2:00 AM. The storm was at its peak. The sea was a chaotic black mirror, whipped into whitecaps that looked like jagged teeth.\n\nElias climbed the spiral stairs, his hand gripping the cold iron railing. His boots, heavy with salt and wear, thumped a steady rhythm against the stone. *Thump. Thump. Thump.*\n\nAt the top, the lantern room hummed with the vibration of"
+  },
+  {
+   "name": "code",
+   "tok_s": 86.4,
+   "hit_rate": 0.9798276439089693,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split 16)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4096 slots, 7.17 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 16-47, expert cache 7124 slots (14.14 GiB), 7124 of its 8192 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-15",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-15 (CUDA0), 16-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 97.7% (5626 hits / 5760 lookups)",
+  "strata serve: decode expert cache hit rate: 98.1% (4237 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 98.8% (4269 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 99.1% (4279 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 99.5% (4298 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 98.9% (28471 hits / 28800 lookups)",
+  "strata serve: decode expert cache hit rate: 99.1% (4282 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 98.1% (128135 hits / 130560 lookups)",
+  "strata serve: decode expert cache hit rate: 98.0% (117109 hits / 119520 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-x2-k22.json

@@ -0,0 +1,81 @@
+{
+ "label": "x2-k22",
+ "extra": [
+  "--layer-split",
+  "22"
+ ],
+ "gpus": "0,2",
+ "needles": [
+  {
+   "tokens": 2726,
+   "prompt_ms": 2537.7,
+   "pp_tok_s": 1074.2,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 8379,
+   "prompt_ms": 4440.0,
+   "pp_tok_s": 1887.2,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 16066,
+   "prompt_ms": 7872.5,
+   "pp_tok_s": 2040.8,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 13405.0,
+   "pp_tok_s": 2069.6,
+   "found": true,
+   "answer": "willow-474<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 3610.4,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 236.0,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 78.1,
+   "hit_rate": 0.9851122208995093,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was seventy-three years old, and he had been the keeper of the Blackwood Point Light for forty of those years. The other thirty-three had been spent in a hospital bed, waiting for the end.\n\nEvery night, Elias climbed the one hundred and twelve steps of the spiral staircase. His knees, worn smooth by the friction of his own boots over decades, protested with a sharp ache. He counted them. *One, two, three\u2026* The rhythm was his heartbeat. If he lost the count, he lost the light.\n\nAt the top, the lantern room hummed with the low, electric thrum of the Fresnel lens. It was a cathedral of glass and brass, a machine that had never failed him. Not once. Not in four decades.\n\nHe polished the brass railing with a rag that smelled of lemon and"
+  },
+  {
+   "name": "code",
+   "tok_s": 104.3,
+   "hit_rate": 0.9842543225870746,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split 22)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 18.13 GiB free",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4640 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 22-47, expert cache 6656 slots (13.34 GiB), 6656 of its 6656 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-21",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-21 (CUDA0), 22-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 97.7% (5609 hits / 5739 lookups)",
+  "strata serve: decode expert cache hit rate: 98.5% (4241 hits / 4306 lookups)",
+  "strata serve: decode expert cache hit rate: 98.9% (4262 hits / 4309 lookups)",
+  "strata serve: decode expert cache hit rate: 99.4% (4290 hits / 4315 lookups)",
+  "strata serve: decode expert cache hit rate: 99.6% (4303 hits / 4319 lookups)",
+  "strata serve: decode expert cache hit rate: 98.8% (28379 hits / 28721 lookups)",
+  "strata serve: decode expert cache hit rate: 99.3% (4285 hits / 4314 lookups)",
+  "strata serve: decode expert cache hit rate: 98.5% (135713 hits / 137764 lookups)",
+  "strata serve: decode expert cache hit rate: 98.4% (118518 hits / 120414 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-x2r-auto.json

@@ -0,0 +1,82 @@
+{
+ "label": "x2r-auto",
+ "extra": [
+  "--layer-split",
+  "auto"
+ ],
+ "gpus": "2,0",
+ "needles": [
+  {
+   "tokens": 2726,
+   "prompt_ms": 2804.2,
+   "pp_tok_s": 972.1,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 8379,
+   "prompt_ms": 5641.7,
+   "pp_tok_s": 1485.2,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 16066,
+   "prompt_ms": 10643.4,
+   "pp_tok_s": 1509.5,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 18363.5,
+   "pp_tok_s": 1510.8,
+   "found": true,
+   "answer": "willow-474<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 4517.2,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 245.6,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 78.9,
+   "hit_rate": 0.9913117283950618,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his woolen collar up to his chin, the fabric stiff with salt and ice. Outside the thick glass of the lantern room, the Atlantic was a churning cauldron of black and white, the waves crashing against the granite base with a violence that shook the very foundations of the tower. It was November, 1924, and the storm had been raging for three days.\n\nElias was not a man who feared the dark. He was a man who feared the silence.\n\nHe checked the clock on the wall. 11:42 PM. The rotation of the great Fresnel lens was steady, a rhythmic *whirrr-click, whirrr-click* that seemed to sync with his own heartbeat. He had been doing this for forty years. His hands were mapped with scars from rope burns and broken glass, his eyes permanently squinted against the glare of the bulb."
+  },
+  {
+   "name": "code",
+   "tok_s": 93.4,
+   "hit_rate": 0.990286044973545,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split auto)",
+  "strata generate: layer split: CUDA1 PCIe probe 57.7 GB/s -> pcie_frac 0.55",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 9.98 GiB free",
+  "strata generate: layer split auto: K=36 - the caches hold 11775 of 12288 profiled pairs (fullest device 100%)",
+  "strata generate: expert cache auto: 18.62 GiB free, 700 MiB reserved -> 6634 slots",
+  "strata generate: expert cache 8828 slots, 16.45 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 36-47, expert cache 2945 slots (5.99 GiB), 2945 of its 3072 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-35",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-35 (CUDA0), 36-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 98.6% (5678 hits / 5760 lookups)",
+  "strata serve: decode expert cache hit rate: 99.1% (4283 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 99.2% (4285 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 99.6% (4303 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 99.7% (4306 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 99.5% (27700 hits / 27840 lookups)",
+  "strata serve: decode expert cache hit rate: 99.4% (4295 hits / 4320 lookups)",
+  "strata serve: decode expert cache hit rate: 99.1% (128474 hits / 129600 lookups)",
+  "strata serve: decode expert cache hit rate: 99.0% (119785 hits / 120960 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-x2t-auto.json

@@ -0,0 +1,99 @@
+{
+ "label": "x2t-auto",
+ "extra": [
+  "--layer-split",
+  "auto"
+ ],
+ "gpus": "0,1",
+ "needles": [
+  {
+   "tokens": 2726,
+   "prompt_ms": 4201.1,
+   "pp_tok_s": 648.9,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 8379,
+   "prompt_ms": 9655.6,
+   "pp_tok_s": 867.8,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 16066,
+   "prompt_ms": 17573.8,
+   "pp_tok_s": 914.2,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 30615.6,
+   "pp_tok_s": 906.2,
+   "found": true,
+   "answer": "willow-474<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 7864.7,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 366.9,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 45.5,
+   "reps": [
+    45.5
+   ],
+   "hit_rate": 0.9703345151405784,
+   "text": "The wind didn\u2019t just blow; it screamed.\n\nElias pulled his wool coat tighter around his shoulders, the fabric stiff with salt and age. He was seventy years old, but he moved with the grace of a man who had spent decades on a narrow spiral staircase. His hands, rough as sandpaper, knew every bolt, every lever, and every creak in the iron bones of the tower.\n\nFor forty years, Elias had been the guardian of the Point. Not just of the light, but of the silence between the waves.\n\nIt was November. The sky was a bruised purple, heavy with rain that hadn\u2019t yet fallen. The sea was a churning cauldron of grey and white, throwing itself against the granite base with a violence that shook the floorboards beneath Elias\u2019s boots.\n\nHe climbed the stairs. One hundred and twelve steps. He counted them by feel, his left hand trailing along the cold iron railing. At the top, the air was"
+  },
+  {
+   "name": "code",
+   "tok_s": 58.0,
+   "reps": [
+    58.0
+   ],
+   "hit_rate": 0.9700542735219787,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv_to_dicts(filepath: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys from the header and values from the row.\n    \n    Args:\n        filepath: Path to the CSV file.\n    \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result: List[Dict[str, Any]] = []\n    with open(filepath, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_to_dicts():\n    import tempfile\n    import os\n    \n    # Create a temporary CSV file\n    with tempfile.NamedTemporaryFile"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 0: 168 windows; per window: wait for the GPU 9.683 ms, pool + plan 0.509 ms, host staging 0.714 ms, commit 0.320 ms",
+  "strata serve: stage 1: 168 windows; per window: wait for the GPU 25.253 ms, pool + plan 0.881 ms, host staging 0.000 ms, commit 0.559 ms",
+  "strata serve: stage 0: 242 windows; per window: wait for the GPU 9.860 ms, pool + plan 0.550 ms, host staging 0.750 ms, commit 0.328 ms",
+  "strata serve: stage 1: 242 windows; per window: wait for the GPU 26.029 ms, pool + plan 0.980 ms, host staging 0.000 ms, commit 0.571 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split auto)",
+  "strata generate: layer split: CUDA1 PCIe probe 3.2 GB/s -> pcie_frac 0.00",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 16.18 GiB free",
+  "strata generate: layer split auto: K=20 - the caches hold 10761 of 12288 profiled pairs (fullest device 100%)",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4603 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 20-47, expert cache 6157 slots (12.19 GiB), 6157 of its 7168 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-19",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-19 (CUDA0), 20-47 (CUDA1), one hand-off per window",
+  "strata generate: layer split across 2 GPUs: CUDA0, then CUDA1 (split auto)",
+  "strata generate: layer split: CUDA1 PCIe probe 3.2 GB/s -> pcie_frac 0.00",
+  "strata generate: layer split: CUDA1 holds its weights, session and head; 16.18 GiB free",
+  "strata generate: layer split auto: K=20 - the caches hold 10761 of 12288 profiled pairs (fullest device 100%)",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3345 slots",
+  "strata generate: expert cache 4603 slots, 8.29 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 20-47, expert cache 6157 slots (12.19 GiB), 6157 of its 7168 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-19",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-19 (CUDA0), 20-47 (CUDA1), one hand-off per window",
+  "strata serve: decode expert cache hit rate: 96.6% (5559 hits / 5754 lookups)",
+  "strata serve: decode expert cache hit rate: 97.5% (4205 hits / 4315 lookups)",
+  "strata serve: decode expert cache hit rate: 97.8% (4217 hits / 4312 lookups)",
+  "strata serve: decode expert cache hit rate: 98.6% (4257 hits / 4318 lookups)"
+ ]
+}

bench/results/2026-09-29-layer-split/data/pp-x3-auto.json

@@ -0,0 +1,99 @@
+{
+ "label": "x3-auto",
+ "extra": [
+  "--layer-split",
+  "auto"
+ ],
+ "gpus": "0,2,1",
+ "needles": [
+  {
+   "tokens": 2726,
+   "prompt_ms": 3657.4,
+   "pp_tok_s": 745.3,
+   "found": true,
+   "answer": "meadow-254<|im_end|>"
+  },
+  {
+   "tokens": 8379,
+   "prompt_ms": 7594.1,
+   "pp_tok_s": 1103.4,
+   "found": true,
+   "answer": "juniper-766<|im_end|>"
+  },
+  {
+   "tokens": 16066,
+   "prompt_ms": 13996.4,
+   "pp_tok_s": 1147.9,
+   "found": true,
+   "answer": "falcon-174<|im_end|>"
+  },
+  {
+   "tokens": 27743,
+   "prompt_ms": 24255.2,
+   "pp_tok_s": 1143.8,
+   "found": true,
+   "answer": "willow-474<|im_end|>"
+  }
+ ],
+ "chat": {
+  "turn1_tokens": 6584,
+  "turn1_ms": 6104.9,
+  "turn2_tokens": 6660,
+  "reused": 6632,
+  "turn2_ms": 393.4,
+  "answer": "lantern-417<|im_end|>",
+  "ok": true
+ },
+ "decode": [
+  {
+   "name": "story",
+   "tok_s": 51.9,
+   "reps": [
+    51.9
+   ],
+   "hit_rate": 1.0,
+   "text": "The wind didn\u2019t just blow; it screamed. It tore through the iron gallery of the Blackwood Point Lighthouse, rattling the glass panes like a prisoner shaking its bars.\n\nElias Thorne was seventy years old, and he had been the keeper for forty of them. He knew every creak in the spiral staircase, every draft that slipped under the door, and every shift in the weather that the barometer couldn\u2019t quite predict. He knew that when the fog rolled in thick enough to swallow the horizon, the world ceased to exist. There was only the light, the stone, and him.\n\nIt was November, the month of storms. The sea was a churning cauldron of grey and white, throwing itself against the granite base with a violence that made the floorboards tremble. Elias stood by the lantern room, his hands wrapped around a mug of coffee that had gone cold hours ago. He wasn\u2019t drinking it. He was holding it for warmth."
+  },
+  {
+   "name": "code",
+   "tok_s": 69.7,
+   "reps": [
+    69.7
+   ],
+   "hit_rate": 1.0,
+   "text": "```python\nimport csv\nfrom typing import List, Dict, Any\n\n\ndef parse_csv(file_path: str) -> List[Dict[str, Any]]:\n    \"\"\"\n    Parse a CSV file into a list of dictionaries.\n    \n    Each dictionary represents a row, with keys being the header values\n    and values being the corresponding cell values.\n    \n    Args:\n        file_path: Path to the CSV file.\n        \n    Returns:\n        A list of dictionaries, one per row (excluding the header).\n    \"\"\"\n    result = []\n    with open(file_path, 'r', newline='', encoding='utf-8') as f:\n        reader = csv.DictReader(f)\n        for row in reader:\n            result.append(dict(row))\n    return result\n\n\n# Tests\ndef test_parse_csv_basic():\n    \"\"\"Test basic CSV parsing with simple data.\"\"\"\n    import tempfile\n    import os\n    \n    # Create a temporary CSV"
+  }
+ ],
+ "stage_timing": [
+  "strata serve: stage 2: 165 windows; per window: wait for the GPU 13.452 ms, pool + plan 0.162 ms, host staging 0.000 ms, commit 0.328 ms",
+  "strata serve: stage 0: 234 windows; per window: wait for the GPU 6.235 ms, pool + plan 0.145 ms, host staging 0.704 ms, commit 0.214 ms",
+  "strata serve: stage 1: 234 windows; per window: wait for the GPU 15.592 ms, pool + plan 0.246 ms, host staging 0.000 ms, commit 0.657 ms",
+  "strata serve: stage 2: 234 windows; per window: wait for the GPU 13.916 ms, pool + plan 0.177 ms, host staging 0.000 ms, commit 0.335 ms"
+ ],
+ "engine_lines": [
+  "strata generate: layer split across 3 GPUs: CUDA0, then CUDA1,2 (split auto)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session; 18.62 GiB free",
+  "strata generate: layer split: CUDA2 PCIe probe 3.2 GB/s -> pcie_frac 0.00",
+  "strata generate: layer split: CUDA2 holds its weights, session and head; 16.18 GiB free",
+  "strata generate: layer split auto: K=12,33 - the caches hold 12288 of 12288 profiled pairs (fullest device 66%)",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3072 slots",
+  "strata generate: expert cache 3072 slots, 5.41 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 12-32, expert cache 5376 slots (10.22 GiB), 5376 of its 5376 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA2 runs layers 33-47, expert cache 3840 slots (7.79 GiB), 3840 of its 3840 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-11",
+  "strata serve: the prompt path allocates its own buffers (too few cache slots to borrow)",
+  "strata serve: layer split: layers 0-11 (CUDA0), 12-32 (CUDA1), 33-47 (CUDA2), one hand-off per window",
+  "strata generate: layer split across 3 GPUs: CUDA0, then CUDA1,2 (split auto)",
+  "strata generate: layer split: CUDA1 PCIe probe 6.4 GB/s -> pcie_frac 0.14",
+  "strata generate: layer split: CUDA1 holds its weights, session; 18.62 GiB free",
+  "strata generate: layer split: CUDA2 PCIe probe 3.2 GB/s -> pcie_frac 0.00",
+  "strata generate: layer split: CUDA2 holds its weights, session and head; 16.18 GiB free",
+  "strata generate: layer split auto: K=12,33 - the caches hold 12288 of 12288 profiled pairs (fullest device 66%)",
+  "strata generate: expert cache auto: 10.46 GiB free, 700 MiB reserved -> 3072 slots",
+  "strata generate: expert cache 3072 slots, 5.41 GiB of VRAM; policy is",
+  "strata generate: layer split: CUDA1 runs layers 12-32, expert cache 5376 slots (10.22 GiB), 5376 of its 5376 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA2 runs layers 33-47, expert cache 3840 slots (7.79 GiB), 3840 of its 3840 profiled pairs; slot 0 verified",
+  "strata generate: layer split: CUDA0 runs layers 0-11"
+ ]
+}

bench/results/2026-09-29-speed-0122/README.md

@@ -0,0 +1,32 @@
+# Speed with engine 0.1.22
+
+RTX 5070 12 GB (PCIe 5.0 x16), Ryzen 5 7600, 64 GB DDR5-5200, Windows 10. The ready-made 0.1.22 engine, one-shot runs
+with what setup writes: `--prefill auto`, the shipped expert profile, `--expert-cache auto`, 8-bit KV above 4K, KV
+streaming (`--kv-resident 32768`) from 64K, MTP (`--spec 4 --spec-min-p 0.5`), greedy, 256 generated tokens, images
+off. The same code-agent prompts as `2026-09-28-speed-0114`. Every run: [`matrix.json`](matrix.json).
+
+## Prompt (tokens/s)
+
+| Model | 1K | 4K | 32K | 64K | 128K | 262K |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: |
+| **Q2_0** | 519 | 1,226 | 1,844 | 1,836 | 1,682 | 1,304 |
+| **IQ2_XS** | 524 | 1,196 | 1,799 | 1,611 | 1,495 | 1,181 |
+| **IQ3_XXS** | 472 | 974 | 1,555 | 1,449 | 1,386 | - |
+| **IQ3_S** | 419 | 893 | 1,499 | 1,285 | 1,245 | - |
+| **Coder** | 660 | 1,522 | 1,871 | 1,938 | 1,779 | - |
+
+## Output (tokens/s)
+
+| Model | 1K | 4K | 32K | 64K | 128K | 262K |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: |
+| **Q2_0** | 84.7 | 88.2 | 77.4 | 71.0 | 63.1 | 61.1 |
+| **IQ2_XS** | 77.2 | 74.6 | 67.5 | 63.5 | 59.1 | 54.7 |
+| **IQ3_XXS** | 65.1 | 62.3 | 49.8 | 53.6 | 47.0 | - |
+| **IQ3_S** | 50.5 | 44.6 | 49.0 | 48.8 | 41.3 | - |
+| **Coder** | 56.9 | 53.6 | 54.9 | 48.5 | 42.8 | - |
+
+- IQ2_XS was measured with Swift 1.5's IQ2_XS (the same speed as the original's); its 262K run had images on.
+- IQ3_XXS and IQ3_S at 262K do not fit 64 GB of RAM; the Coder at 262K was not run again (0.1.14: 1,034 tokens/s).
+- Against 0.1.14: prompts 1.2-1.5x faster at 32K-128K (the 0.1.22 prompt path: tensor-core attention, bit-identical
+  batched kernels, faster PLE reads). Output speed is the engine's same decode; one run per cell moves a few percent
+  with the text (speculative drafts), so compare it across many runs, not one.

bench/results/2026-09-29-speed-0122/matrix.json

@@ -0,0 +1,380 @@
+[
+ {
+  "model": "Q2_0",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 40.0,
+  "decode_tok_s": 84.71,
+  "decoded": 256,
+  "prefill_tok_s": 519.48,
+  "ttft_s": 3.62,
+  "spec_accept": 0.786,
+  "tokens_per_round": 2.77,
+  "vram_slots": 4176
+ },
+ {
+  "model": "Q2_0",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 41.6,
+  "decode_tok_s": 88.23,
+  "decoded": 256,
+  "prefill_tok_s": 1226.49,
+  "ttft_s": 4.43,
+  "spec_accept": 0.839,
+  "tokens_per_round": 3.24,
+  "vram_slots": 4123
+ },
+ {
+  "model": "Q2_0",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 56.6,
+  "decode_tok_s": 77.37,
+  "decoded": 256,
+  "prefill_tok_s": 1843.83,
+  "ttft_s": 18.63,
+  "spec_accept": 0.808,
+  "tokens_per_round": 3.04,
+  "vram_slots": 3833
+ },
+ {
+  "model": "Q2_0",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 72.9,
+  "decode_tok_s": 71.0,
+  "decoded": 256,
+  "prefill_tok_s": 1835.64,
+  "ttft_s": 36.48,
+  "spec_accept": 0.685,
+  "tokens_per_round": 2.77,
+  "vram_slots": 3780
+ },
+ {
+  "model": "Q2_0",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 118.1,
+  "decode_tok_s": 63.08,
+  "decoded": 256,
+  "prefill_tok_s": 1681.64,
+  "ttft_s": 77.92,
+  "spec_accept": 0.7,
+  "tokens_per_round": 2.56,
+  "vram_slots": 3697
+ },
+ {
+  "model": "Q2_0",
+  "tier": "262k",
+  "prompt_tokens": 259943,
+  "exit": 0,
+  "wall_s": 242.0,
+  "decode_tok_s": 61.12,
+  "decoded": 256,
+  "prefill_tok_s": 1303.83,
+  "ttft_s": 200.94,
+  "spec_accept": 0.627,
+  "tokens_per_round": 2.5,
+  "vram_slots": 3525
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "262k",
+  "prompt_tokens": 259943,
+  "exit": 0,
+  "wall_s": 263.6,
+  "decode_tok_s": 54.68,
+  "decoded": 256,
+  "prefill_tok_s": 1180.58,
+  "ttft_s": 221.91,
+  "spec_accept": 0.632,
+  "tokens_per_round": 2.49,
+  "vram_slots": 3727
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 56.0,
+  "decode_tok_s": 50.5,
+  "decoded": 256,
+  "prefill_tok_s": 419.48,
+  "ttft_s": 3.98,
+  "spec_accept": 0.811,
+  "tokens_per_round": 3.17,
+  "vram_slots": 2617
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 60.0,
+  "decode_tok_s": 44.58,
+  "decoded": 256,
+  "prefill_tok_s": 892.84,
+  "ttft_s": 5.53,
+  "spec_accept": 0.672,
+  "tokens_per_round": 2.69,
+  "vram_slots": 2580
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 75.6,
+  "decode_tok_s": 48.96,
+  "decoded": 256,
+  "prefill_tok_s": 1499.09,
+  "ttft_s": 22.65,
+  "spec_accept": 0.79,
+  "tokens_per_round": 2.9,
+  "vram_slots": 2383
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 102.1,
+  "decode_tok_s": 48.8,
+  "decoded": 256,
+  "prefill_tok_s": 1284.81,
+  "ttft_s": 51.58,
+  "spec_accept": 0.797,
+  "tokens_per_round": 2.94,
+  "vram_slots": 2281
+ },
+ {
+  "model": "IQ3_S",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 159.2,
+  "decode_tok_s": 41.27,
+  "decoded": 256,
+  "prefill_tok_s": 1244.55,
+  "ttft_s": 104.81,
+  "spec_accept": 0.694,
+  "tokens_per_round": 2.73,
+  "vram_slots": 2292
+ },
+ {
+  "model": "Coder",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 34.5,
+  "decode_tok_s": 56.94,
+  "decoded": 256,
+  "prefill_tok_s": 660.44,
+  "ttft_s": 2.42,
+  "spec_accept": 0.74,
+  "tokens_per_round": 2.91,
+  "vram_slots": 2650
+ },
+ {
+  "model": "Coder",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 18.8,
+  "decode_tok_s": 53.65,
+  "decoded": 256,
+  "prefill_tok_s": 1521.84,
+  "ttft_s": 3.21,
+  "spec_accept": 0.736,
+  "tokens_per_round": 2.91,
+  "vram_slots": 2615
+ },
+ {
+  "model": "Coder",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 33.1,
+  "decode_tok_s": 54.85,
+  "decoded": 256,
+  "prefill_tok_s": 1870.76,
+  "ttft_s": 17.7,
+  "spec_accept": 0.806,
+  "tokens_per_round": 3.11,
+  "vram_slots": 2418
+ },
+ {
+  "model": "Coder",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 50.2,
+  "decode_tok_s": 48.46,
+  "decoded": 256,
+  "prefill_tok_s": 1937.69,
+  "ttft_s": 33.99,
+  "spec_accept": 0.653,
+  "tokens_per_round": 2.56,
+  "vram_slots": 2382
+ },
+ {
+  "model": "Coder",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 90.3,
+  "decode_tok_s": 42.75,
+  "decoded": 256,
+  "prefill_tok_s": 1778.66,
+  "ttft_s": 73.12,
+  "spec_accept": 0.588,
+  "tokens_per_round": 2.27,
+  "vram_slots": 2326
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 43.6,
+  "decode_tok_s": 77.16,
+  "decoded": 256,
+  "prefill_tok_s": 523.77,
+  "ttft_s": 3.46,
+  "spec_accept": 0.811,
+  "tokens_per_round": 3.02,
+  "vram_slots": 4361
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 43.7,
+  "decode_tok_s": 74.58,
+  "decoded": 256,
+  "prefill_tok_s": 1196.03,
+  "ttft_s": 4.47,
+  "spec_accept": 0.824,
+  "tokens_per_round": 3.36,
+  "vram_slots": 4309
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 59.6,
+  "decode_tok_s": 67.46,
+  "decoded": 256,
+  "prefill_tok_s": 1798.65,
+  "ttft_s": 19.06,
+  "spec_accept": 0.789,
+  "tokens_per_round": 3.02,
+  "vram_slots": 4030
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 81.4,
+  "decode_tok_s": 63.52,
+  "decoded": 256,
+  "prefill_tok_s": 1610.73,
+  "ttft_s": 41.37,
+  "spec_accept": 0.705,
+  "tokens_per_round": 2.6,
+  "vram_slots": 3978
+ },
+ {
+  "model": "IQ2_XS",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 126.0,
+  "decode_tok_s": 59.11,
+  "decoded": 256,
+  "prefill_tok_s": 1494.53,
+  "ttft_s": 87.51,
+  "spec_accept": 0.704,
+  "tokens_per_round": 2.71,
+  "vram_slots": 3897
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "1k",
+  "prompt_tokens": 1017,
+  "exit": 0,
+  "wall_s": 48.6,
+  "decode_tok_s": 65.12,
+  "decoded": 256,
+  "prefill_tok_s": 471.53,
+  "ttft_s": 3.65,
+  "spec_accept": 0.876,
+  "tokens_per_round": 4.08,
+  "vram_slots": 3281
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "4k",
+  "prompt_tokens": 3562,
+  "exit": 0,
+  "wall_s": 48.5,
+  "decode_tok_s": 62.33,
+  "decoded": 256,
+  "prefill_tok_s": 974.3,
+  "ttft_s": 5.17,
+  "spec_accept": 0.826,
+  "tokens_per_round": 3.15,
+  "vram_slots": 3243
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "32k",
+  "prompt_tokens": 31566,
+  "exit": 0,
+  "wall_s": 64.9,
+  "decode_tok_s": 49.83,
+  "decoded": 256,
+  "prefill_tok_s": 1554.53,
+  "ttft_s": 21.84,
+  "spec_accept": 0.766,
+  "tokens_per_round": 2.91,
+  "vram_slots": 3009
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "64k",
+  "prompt_tokens": 64162,
+  "exit": 0,
+  "wall_s": 91.7,
+  "decode_tok_s": 53.58,
+  "decoded": 256,
+  "prefill_tok_s": 1449.37,
+  "ttft_s": 45.84,
+  "spec_accept": 0.748,
+  "tokens_per_round": 2.8,
+  "vram_slots": 2965
+ },
+ {
+  "model": "IQ3_XXS",
+  "tier": "128k",
+  "prompt_tokens": 128478,
+  "exit": 0,
+  "wall_s": 140.9,
+  "decode_tok_s": 46.97,
+  "decoded": 256,
+  "prefill_tok_s": 1386.04,
+  "ttft_s": 94.33,
+  "spec_accept": 0.607,
+  "tokens_per_round": 2.47,
+  "vram_slots": 2901
+ }
+]

docs/DETAILS.md

@@ -12,7 +12,7 @@ New here? Start with the [README](../README.md) - it has everything you need to
 
 ## Speed (measured)
 
-RTX 5070 **12 GB**, Ryzen 5 7600 (6 cores), 64 GB DDR5-5200, Windows, engine 0.1.14 with the settings setup writes
+RTX 5070 **12 GB**, Ryzen 5 7600 (6 cores), 64 GB DDR5-5200, Windows, engine 0.1.22 (prompts) / 0.1.14 (output) with the settings setup writes
 (`--prefill auto`, 8-bit KV above 4K, KV streaming from 64K). One code-agent prompt per length, 256 generated tokens,
 MTP speculative decoding on. "262K" is the model's full context window (a 259,943-token prompt). The IQ2_XS row was
 measured with Swift 1.5's IQ2_XS, which runs at the original's speed.
@@ -21,11 +21,14 @@ measured with Swift 1.5's IQ2_XS, which runs at the original's speed.
 
 | Model | 1K | 4K | 32K | 64K | 128K | 262K |
 | --- | ---: | ---: | ---: | ---: | ---: | ---: |
-| **Q2_0** | 494 | 1,007 | 1,308 | 1,294 | 1,208 | 967 |
-| **IQ2_XS** | 461 | 811 | 1,238 | 1,136 | 1,071 | 886 |
-| **IQ3_XXS** | 415 | 770 | 1,108 | 1,065 | 1,015 | - |
-| **IQ3_S** | 396 | 737 | 1,070 | 1,070 | 931 | - |
-| **Coder** | 599 | 1,152 | 1,298 | 1,350 | 1,266 | 1,034 |
+| **Q2_0** | 519 | 1,226 | 1,844 | 1,836 | 1,682 | 1,304 |
+| **IQ2_XS** | 524 | 1,196 | 1,799 | 1,611 | 1,495 | 1,181* |
+| **IQ3_XXS** | 472 | 974 | 1,555 | 1,449 | 1,386 | - |
+| **IQ3_S** | 419 | 893 | 1,499 | 1,285 | 1,245 | - |
+| **Coder** | 660 | 1,522 | 1,871 | 1,938 | 1,779 | 1,034** |
+
+Engine 0.1.22 (the prompt path of 0.1.23 is the same); `bench/results/2026-09-29-speed-0122`. \* measured with
+images on (the image encoder's VRAM reserve leaves fewer experts cached). \*\* not measured again: 0.1.14.
 
 ### Output (tokens/s)
 
@@ -216,6 +219,28 @@ install; `START-HERE.bat --calibrate` (Linux: `./setup.sh --calibrate`) does it
 speed with each setting and keeps one only when it is more than 3% faster. The result is remembered per PC and model
 (in the settings file next to the data folder's record), so updates keep it.
 
+### Running it at startup (Task Scheduler)
+
+To have the model up at logon, people start the serve from **Task Scheduler** (or a service). Beware: Windows
+throttles such contexts, and the model's ~40 GB expert load then crawls at **~0.05 GiB/s (13-14 minutes)**
+instead of **~1.4-1.5 GiB/s (~35 seconds)** - a 24x slower start. Measured on an RTX 5070 Ti + Ryzen 7 9800X3D
++ NVMe, same binary, same args, same cache state:
+
+| How the serve starts | Expert load |
+| --- | ---: |
+| Double-click / terminal / SSH | 1.42-1.52 GiB/s (~35 s) |
+| Task Scheduler with its defaults | 0.05 GiB/s (821-841 s) |
+| Task Scheduler with the two settings below | 1.42 GiB/s (35 s) |
+
+In the task's properties set both of these (the defaults are the opposite):
+
+- **Priority level: Normal** (Options tab; the default is Below normal), and
+- **Run with highest privileges** (General tab; without it the task runs with a limited user token - which
+  also strips `SeLockMemoryPrivilege`, the privilege Windows large pages need).
+
+(Both were changed at once, so the isolated effect of each is not measured.) If the model still starts
+slowly, the engine prints a hint under its `loaded ... GiB at ...` line naming this cause.
+
 ### Chat in the terminal (optional)
 
 ```
@@ -254,11 +279,14 @@ The server listens on `http://127.0.0.1:8080` (change with `--port` in setup, or
 | --- | --- |
 | OpenAI Chat Completions (stream and non-stream, tools) | `POST /v1/chat/completions` |
 | Anthropic Messages (stream and non-stream, tools) | `POST /v1/messages` |
-| Model list / health | `GET /v1/models`, `GET /health` |
-| What the model is doing right now | `GET /status` |
+| Model list / health | `GET /v1/models`, `GET /models`, `GET /health` |
+| Model properties | `GET /props` (also accepts `?model=<loaded-model-id>`) |
+| What the model is doing right now | `GET /status`, `GET /slots` (single slot, busy or idle) |
 | Everything the Monitor tab shows (engine, live state, last requests, hardware) | `GET /metrics` |
 | The MCP servers, their state and tools ([below](#tools-from-mcp-servers)) | `GET /mcp` |
 
+`/models` and `/v1/models` list only the loaded model, with its context limit and input modalities. `/props` exposes the original chat template, context limit, configured generation defaults (shared settings take precedence), model path and engine version when available. Context means the full engine context, not the resident KV window. `n_predict: -1` means no fixed output cap. Unconfigured sampling fields are omitted. `autoload` has no effect; an unknown `model` returns 404. These metadata endpoints and `/slots` require the API key when one is configured. They do not load, unload or restart models.
+
 ```bash
 curl http://127.0.0.1:8080/v1/chat/completions -H "Content-Type: application/json" -d '{
   "model": "strata", "messages": [{"role": "user", "content": "Write a haiku about GPUs."}], "max_tokens": 512 }'

docs/MULTI_GPU.md

@@ -0,0 +1,122 @@
+# Strata on two or three GPUs (layer split)
+
+One model can run across several NVIDIA cards in one PC. The layers are split into contiguous ranges, one per GPU:
+the first card runs layers 0 to K-1, the next card runs K onward, and so on; the last card also runs the output head
+and the draft (MTP) layer. Each card keeps an expert cache for **its own layers only**, so two cards hold about twice
+the experts one card holds - for the Coder model on a 16 GB + 24 GB pair, nearly all of them, which is where the
+speed comes from (decode then barely touches the CPU pool).
+
+This is pipeline (layer) parallelism, not tensor parallelism: a token crosses from one card to the next once per
+verify window (a few hundred KB through pinned RAM), not twice per layer. No NVLink or peer-to-peer access is
+needed; cards on x4 or x1 slots work, and the PCIe share of each card is probed on its own link.
+
+## Using it
+
+**Nothing to type.** `START-HERE.bat` (Linux: `./setup.sh`) lists your NVIDIA cards and says for each one whether
+Strata can use it:
+
+```
+  Your NVIDIA GPUs:
+    GPU 0: NVIDIA GeForce RTX 5080, 16 GB VRAM - can be used
+    GPU 1: NVIDIA GeForce RTX 2080 Ti, 22 GB VRAM - not supported - older than the RTX 30 series (compute capability 7.5; Strata needs 8.0 or newer)
+    GPU 2: NVIDIA GeForce RTX 3090, 24 GB VRAM - can be used
+  ...
+  1) GPU 0 (NVIDIA GeForce RTX 5080, 16 GB) + GPU 2 (NVIDIA GeForce RTX 3090, 24 GB) together   (recommended)
+  2) GPU 2 (NVIDIA GeForce RTX 3090, 24 GB) only
+  3) GPU 0 (NVIDIA GeForce RTX 5080, 16 GB) only
+Which GPUs? [1]:
+```
+
+When two or more cards can share the model, the two best together are recommended (the newest generation first:
+it becomes the main card). A model installed on one card asks once, at its next start, whether to use both from
+now on; the answer is kept.
+
+**Choosing yourself** (at setup or at any start):
+
+```
+--gpus 0,2                 these cards together, as nvidia-smi numbers them; the first is the main one. Remembered.
+--gpus all                 every card that can share the model
+--gpu 0                    one card (at a start: for that start only)
+--layer-split auto         (default) or the first layer of each later card, e.g. 18 or 16,32
+```
+
+**Not supported** (setup says so and names the cards that can be used instead):
+- a card older than the RTX 30 series (compute capability below 8.0: RTX 20, GTX 16/10);
+- a card with less than 8 GB of VRAM, together with others (each card holds a copy of the dense weights and its
+  own prompt buffers);
+- AMD and Intel GPUs, and a mix of NVIDIA with them.
+
+Or edit an existing config (`strata-*.json`), then restart:
+
+```json
+"gpu": [0, 2],
+"layer_split": "auto"
+```
+
+The engine flags behind it: `--layer-split K1[,K2..]|auto` and `--split-device D1[,D2..]` (the later stages'
+devices; default the next visible ones). `--layer-split K --split-device 0` runs both stages on one card sharing
+everything - the bit-exact check of the hand-off, not a speed mode.
+
+**auto** tries every placement (all of them for two or three cards; proportional to the free VRAM beyond that) and
+keeps the one whose caches would hold the most of the expert profile, hottest pairs weighted most; ties go to the
+placement that leaves the fullest card the most room. The startup log prints the choice:
+
+```
+strata generate: layer split auto: K=19 - the caches hold 11767 of 12288 profiled pairs (fullest device 100%)
+strata serve: layer split: layers 0-18 (CUDA0), 19-47 (CUDA1), one hand-off per window
+```
+
+## What each card holds
+
+- **every card**: a copy of the dense weights (~3.4 GB for the Coder), its own session state (the KV cache of the full
+  context), its verify window and its prompt-path buffers, and an expert cache for its layers filled from the profile;
+- **the last card**: also the output head and the draft layer (~0.8 GB);
+- **host RAM**: the expert arena once, shared by all cards (the CPU pool computes whatever no card holds).
+
+Prompts are read in chunks that flow through the cards in turn; while a later card reads chunk c, the first card
+already reads chunk c+1. Conversation checkpoints save and restore every card's state; the adaptive expert swaps copy
+into the card that owns the layer.
+
+## Limits (for now)
+
+- **Works across cards** (bench/results/2026-09-29-layer-split-limits):
+  - images (`--vision`): each card keeps its own image-position table;
+  - control vectors and the experimental speed projection: each card holds the vector's tables, switched on and
+    off per request on all of them;
+  - KV streaming (`--kv-resident`): each card streams the KV of its own session;
+  - mid-prompt checkpoints (`--prompt-cache-every`): each card saves its part of a checkpoint when it has read that
+    chunk;
+  - the older helper-GPU caches (`--expert-cache-remote`, docs/SECOND_GPU.md): they take the visible GPUs no stage
+    runs on, and hold only experts no stage's cache holds. On the test rig, a 2080 Ti helper made decoding slower,
+    as it did without a split: its per-layer round trip costs more than the CPU pool needs for those experts.
+- `--mmap-experts` needs a canonical pack (`experts.bin`), with or without a split; a native (IQ) pack says so at
+  start.
+- The prompt path has its own buffers on every card (1.5 GB each at the default 2048-token chunk; `--prefill 1024`
+  halves that) instead of borrowing cache slots as one card does. An explicit `--expert-cache` on the first card is
+  capped to leave room for them.
+- On Windows only 8 GiB of the expert arena is pinned (more, mapped into two GPU contexts, leaves WDDM refusing
+  allocations); the PCIe share covers those layers.
+- Every card needs compute capability 8.0 (RTX 30 or newer). A Turing card (RTX 20, sm_75) builds only with the
+  experimental `-DSTRATA_EXPERIMENTAL_SM75=ON`.
+
+## Measured
+
+The Coder on an RTX 5080 + RTX 3090 (Ryzen 9 9950X3D), 32K context; details in
+`bench/results/2026-09-29-layer-split/`:
+
+| | Prompt 16K / 28K tok/s | Decode story / code tok/s |
+|---|---|---|
+| 5080 alone | 1,726-2,017 / 1,970 | 83-87 / 88-105 |
+| 5080 + 3090, best split (K=26) | 2,039 / 2,357 | 84 / 110 |
+| 5080 + 3090, auto (K=22) | 2,037 / 2,073 | 80 / 109 |
+
+- **Prompts gain the most** (+18-20%): each card reads its own layers of the chunk while the other reads the next.
+- **Decode is on par with the faster card alone**, and ahead on code. Once both caches hold nearly every routed
+  expert, the per-layer GPU time decides.
+- **Correctness:** one GPU is byte-identical to 0.1.20, and the hand-off itself is bit-exact.
+
+**Which cards and in what order:**
+- Put the fastest card first; auto gives it as many layers as its cache allows.
+- Leave out a much slower card when two already hold the model. An RTX 2080 Ti as a third card made the 5080 +
+  3090 pair slower (68 / 90 tok/s decode): every extra card costs its own round per window.
+- More cards pay off when the model's routed experts do not fit the faster ones.

docs/SECOND_GPU.md

@@ -0,0 +1,84 @@
+# CUDA1–3 expert experiment
+
+CUDA0 keeps the dense layers, KV/state, MTP and its existing expert cache. Up to
+three independent expert caches fill CUDA1, CUDA2 and CUDA3 in that order. Each
+expert has one owner: CUDA0, one secondary GPU, or the CPU pool. Each secondary
+GPU computes its rows while the CPU pool works; the results return through
+private pinned host buffers before CUDA0 continues the layer. The GPU launches
+can overlap, but each layer waits for all three.
+
+The shipped profile ranks 8,000 `(layer, expert)` pairs. CUDA0 takes its usual
+`--expert-cache auto` allocation. With only CUDA1, it takes the first remaining
+pairs. When CUDA2 or CUDA3 is enabled, the remaining profiled pairs are assigned
+round robin to the secondary GPUs. The ranked list is then extended with all
+other pairs in expert-then-layer order and distributed in the same way. This
+fills the additional VRAM and lets all secondary GPUs see frequent experts, but
+unranked experts may hardly ever be routed. The startup
+log reports slots and GiB per card; each request reports the number of expert
+entries actually computed on each secondary GPU. Remote outputs are packed
+before returning over PCIe/USB4: the request log compares MiB transferred
+with the previous full-row transfer. The input and routing metadata use pinned
+host staging. This reduces transfer traffic and launch overhead, but does not
+release the expert arena in system RAM or significantly change VRAM use.
+
+The optional `--gpu-placement layer` gives each layer to exactly one secondary
+GPU (layer number modulo the number of enabled cards). It fills that card with
+the most frequently routed experts of its layers first, then the unranked ones.
+This can reduce USB4 device switches and synchronization when several secondary
+cards serve the same layer in the default `stripe` mode. All three cards still
+keep their requested expert slots; the primary CUDA0 cache and CPU fallback
+continue to work. The log counts active layer launches on each card, so the
+tradeoff can be measured alongside tokens per second. This placement is
+experimental and may be slower if a given layer needs more than one card's
+compute capacity.
+
+The experiment requires the corresponding visible CUDA devices and a CUDA
+enabled build. Each card must keep at least 512 MiB free. Tiers must be enabled
+in order. On Windows, CUDA registration of the host expert arena is capped at
+8 GiB to leave room for the contexts and MTP on CUDA0. The rest remains
+available to the CPU pool; the PCIe expert path is available only for the
+registered layers. Without secondary GPUs, the original uncapped registration
+behavior applies.
+
+On an existing Linux installation, apply the patch to its source tree, then run:
+
+```sh
+./setup.sh --setup --gpu1-experts 5000 --gpu2-experts 5000 --gpu3-experts 5000
+```
+
+On Windows, from PowerShell in the installation folder:
+
+```powershell
+.\START-HERE.bat --setup --gpu1-experts 5000 --gpu2-experts 5000 --gpu3-experts 5000
+```
+
+To test the layer placement instead, add `--gpu-placement layer` to the setup
+command. To go back, run the same command with `--gpu-placement stripe`. After
+the new engine is built, the mode can also be changed without a rebuild: edit
+`--expert-cache-remote-placement` in `strata-iq3_xxs.json` from `layer` to
+`stripe` or vice versa and restart the server. If the argument is absent, the
+default is `stripe`.
+
+Choose the same model, size, context and image settings as your current
+installation when prompted. Setup recompiles the engine, updates its server
+configuration, and starts it. Later starts keep those settings. To tune the
+slots without recompiling, edit the values after `--expert-cache-device1`,
+`--expert-cache-device2` and `--expert-cache-device3` in the stored
+`strata-iq3_xxs.json`, then restart with `.\run-iq3_xxs.bat`.
+
+Compare identical requests at one, two and four GPUs, preferably with several
+repeats. VRAM use alone does not show useful offload: compare the per-request
+CUDA1–3 counts and tokens per second. More GPU contexts and synchronization
+may lower the speed. Prompt prefill still uses CUDA0; secondary GPUs serve
+decode, including MTP verification. No peer-to-peer access is required.
+
+Longer contexts reserve more KV/state memory on CUDA0, which reduces its
+automatic expert cache. On a 64 GB PC `setup.py` limits IQ3_XXS to 128K even
+if 262K was selected; the secondary caches still require the host expert arena.
+To try 262K on 64 GB, choose Q2_0 or IQ2_XS instead and check that CUDA0 still
+has enough free VRAM after loading. Benchmark long prompts separately from
+short decode requests.
+
+The existing warning about the CUDA0 expert-cache GPU hit path still applies:
+its outputs diverge from cache-off runs. Treat performance as experimental
+until the generated tokens have been validated.

include/strata/artifact/gguf_reader.hpp

@@ -26,6 +26,7 @@
 #include <string>
 #include <vector>
 #include <map>
+#include <set>
 #include <stdexcept>
 #include <algorithm>
 
@@ -418,9 +419,14 @@ private:
             meta_.emplace(std::move(key), read_value(c, t));
         }
         tensors_.reserve((size_t)n_tensors);
+        // GGUF has no index to arbitrate between two tensors of one name: find() is first-match, so a
+        // duplicate would silently win by position.  Refuse the file at open instead, naming both.
+        std::set<std::string> names;
         for (uint64_t i = 0; i < n_tensors; ++i) {
             TensorInfo t;
             t.name = c.str();
+            if (!names.insert(t.name).second)
+                throw std::runtime_error("GGUF: duplicate tensor name '" + t.name + "' in " + path_);
             const uint32_t nd = c.read<uint32_t>();
             if (nd == 0 || nd > 4) throw std::runtime_error("GGUF: bad n_dims for " + t.name);
             t.shape.resize(nd);

include/strata/core/expert_cache.hpp

@@ -126,6 +126,10 @@ public:
     /// against a `cudaStreamNonBlocking` one.  The first version used the async form and `verify_slot` refused
     /// the whole run with "slot 0 differs from the arena at byte 0" - which is the check doing its job.
     bool fill_slot_blocking(int32_t slot, const uint8_t* host_blob, std::string& err, int64_t bytes = 0);
+    /// perf-review D-4: `fill_slot_blocking`'s copy on the same (legacy) stream, but queued: many slots are refilled
+    /// with one `sync_queued` at the end instead of a wait per slot. Same ordering against earlier work, same bytes.
+    bool fill_slot_queued(int32_t slot, const uint8_t* host_blob, std::string& err, int64_t bytes = 0);
+    bool sync_queued(std::string& err);
 
     /// Reads `slot` back to the host and compares it to `host_blob`, byte for byte.  **THE ONLY THING THAT SAYS
     /// THE CACHE HOLDS THE EXPERT IT CLAIMS TO.**  A slot table that is right about indices and wrong about

include/strata/core/expert_source.hpp

@@ -31,6 +31,8 @@
 
 namespace strata::core {
 
+class RemoteExperts;
+
 /// Where one routed expert's bytes come from.
 ///
 /// Phase 2 has NO cache (`phase-2-correct-engine.md`: hit rate `h = 0`), so the only implementation is a
@@ -92,6 +94,8 @@ struct GpuPlanSink {
 struct ExpertDispatch {
     strata::kernels::cpu::ExpertPool* pool = nullptr;
     ExpertSource* src = nullptr;
+    RemoteExperts* remote[3] = {}; ///< optional CUDA1..3 tiers for otherwise CPU-served rows
+    int remote_count = 0;
     int64_t n_expert = strata::kernels::cpu::NE;
 
     /// Counters, for the driver to report rather than for control flow.
@@ -320,7 +324,8 @@ public:
 
     /// Allocates and loads `<pack_dir>/experts.bin`.  Prints nothing; the caller reports `note()` and the load
     /// rate, because those are the two numbers that say whether the arena is the one that was asked for.
-    bool open(const std::string& pack_dir, int64_t n_layers, int64_t n_expert, int threads, std::string& err);
+    bool open(const std::string& pack_dir, int64_t n_layers, int64_t n_expert, int threads, std::string& err,
+              uint64_t max_pinned_bytes = 0);
     /// Plan v0.3 P6: a native pack without experts.bin takes its experts from the model's shard 1.
     void set_gguf(const std::string& shard1) { gguf_ = shard1; }
     void close();
@@ -336,6 +341,12 @@ public:
     /// only true if the engine says what it got.
     const std::string& note() const { return note_; }
     double load_gib_per_second() const { return gib_per_s_; }
+    // Loader fix: the load, split.  `load_seconds()` is the wall clock of the load loop; the other two are
+    // sums over the reader threads (see LoadStats), so on their own they say how much of that wall was spent
+    // waiting for the disk and how much in memcpy + FNV-1a.
+    double load_seconds() const { return load_seconds_; }
+    double load_read_seconds() const { return load_read_s_; }
+    double load_copy_seconds() const { return load_copy_s_; }
 
 private:
     void* arena_ = nullptr;          ///< the PinnedArena, owned
@@ -347,6 +358,9 @@ private:
     int64_t reads_ = 0;
     std::string note_;
     double gib_per_s_ = 0.0;
+    double load_seconds_ = 0.0;
+    double load_read_s_ = 0.0;
+    double load_copy_s_ = 0.0;
     uint64_t pinned_bytes_ = 0;
     std::string gguf_;
 };

include/strata/core/mtp.hpp

@@ -76,6 +76,7 @@ public:
 private:
     bool record_forward(int T, int step_row0, cudaStream_t cs, std::string& err);
     bool capture_prefill(int T, std::string& err);
+    bool capture_prefill_dev(int T, std::string& err);   ///< E-4: without the mapped staging (inputs copied on device)
     bool capture_round(int T, std::string& err);
     bool capture_step(int j, std::string& err);
     cudaGraphExec_t step_exec_[9] = {};
@@ -89,11 +90,15 @@ private:
     const NativeHead* head_ = nullptr;
     const float* window_R_ = nullptr;
     int max_t_ = 0;
+    int device_ = -1;   ///< the device `load` ran on: the public calls switch to it (layer split)
     int max_drafts_ = 1 << 30;
     int64_t n_vocab_ = 0;
     uint64_t vram_ = 0;
     cudaStream_t cs_ = nullptr;
     cudaGraphExec_t prefill_exec_[9] = {};
+    cudaGraphExec_t prefill_dev_exec_[9] = {};
+    int32_t* pf_dev_ = nullptr;   ///< E-4: a prompt's rows' token / step / position records, uploaded at once
+    int64_t pf_cap_ = 0;          ///< its capacity in ints
     cudaGraphExec_t round_exec_[9] = {};
 
     struct Tensor { std::string name, kind; int64_t rows = 0, cols = 0; uint64_t off = 0, bytes = 0; };

include/strata/core/on_device.hpp

@@ -0,0 +1,22 @@
+#pragma once
+// Multi-GPU: make `device` current for a scope and restore the caller's device after it.  An object that owns CUDA
+// streams, graphs and buffers on one device (a verify stage, the drafter) wraps its public calls in this, so the
+// caller's thread may be on any device.  A negative device, or the device already current, does nothing.
+
+#include <cuda_runtime.h>
+
+namespace strata::core {
+
+struct OnDevice {
+    int previous = -1;
+    explicit OnDevice(int device) {
+        int cur = 0;
+        if (device >= 0 && cudaGetDevice(&cur) == cudaSuccess && cur != device && cudaSetDevice(device) == cudaSuccess)
+            previous = cur;
+    }
+    ~OnDevice() { if (previous >= 0) cudaSetDevice(previous); }
+    OnDevice(const OnDevice&) = delete;
+    OnDevice& operator=(const OnDevice&) = delete;
+};
+
+}  // namespace strata::core

include/strata/core/pinned.hpp

@@ -39,7 +39,8 @@ struct PinnedArena {
     explicit PinnedArena(uint64_t bytes, uint64_t slice = 0);
     /// Plan v0.3 P6: slices of different sizes (one per layer of a native pack), given as their start offsets
     /// followed by the end of the last one.  `slice_starts` holds the registered ones.
-    PinnedArena(uint64_t bytes, const std::vector<uint64_t>& bounds);
+    /// `max_pinned_bytes`: optional cap on CUDA registration. 0 preserves the normal unrestricted path.
+    PinnedArena(uint64_t bytes, const std::vector<uint64_t>& bounds, uint64_t max_pinned_bytes = 0);
     std::vector<uint64_t> slice_starts;
     ~PinnedArena();
     PinnedArena(const PinnedArena&) = delete;
@@ -51,15 +52,40 @@ struct PinnedArena {
 
 struct LoadStats {
     double seconds = 0.0;
+    // Loader fix: the two halves of `seconds`, both SUMMED OVER THE READER THREADS rather than wall clock, so
+    // either can exceed `seconds` when the threads overlap.  They exist to separate the read from the copy and
+    // the hash: an 8 MiB chunk that reaches the disk as ~2048 4 KiB reads spends its time in the first one.
+    double read_seconds = 0.0;                  // inside the read call only: no seek, no copy, no hash
+    double copy_seconds = 0.0;                  // memcpy + FNV-1a
+    // A short read is not a slow load, it is a WRONG one: the caller must refuse the pack rather than run
+    // on an arena whose tail was never written.  `seconds` alone cannot say that (it is -1.0 on failure, but
+    // the caller's own check is on the REQUESTED byte count, which a refused read does not change).
+    bool ok = true;                             // false: the load failed, see `error`
+    std::string error;                          // why it failed, for the caller's message
     uint64_t bytes = 0;
     uint64_t layers = 0;
     std::vector<uint64_t> layer_checksums;      // one FNV-1a per layer
     double gib_per_second() const { return seconds > 0 ? (double) bytes / (1024.0 * 1024 * 1024) / seconds : 0.0; }
+    // Aggregate rate of the readers WHILE THEY WERE INSIDE THE READ CALL: `bytes` over the mean per-thread
+    // read time.  `gib_per_second()` above divides the same bytes by the wall clock of the whole loop, which
+    // also contains the copy and the hash, so the two differ by how much of the loop was not I/O.
+    double read_gib_per_second(int threads) const {
+        if (read_seconds <= 0.0 || threads < 1) return 0.0;
+        return (double) bytes / (1024.0 * 1024 * 1024) / (read_seconds / (double) threads);
+    }
 };
 
 // Load `layers` layers of the expert arena into `dst` with `threads` readers, `chunk` bytes at a time.
 // Each thread opens its OWN handle and seeks, which is the portable form of parallel pread: a shared handle
 // needs a lock around the seek and defeats the parallelism on Windows.
+//
+// The reads go through `fread()` on a per-thread `FILE*`, NOT through `std::ifstream`.  MSVC's
+// `basic_filebuf::xsgetn` splits every request larger than `_INTERNAL_BUFSIZ - 1` into 4095-byte `fread()`
+// calls, so an 8 MiB `chunk` reached the disk as ~2048 4 KiB operations and the chunking did nothing at all
+// (measured on a 42.9 GB pack: 1222 s, 4096 bytes per operation, 0.03 GiB/s).  `fread()` hands a request
+// larger than the stream buffer straight to `_read()`/`ReadFile()`, which is what `chunk` is for.  The
+// control flow, the copy, the per-layer FNV-1a and therefore the checksums are unchanged, so a pack loaded
+// by the old and the new reader must produce identical `layer_checksums`.
 LoadStats load_experts(const std::string& path, uint8_t* dst, uint64_t blob_bytes, uint64_t blobs_per_layer,
                        uint64_t layers, int threads, uint64_t chunk);
 /// Plan v0.3 P6: the same with one byte range per layer (`layer_off[L]`, `layer_bytes[L]`).

include/strata/core/remote_experts.hpp

@@ -0,0 +1,85 @@
+#pragma once
+
+#include "strata/core/expert_cache.hpp"
+#include "strata/core/expert_source.hpp"
+
+#include <cuda_runtime.h>
+
+#include <cstdint>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace strata::core {
+
+/// A static, profile-filled expert tier on another CUDA device. CUDA0 keeps all
+/// dense weights and state; results return through the existing pinned CPU rows.
+class RemoteExperts {
+public:
+    RemoteExperts() = default;
+    ~RemoteExperts();
+    RemoteExperts(const RemoteExperts&) = delete;
+    RemoteExperts& operator=(const RemoteExperts&) = delete;
+
+    /// Initialise the device before the host expert arena registers
+    /// tens of GiB of portable mapped memory with CUDA.
+    static bool preflight(int device, double& free_gib, std::string& err);
+    bool open(int device, int slots, int64_t layers, int64_t experts,
+              const std::vector<std::pair<int32_t, int32_t>>& ranked,
+              const ExpertCache& primary, ExpertSource& source,
+              std::vector<uint8_t>& claimed, std::string& err);
+    void close();
+
+    /// `kind` is the primary verifier's classification (-1 = CPU candidate),
+    /// or null on the one-token path. Entries already served on CUDA0 are excluded.
+    bool begin(int64_t layer, const float* x, const int32_t* ids, int64_t n_tok,
+               int64_t k, const int32_t* kind, const int32_t* primary_res,
+               std::string& err);
+    bool owns(int64_t index) const { return owned_[(size_t) index] != 0; }
+    bool finish(float* out, std::string& err);
+    int64_t resident() const { return cache_.resident(); }
+    int64_t computed() const { return computed_; }
+    int64_t launched_layers() const { return launched_layers_; }
+    double gib() const { return cache_.gib(); }
+    uint64_t returned_bytes() const { return returned_bytes_; }
+    uint64_t full_row_bytes() const { return full_row_bytes_; }
+    /// host time spent in begin() (staging + launches) and in finish() (waiting for this GPU), cumulative
+    double ms_begin() const { return ms_begin_; }
+    double ms_wait() const { return ms_wait_; }
+
+private:
+    int device_ = -1;
+    int64_t n_expert_ = 0;
+    int32_t groups_ = 0;
+    int64_t computed_ = 0;
+    int64_t launched_layers_ = 0;
+    uint64_t returned_bytes_ = 0;
+    uint64_t full_row_bytes_ = 0;
+    double ms_begin_ = 0, ms_wait_ = 0;
+    ExpertCache cache_;
+    cudaStream_t stream_ = nullptr;
+    float* h_x_ = nullptr;
+    float* h_out_ = nullptr;
+    void* h_meta_ = nullptr;
+    float* d_x_ = nullptr;
+    float* z_x_ = nullptr;     ///< h_x_ as the helper GPU sees it (zero-copy: no input copy per layer)
+    float* z_out_ = nullptr;   ///< h_out_ as the helper GPU sees it (zero-copy: no result copy)
+    bool zero_copy_ = false;
+    float* d_out_ = nullptr;
+    uint8_t* d_q8_ = nullptr;
+    float* d_scales_ = nullptr;
+    void* d_scratch_ = nullptr;
+    void* d_meta_ = nullptr;  ///< one contiguous upload of grouped indices, instead of five small copies
+    int32_t* d_start_ = nullptr;
+    int32_t* d_dst_ = nullptr;
+    int32_t* d_tok_ = nullptr;
+    int32_t* d_count_ = nullptr;
+    unsigned long long* d_ptr_ = nullptr;
+    std::vector<uint8_t> owned_;
+    std::vector<uint8_t> layers_present_;
+    std::vector<int32_t> group_of_, group_id_;
+    std::vector<int32_t> start_, dst_, tok_, original_row_;
+    std::vector<unsigned long long> ptr_;
+};
+
+} // namespace strata::core

include/strata/core/verify.hpp

@@ -32,6 +32,7 @@
 
 #include <cstdint>
 #include <string>
+#include <vector>
 
 namespace strata::core {
 
@@ -44,6 +45,8 @@ using PoolMultiFn = void (*)(void* user, const float* x_f, const int32_t* ids, i
 struct VerifyHits {
     const int32_t* d_res = nullptr;      ///< device [n_layers * n_expert] slot or -1
     const uint8_t* cache_base = nullptr; ///< slot 0 of the VRAM expert arena
+    const uint64_t* slot_off = nullptr;   ///< E-6: host per-slot offsets when slots differ in size (null: slot * blob)
+    int64_t n_slots = 0;                  ///< E-6: how many (for the device copy)
     int64_t blob = 0;
 };
 
@@ -69,6 +72,7 @@ public:
     /// parameters would otherwise be baked forever - so this can change between requests freely.
     void set_sampling(const strata::kernels::SamplerParams& sp) {
         sampling_ = sp;   // row t of a window at pos0 draws Philox(seed, pos0 + t): see run()
+        if (next_) next_->set_sampling(sp);
     }
 
     /// The penalty histories for `sampling_.penalty_last_n`: ONE ROW PER WINDOW ROW, T rows of `history_len`
@@ -80,18 +84,35 @@ public:
     void set_history(const int32_t* history, int history_len) {
         hist_d_ = history;
         hist_len_ = history_len;
+        if (next_) next_->set_history(history, history_len);
     }
     /// Off: `run` skips the request's head sampling and `out` is the recorded greedy pick.  For windows whose
     /// picks are discarded - a prompt read through windows commits every token - so they cost no sampler launch
     /// or sync and never read a history staged for another position.
-    void set_head_sampling(bool on) { head_sampling_ = on; }
+    void set_head_sampling(bool on) { head_sampling_ = on; if (next_) next_->set_head_sampling(on); }
+
+    /// LAYER SPLIT (multi-GPU): this verifier runs layers [layer_begin, layer_end) of every window.  A stage that
+    /// does not start at layer 0 takes its residual from `handoff_in` instead of embedding the tokens; a stage that
+    /// does not end at the last layer writes its residual to `handoff_out` and has no head.  The hand-off holds,
+    /// per token, the residual R (hc x n_embd), the last layer's pending write bo (n_embd) and inject (hc): the next
+    /// stage folds that write into its first read exactly as the unsplit window does, so the split is bit-exact.
+    /// Both pointers must be device-visible (mapped pinned memory, portable when the stages are on two devices).
+    /// Set before `init`.  Default: the whole model, no hand-off.
+    void set_stage(int64_t layer_begin, int64_t layer_end, const float* handoff_in, float* handoff_out) {
+        lb_ = layer_begin; le_ = layer_end; hand_in_ = handoff_in; hand_out_ = handoff_out;
+    }
+    /// The next stage: `run` and `commit` continue into it (its pool calls get `next_user`); sampling settings
+    /// and `final_R` are the last stage's.
+    void set_next(Verifier* next, void* next_user) { next_ = next; next_user_ = next_user; }
+    /// floats per token in a hand-off buffer
+    static int64_t handoff_floats(const ModelGeometry& g) { return (int64_t) g.hc * g.n_embd + g.n_embd + g.hc; }
 
     /// Keep the first `n_keep` (1..T) tokens of the last window; advances `ss.ple_prev` by them.
     bool commit(int n_keep, std::string& err);
 
     /// Token t's residual after the last layer, (hc, n_embd) on the device, valid until the next `run`.
     const float* final_R(int t) const;
-    const float* final_R_all() const { return R_; }
+    const float* final_R_all() const { return next_ ? next_->final_R_all() : R_; }
 
     /// The GPU plan the pool writes each layer (VRAM hits + the PCIe share of the misses); give it to the
     /// dispatch (`ExpertDispatch::plan`) before the first `run`.
@@ -104,9 +125,13 @@ public:
     /// arena directly, 2 = a copy kernel stages it inside the graph (no API calls on the pool's thread; best when
     /// the CPU is RAM-bound, Q2_0).  Set before the first `run`.
     void set_pcie_mode(int mode) { sink_.pcie_mode = mode; }
+    /// the pool never plans a PCIe share (--pcie-frac 0): the window skips that path.  Before the first run.
 
     double ms_wait = 0, ms_pool = 0, ms_host = 0, ms_commit = 0;
     int64_t windows = 0;
+    /// STRATA_VERIFY_PROFILE=1 - GPU stage times of the windows since the last call (ms per
+    /// window), as one line; empty when off.
+    std::string profile_report();
 
 private:
     bool capture(int T, std::string& err);
@@ -119,8 +144,24 @@ private:
     const int32_t* hist_d_ = nullptr;   ///< penalty-history row (set_history); null = no penalties apply
     int hist_len_ = 0;
     bool head_sampling_ = true;          ///< set_head_sampling
+    int device_ = -1;                    ///< the device `init` ran on: run/commit switch to it (layer split)
+    bool device_plan_ = false;            ///< E-6: resident-only layers planned on the device (STRATA_VERIFY_DEVICE_PLAN)
+    uint32_t* skip_ = nullptr;            ///< E-6: per group, the ring whose plan the device built (0: the host's)
+    unsigned long long* slot_off_d_ = nullptr;   ///< E-6: the slot offsets on the device
+    int64_t lb_ = 0, le_ = -1;           ///< set_stage: the layers this verifier runs (-1: to the last)
+    const float* hand_in_ = nullptr;
+    float* hand_out_ = nullptr;
+    Verifier* next_ = nullptr;
+    void* next_user_ = nullptr;
+    bool ple_stage() const { return lb_ <= 1 && 1 < le_; }   ///< holds layer 1, where the PLE block runs
     bool capture_commit(std::string& err);
     bool record_window(int T, cudaStream_t cs, std::string& err);
+    static constexpr int kProfPer = 32;              // stamps per layer
+    bool prof_on_ = false;
+    unsigned long long* prof_ = nullptr;              // device: n_layers * kProfPer + 4 stamps
+    std::vector<unsigned long long> prof_h_;
+    double prof_sum_[2][kProfPer] = {};   // [GDN / QSA layers][stage]
+    int64_t prof_windows_ = 0;
 
     const WeightTable* wt_ = nullptr;
     const ModelGeometry* g_ = nullptr;

include/strata/kernels/bf16_gemv.hpp

@@ -44,5 +44,9 @@ void bf16_gemv_split(const uint16_t* x, const uint16_t* w, float* y, int64_t n_i
 /// No allocations or synchronization, including when stream is null (the CUDA default stream).
 void bf16_gemv_fp32_mmvf(const float* x, const uint16_t* w, float* y,
                          int64_t n_in, int64_t n_out, void* stream);
+/// bf16_gemv_fp32_mmvf for n_tok (1..8) activation rows x[t * ldx], outputs y[t * ldy + j]; one launch,
+/// the weight read once, every output bit-identical to its own single-row call.
+void bf16_gemv_fp32_mmvf_multi(const float* x, int64_t ldx, const uint16_t* w, float* y, int64_t ldy,
+                               int64_t n_in, int64_t n_out, int n_tok, void* stream);
 
 }  // namespace strata::kernels

include/strata/kernels/cvec.hpp

@@ -43,7 +43,12 @@ const Cvec& cvec();
 bool cvec_upload(const std::vector<float>& dir, const std::vector<float>& s, int mode, int first, int last,
                  int64_t n_embd, int64_t hc, std::string& err);
 
-/// The per-request switch.  Synchronizes the device when it changes, so call it between requests.
+/// A layer split: put the loaded vector's tables on the CURRENT device too (cvec_apply uses the tables of the device
+/// it runs on).  No-op without a vector or when this device has them already.
+bool cvec_replicate(std::string& err);
+
+/// The per-request switch, on every device that holds the vector.  Synchronizes them when it changes, so call it
+/// between requests.
 void cvec_set_enabled(bool on);
 bool cvec_enabled();
 

include/strata/kernels/elementwise.hpp

@@ -102,6 +102,10 @@ void doorbell_wait(const uint32_t* d_flag, const uint32_t* d_seq, void* stream);
 /// device memory with a kernel, so the handoff stays on the compute queue (a memcpy node is a copy-engine
 /// operation, which WDDM submits separately and which measured 67 flushes per token).
 void copy_from_mapped(float* dst, const float* src, int64_t n, void* stream);
+/// `rows` rows of `width` floats from mapped memory, except the rows listed in hit_rows[0, *count) (device),
+/// which are written +0.0 instead (a verify window's GPU-computed entries: the pool leaves zeros there).
+void copy_rows_from_mapped(float* dst, const float* src, int64_t rows, int64_t width, const int32_t* hit_rows,
+                           const int32_t* count, void* stream);
 
 /// Plan v0.3 P3: the doorbell's payload and its ring in ONE kernel.  Copies `x` (n floats), `ids` and `weights`
 /// (k each) into the mapped host regions, fences, and increments the mapped sequence number - replacing three

include/strata/kernels/fused_gr.hpp

@@ -46,6 +46,7 @@ void fused_gr_read(const FusedGrArgs& a, void* stream);
 /// weight pointers and eps must be the same for every t); `xn_scratch` is n_tok * hc * n_embd floats.  Every
 /// token's outputs are bitwise `fused_gr_read(a[t])`.
 constexpr int kFusedGrMaxT = 8;
-void fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, void* stream);
+void fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, void* stream,
+                         unsigned long long* stamp_buf = nullptr, int stamp_i0 = 0);
 
 }  // namespace strata::kernels

include/strata/kernels/mrope.hpp

@@ -18,6 +18,7 @@
 
 namespace strata::kernels {
 
+/// The table of the CURRENT device (a layer split sets one per device; null = the identity).
 void mrope_table_set(const int32_t* device_table);
 const int32_t* mrope_table();
 

include/strata/kernels/native_moe.hpp

@@ -15,4 +15,7 @@ bool native_moe_combine_enabled();
 // Requires a nonnull ordered stream and disjoint output. No allocation or sync.
 void native_moe_combine(const float* parts, const float* weights, const float* shared,
                         float* output, int64_t n_embd, int64_t k, void* stream);
+/// n_tok rows (parts [n,k,N], weights [n,k], shared/output [n,N]) in one launch, each as the single call.
+void native_moe_combine_multi(const float* parts, const float* weights, const float* shared, float* output,
+                              int64_t n_embd, int64_t k, int n_tok, void* stream);
 }

include/strata/kernels/native_qsa_indexer.hpp

@@ -27,6 +27,13 @@ bool native_qsa_indexer_enabled();
 // max_cells is a positive fixed capacity; pos_base+max_cells must fit int32.
 // Out-of-range device positions are ignored defensively, not host-validated.
 // This adapter does not alter score accumulation, top-k, or finite 1e9 tail bias.
+// perf-review C-2: the appends of cells [p0, p0 + n) at once, leaving the buffers exactly as n calls of the
+// single append in order would (the prompt path appends a chunk before any query reads the state). raw [n, 128].
+// Host-side positions (not for a captured graph).
+void native_qsa_indexer_append_batch(const float* raw, int64_t n, int64_t p0, int32_t pos_base, const float* gamma,
+                                     float epsilon, const QsaIndexerBuffers& b, const QsaShapes& s, int64_t max_cells,
+                                     float freq_base, void* stream);
+
 void native_qsa_indexer_append(const float* raw, const int32_t* relative_pos_device,
                                int32_t pos_base, const float* gamma, float epsilon,
                                const QsaIndexerBuffers& buffers, const QsaShapes& shapes,

include/strata/kernels/native_router.hpp

@@ -14,4 +14,6 @@ bool native_router_enabled();
 // four-byte aligned and outputs disjoint from each other and the input.
 // Requires a nonnull ordered CUDA stream. No allocation or synchronization.
 void native_router_top10(const float* logits, int32_t* ids, float* weights, void* stream);
+/// n_tok rows at once (logits [n,512], ids/weights [n,10]); each row exactly as the single call.
+void native_router_top10_multi(const float* logits, int32_t* ids, float* weights, int n_tok, void* stream);
 }

include/strata/kernels/qsa_prompt_attn.hpp

@@ -0,0 +1,31 @@
+// include/strata/kernels/qsa_prompt_attn.hpp - perf-review D-1: the prompt path's QSA attention on tensor cores.
+//
+// `qsa_decode_attn_batch` serves a prompt one query at a time with the decode kernel: FP32 dot products with a warp
+// reduction per head and cell, 64-cell chunks whose partial sums go through global scratch to a second (merge)
+// kernel. At 32K that is 21% of the prompt (5.6 s of 26 s on a 5070, Q2_0, int8 KV), and it is limited by
+// instruction issue, not memory (~150 GB/s of logical reads, 3.4 TFLOP/s).
+//
+// This kernel keeps the same per-query selection (no masking, no union) and reads the same pools, but:
+//   * one block per (query, KV head) walks all its selected cells in chunks of 32 with an online softmax: no
+//     split-K scratch and no merge kernel;
+//   * q.k and p.v are m16n8k16 FP16 MMAs with FP32 accumulation over the 12 query heads of the KV head (+4 pad rows);
+//   * the stored values enter exactly: int8 codes are exact in FP16 and their per-64 scales are applied in FP32
+//     (to the q.k partial of each 64-dim group, and folded into p for p.v); FP16 KV is used as is;
+//   * q and p are split into FP16 hi + lo parts (two MMAs each), so they keep ~22 bits: the result differs from
+//     the FP32 kernel by summation order and the exp2 rounding, not by an FP16 cast.
+// Not bitwise equal to `qsa_decode_attn_batch`; `qsa_prompt_attn_parity` bounds the difference and the prompt
+// quality gate (needles, teacher-forced top-1) checks it end to end. Q4_0 KV is not handled (returns false).
+#pragma once
+
+#include "strata/kernels/qsa_decode_attn.hpp"
+
+#include <cstdint>
+
+namespace strata::kernels {
+
+/// Same arguments and output as `qsa_decode_attn_batch` minus the scratch. Returns false (nothing launched) when the
+/// pools are Q4_0 or the geometry is not 24 heads / 2 KV heads / 256: the caller then uses the old kernel.
+bool qsa_prompt_attn_batch(const float* q, const QsaAttnPools& pools, const int32_t* ids, const int32_t* steps,
+                           int64_t cap, const QsaShapes& s, float* attn, int64_t n_q, void* stream);
+
+}  // namespace strata::kernels

include/strata/kernels/qsa_select.hpp

@@ -24,7 +24,9 @@ namespace strata::kernels {
 
 /// scores [nq, max_blocks]; q_idx [nq, idx_n_head, idx_dim] (normed and rotated); steps [nq, kStepCount].
 void qsa_block_scores(const float* pooled, const float* dead, const float* q_idx, const int32_t* steps, int64_t nq,
-                      int64_t max_blocks, const QsaShapes& s, float* scores, void* stream);
+                      int64_t max_blocks, const QsaShapes& s, float* scores, void* stream,
+                      int64_t active_blocks = -1);   ///< perf-review C-1: > 0 launches only this many blocks (the
+                                                     ///< batch's largest n_bid + 1; not for a captured graph)
 
 /// ids [nq, cap] (cells, ascending); `cap` >= the largest selection width.
 void qsa_block_topk(const float* scores, const int32_t* steps, int64_t nq, int64_t max_blocks, int64_t cap,

include/strata/kernels/verify_kernels.hpp

@@ -38,6 +38,24 @@ void gdn_step_norm_multi(float* state, const float* h, int conv_channels, const
 /// Spin until *flag >= value (a mapped host flag).  The value is fixed at capture, so several rings can be
 /// outstanding at once (the split verify window keeps two).
 void wait_flag_ge(const uint32_t* flag, uint32_t value, void* stream);
+/// the GPU's %globaltimer (ns) into buf[i] (a one-thread kernel: the verify window's stage profiler).
+void gpu_stamp(unsigned long long* buf, int i, void* stream);
+
+// ---- perf-review E-6: a layer whose routed experts are all in VRAM needs nothing from the host
+/// One group's plan, built on the device when every routed expert of its n*k entries is resident: the host pool's
+/// layout (counts | start | dst | tok | pad | ptr | ptr2 | start2, `capx` entries) and order (distinct experts in
+/// routing order, their entries ascending), no PCIe groups.  *skip = ring when it did, else 0.
+void resident_plan(const int32_t* ids, int n_entries, int k, const int32_t* res_layer, int n_expert,
+                   const uint8_t* cache_base, const unsigned long long* slot_off, long long blob, int32_t* plan,
+                   long long capx, uint32_t* skip, uint32_t ring, void* stream);
+/// wait_flag_ge that also returns when *skip == value (device memory).
+void wait_flag_ge_or(const uint32_t* flag, uint32_t value, const uint32_t* skip, void* stream);
+/// copy_i32_from_mapped unless *skip == value.
+void copy_i32_from_mapped_unless(int32_t* dst, const int32_t* src, long long n, const uint32_t* skip, uint32_t value,
+                                 void* stream);
+/// copy_from_mapped, or zeros when *skip == value (n a multiple of 4, 16-byte aligned).
+void copy_or_zero_from_mapped(float* dst, const float* src, long long n, const uint32_t* skip, uint32_t value,
+                              void* stream);
 
 /// Rows of the S2/S4/S8 embedding for T token ids read from DEVICE memory; out (T, n).  Bitwise `embedding_gather`.
 void embedding_gather_dev(const uint8_t* codes, const float* scales, const float* offsets, const int32_t* tokens,

include/strata/platform/direct_file.hpp

@@ -2,7 +2,8 @@
 //
 // The n-gram table (26.8 GiB) stays on the SSD and must never occupy RAM, including the OS file cache. A memory
 // map cannot promise that; an unbuffered read can. On Windows this is FILE_FLAG_NO_BUFFERING | OVERLAPPED with
-// an I/O completion port; on Linux, O_DIRECT (synchronous pread for now; io_uring is phase L).
+// an I/O completion port; on Linux, O_DIRECT preads.  Both issue the reads from a small thread pool (submit only
+// queues), so many reads are in flight: STRATA_IO_THREADS sets its size (default 4 on Windows, 16 on Linux).
 //
 // Contract of every read: offset, length and buffer address are multiples of `alignment()` (4096 here).
 // Reads past end of file return the bytes that exist; `Completion::bytes` says how many.

include/strata/prefill/prefill.hpp

@@ -72,6 +72,12 @@ public:
     /// K/V from them.  The prefill stream is synchronized before the call.
     std::function<bool(const float* R_rows, int64_t T, int64_t pos0, std::string& err)> on_chunk;
 
+    /// Layer split: called by every stage when it has read a chunk, with the position reached, while its own state
+    /// is still at that chunk's end (its stream synchronized; the last stage calls it just before `on_chunk`).  An
+    /// earlier stage is a chunk or more ahead of the last one by the time `on_chunk` runs, so this is where a
+    /// mid-prompt checkpoint takes each stage's part.  Runs on that stage's thread, with its device current.
+    std::function<bool(int64_t done, std::string& err)> on_stage_chunk;
+
     /// Checked before every chunk: true stops the prompt early (`run` returns false with err "cancelled").
     std::function<bool()> should_stop;
 
@@ -79,7 +85,19 @@ public:
     /// embedding where non-null (an image's <|image_pad|> cells).  Null (default): every position embeds its token.
     const float* const* embd_rows = nullptr;
 
+    /// LAYER SPLIT (multi-GPU): this prompt path runs layers [layer_begin, layer_end) (-1: to the last) on the
+    /// device `init` runs on.  A stage that does not start at layer 0 reads each chunk's residual rows from the
+    /// previous stage instead of embedding the tokens; a stage that does not end at the last layer copies its rows
+    /// to pinned host buffers (two, allocated by `init`) and runs `next` on them - on a thread, so the next stage
+    /// reads chunk c while this one reads chunk c + 1.  `on_chunk` belongs on the last stage.  Set before `init`.
+    void set_stage(int64_t layer_begin, int64_t layer_end, Prefill* next) {
+        stage_lb_ = layer_begin; stage_le_ = layer_end; next_ = next;
+    }
+
 private:
+    int64_t stage_lb_ = 0, stage_le_ = -1;
+    Prefill* next_ = nullptr;
+    const float* hand_in_ = nullptr;    ///< the previous stage's rows of the chunk being read (host, pinned)
     bool carve(std::size_t T, void* alloc);   // the device buffers of a chunk (prefill.cpp's Alloc)
     struct Impl;
     std::unique_ptr<Impl> impl_;

serve/frontend.py

@@ -38,7 +38,8 @@ class ChatTemplate:
         env = ImmutableSandboxedEnvironment(trim_blocks=True, lstrip_blocks=True, extensions=["jinja2.ext.loopcontrols"])
         env.filters["tojson"] = tojson
         env.globals["raise_exception"] = raise_exception
-        self.template = env.from_string(Path(path).read_text(encoding="utf-8"))
+        self.source = Path(path).read_text(encoding="utf-8")
+        self.template = env.from_string(self.source)
 
     def render(self, messages: list[dict], tools: list[dict] | None = None, add_generation_prompt: bool = True,
                **kwargs) -> str:
@@ -144,7 +145,7 @@ def openai_to_messages(req: dict) -> tuple[list[dict], list[dict] | None, dict]:
         role = m.get("role")
         if role == "developer":
             role = "system"
-        out = {"role": role, "content": _parts_of(m.get("content")) if role == "user" else _text_of(m.get("content"))}
+        out = {"role": role, "content": _parts_of(m.get("content")) if role in ("user", "tool", "assistant") else _text_of(m.get("content"))}
         if m.get("reasoning_content"):
             out["reasoning_content"] = m["reasoning_content"]
         if m.get("tool_calls"):

serve/server.py

@@ -4,7 +4,7 @@
     python -m serve.server --engine strata --config strata.json --port 8080   (the real engine, resident)
 
 Endpoints: POST /v1/chat/completions (OpenAI, stream and non-stream), POST /v1/messages (Anthropic, stream and
-non-stream), GET /v1/models, GET /health, GET /mcp. One sequence at a time behind a FIFO (plan: one resident sequence).
+non-stream), GET /v1/models, GET /models, GET /props, GET /slots, GET /health, GET /mcp. One sequence at a time behind a FIFO (plan: one resident sequence).
 Tools from MCP servers (serve/mcp.py, `"mcp_servers"` in the config or --mcp-config) are offered only to requests that
 ask for them with `"strata_mcp": true` - the web app does; other clients see exactly the API they always saw.
 Images (optional, when the config has a "vision" entry): OpenAI image_url parts and Anthropic image blocks (base64
@@ -24,6 +24,7 @@ import argparse
 import collections
 import base64
 import hashlib
+import codecs
 import json
 import os
 import queue
@@ -37,6 +38,7 @@ import uuid
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
 from typing import Iterator, Protocol
+from urllib.parse import parse_qs, urlsplit
 
 ROOT = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(ROOT / "tools"))
@@ -48,6 +50,10 @@ from serve.mcp import McpCancelled, hub_from_config  # noqa: E402
 IM_END = "<|im_end|>"
 IMAGE_PAD = "<|image_pad|>"
 CTX_SLACK = 8               # `strata --serve` rejects prompt + max_new + 8 > context: keep the same margin here
+# The live tok/s is a rate over a window, not a mean since the first token: a mean reads ~1/elapsed at the first
+# token (the Monitor showed five-digit numbers) and then undershoots for the first second of every answer.
+RATE_WINDOW_S = 2.0
+RATE_MIN_SPAN_S = 0.25      # younger than this there is no rate yet: the mean so far, with the span floored here
 
 
 # ------------------------------------------------------------------------------------------------ engines
@@ -149,6 +155,8 @@ class StrataEngine:
     def __init__(self, exe: str, args: list[str], cwd: str | None = None, log: str | None = None,
                  env: dict | None = None):
         self.spawn = (exe, list(args), cwd, log, env)   # to start it again after it died (issue #27)
+        paths = {k: v for k, v in zip(args, args[1:]) if k in ("--native", "--pack")}
+        self.model_path = paths.get("--native") or paths.get("--pack", "pack/full")
         self.log_path = log
         self.log = open(log, "a", encoding="utf-8") if log else subprocess.DEVNULL
         loading = threading.Event()                     # set once READY: the narrator below stops
@@ -299,8 +307,9 @@ class StrataEngine:
         the HTTP layer turns it into an SSE comment, which keeps clients' watchdogs calm and notices a client that
         has gone.  A consumer that stops early (or `cancel`) makes the engine STOP, so it does not run to max_new."""
         self.progress = None
-        head = f"GENI {int(max_new)}{self.projection_key(sampling or {})} {embeddings}" if embeddings else \
-            f"GEN {int(max_new)}{self.sampling_keys(sampling or {}) if not embeddings else ''}"
+        # an image request takes the same sampling keys as text (#75: it used to decode greedily whatever was asked)
+        head = f"GENI {int(max_new)}{self.sampling_keys(sampling or {})} {embeddings}" if embeddings else \
+            f"GEN {int(max_new)}{self.sampling_keys(sampling or {})}"
         try:
             self.proc.stdin.write(f"{head} {','.join(str(int(t)) for t in ids)}\n")
             self.proc.stdin.flush()
@@ -456,13 +465,32 @@ class Vision:
             self.proc.kill()
 
 
+def gpu_list(cfg: dict) -> list[int]:
+    """The config's "gpu": one card (2), or several for a layer split ([0, 2] or "0,2"), numbered as nvidia-smi
+    numbers them; [] when it names none."""
+    g = cfg.get("gpu")
+    if g is None or g == "":
+        return []
+    items = g if isinstance(g, (list, tuple)) else str(g).split(",")
+    return [int(str(x).strip()) for x in items if str(x).strip() != ""]
+
+
+def engine_args(cfg: dict) -> list[str]:
+    """The engine's arguments: the config's, and with several GPUs the layer split across them ("layer_split" in the
+    config: "auto" by default, or the first layer of each later GPU's share, e.g. "18" or "16,32")."""
+    args = list(cfg["args"])
+    if len(gpu_list(cfg)) > 1 and "--layer-split" not in args:
+        args += ["--layer-split", str(cfg.get("layer_split") or "auto")]
+    return args
+
+
 def child_env(cfg: dict) -> dict:
     """The engine's environment: the CUDA libraries setup installed (pip's nvidia packages, or the toolkit that
     compiled it) first on the library search path."""
     env = dict(os.environ)
-    if cfg.get("gpu") is not None:                   # issue #51: the GPU to run on, numbered as nvidia-smi does; CUDA's
+    if gpu_list(cfg):                                # issue #51: the GPU(s) to run on, numbered as nvidia-smi does; CUDA's
         env["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"      # own default order (fastest first) can number the cards otherwise
-        env["CUDA_VISIBLE_DEVICES"] = str(cfg["gpu"])
+        env["CUDA_VISIBLE_DEVICES"] = ",".join(str(i) for i in gpu_list(cfg))
     dirs = [d for d in cfg.get("lib_dirs") or [] if Path(d).is_dir()]
     if dirs:
         var = "PATH" if os.name == "nt" else "LD_LIBRARY_PATH"
@@ -496,13 +524,19 @@ class ByteTokenizer:
 
 # ------------------------------------------------------------------------------------------------ core
 class Detokenizer:
-    """Incremental decode: re-decode the generated ids and emit only the new, complete suffix (a multi-byte
-    character split across tokens is held until complete)."""
+    """Incremental decode: each token's bytes go through an incremental UTF-8 decoder, which emits the complete
+    characters and holds a multi-byte character split across tokens until it is complete (invalid bytes become
+    U+FFFD, as a whole decode with errors="replace" makes them).  Constant time per token - the old re-decode of
+    every generated id cost 2 ms per token after 8K tokens and 4 ms after 16K (perf-review F-1).  A tokenizer
+    without `token_bytes` (the tests' byte tokenizer) keeps the re-decode."""
 
     def __init__(self, tok):
         self.tok, self.ids, self.sent = tok, [], 0
+        self.inc = codecs.getincrementaldecoder("utf-8")(errors="replace") if hasattr(tok, "token_bytes") else None
 
     def push(self, t: int) -> str:
+        if self.inc is not None:
+            return self.inc.decode(self.tok.token_bytes(t))
         self.ids.append(t)
         text = self.tok.decode(self.ids)
         if text.endswith("�"):
@@ -524,10 +558,14 @@ class Service:
         self.embeddings = threading.local()           # the current request's image embeddings file (GENI)
         self.api_key = ""                              # when set, /v1/* needs it (Bearer or x-api-key)
         self.status = {"busy": False, "queued": 0}      # GET /status: what the model is doing right now
+        self.rate = collections.deque(maxlen=32)        # (time, generated) samples for the live tok/s window
         self.history = collections.deque(maxlen=500)    # the last finished requests, newest last (GET /metrics)
         # since the server started (the Monitor's totals, issue #35)
         self.totals = {"since": time.time(), "requests": 0, "prompt_tokens": 0, "reused": 0, "output_tokens": 0,
                        "prompt_ms": 0.0, "decode_ms": 0.0}
+        self.last_timings = None                         # the last finished request's, llama.cpp's names (/v1/status)
+        self.last_request_at = None                      # when a request last started or finished
+        self.started_at = time.time()
         self.status_lock = threading.Lock()
         self.mcp = None                                  # serve/mcp.py's McpHub when MCP servers are configured
         self.stop_ids = set(tokenizer.encode(IM_END, parse_special=True) +
@@ -572,10 +610,26 @@ class Service:
         """The hardware sampler behind GET /metrics (serve/telemetry.py), recording this server's tok/s too."""
         if getattr(self, "telemetry", None) is None:
             from serve.telemetry import Telemetry
-            self.telemetry = Telemetry(extra=lambda: {"tok_s": self._tok_s()},
-                                       gpu_index=int(getattr(self, "gpu_index", 0) or 0))
+            self.telemetry = Telemetry(extra=lambda: {"tok_s": self._tok_s(), "tok_s_mean": self._tok_s_mean()},
+                                       gpu_index=int(getattr(self, "gpu_index", 0) or 0),
+                                       gpu_indices=getattr(self, "gpu_indices", None))
 
     def _tok_s(self):
+        """tok/s over the last RATE_WINDOW_S seconds.  Returns 0.0 while nothing is generating."""
+        with self.status_lock:
+            s = dict(self.status)
+            rate = list(self.rate)
+        if not s.get("busy") or not s.get("first_token"):
+            return 0.0
+        now = time.time()
+        newest = rate[-1] if rate else None
+        oldest = next(((t, g) for t, g in rate if now - t <= RATE_WINDOW_S), None)
+        if newest and oldest and newest[0] - oldest[0] >= RATE_MIN_SPAN_S:
+            return max(0.0, (newest[1] - oldest[1]) / (newest[0] - oldest[0]))
+        return s["generated"] / max(RATE_MIN_SPAN_S, now - s["first_token"])
+
+    def _tok_s_mean(self):
+        """The whole-request mean since the first token (the old formula), kept so the two can be compared."""
         with self.status_lock:
             s = dict(self.status)
         if not s.get("busy") or not s.get("first_token"):
@@ -602,7 +656,9 @@ class Service:
                 "prompt_read": None, "prompt_total": None, "generated": s.get("generated") if s.get("busy") else None,
                 "max_tokens": s.get("max_tokens") if s.get("busy") else None,
                 "elapsed_s": round(now - s["started"], 1) if s.get("busy") and s.get("started") else None,
-                "tok_s": round(self._tok_s(), 1) if state == "generating" else None}
+                "tok_s": round(self._tok_s(), 1) if state == "generating" else None,
+                "tok_s_mean": round(self._tok_s_mean(), 1) if state == "generating" else None,
+                "tok_s_window_s": RATE_WINDOW_S if state == "generating" else None}
         if state == "reading" and progress:
             live["prompt_read"], live["prompt_total"] = progress
         engine = {"model": self.model, "max_context": self.engine.max_context, "images": self.vision is not None,
@@ -613,6 +669,41 @@ class Service:
                 "hardware_static":
                 tel["static"], "history": tel["history"], "time": now}
 
+    def v1_status(self) -> dict:
+        """GET /v1/status: what this server is and does, for a client that would rather ask than guess (a front-end
+        that polls its OpenAI-compatible server's status, collabosm's for one): the model and its window, images,
+        the APIs, what is running, and the last request's timings in llama.cpp's names.  /metrics has the rest."""
+        with self.status_lock:
+            s, totals = dict(self.status), dict(self.totals)
+            last_t, last_at = (dict(self.last_timings) if self.last_timings else None), self.last_request_at
+        tel = self.telemetry.snapshot() if getattr(self, "telemetry", None) else {"now": {}, "static": {}}
+        hw, static = tel.get("now") or {}, tel.get("static") or {}
+
+        def scaled(v, unit, digits=0):
+            return round(v / unit, digits) if isinstance(v, (int, float)) else None
+
+        busy, ctx = bool(s.get("busy")), self.engine.max_context
+        images = self.vision is not None
+        return {
+            "service": "strata", "model": self.model,
+            "engine": (getattr(self.engine, "info", {}) or {}).get("version"),
+            "started": int(self.started_at), "uptime_s": int(time.time() - self.started_at),
+            "cache_max_tokens": ctx,
+            "context": {"native": ctx, "max_positions": ctx},
+            "concurrency": {"serving": 1, "requested": 1},       # one request at a time; more wait their turn
+            "dialects": ["/v1/chat/completions", "/v1/messages"],
+            "vision": {"enabled": images, "available": images, "error": None},
+            "activity": {"requests": totals["requests"] + int(busy), "in_flight": int(busy) + int(s.get("queued") or 0),
+                         "last_request_at": int(last_at) if last_at else None},
+            "last_timings": last_t,
+            "machine": {
+                "at": int(time.time()),
+                "gpu": {"name": static.get("gpu_name"), "used_mib": scaled(hw.get("gpu_mem_used"), 2 ** 20),
+                        "total_mib": scaled(hw.get("gpu_mem_total"), 2 ** 20), "util_pct": hw.get("gpu_util"),
+                        "temp_c": hw.get("gpu_temp"), "power_w": hw.get("gpu_power")} if static.get("gpu_name") else None,
+                "ram": {"used_gib": scaled(hw.get("ram_used"), 2 ** 30, 1),
+                        "total_gib": scaled(hw.get("ram_total"), 2 ** 30, 1)} if hw.get("ram_total") else None}}
+
     def prepare(self, messages, tools, kwargs, max_new=None):
         """-> (ids, thinking, max_new). An unset or non-positive max_new (some clients send -1) means "unlimited":
         the rest of the context."""
@@ -665,6 +756,7 @@ class Service:
             s["generated"] = n
             if s.get("first_token") is None:
                 s["first_token"] = time.time()
+            self.rate.append((time.time(), n))          # the live rate's window over the last RATE_WINDOW_S
             for ev in evs:
                 if ev.kind == "reasoning":
                     s["phase"] = "thinking"
@@ -702,8 +794,12 @@ class Service:
             sampling = {**defaults, **req_values}
         parser = OutputParser(thinking=thinking, tools=tools, stream_tools=True)
         detok, n, finish = Detokenizer(self.tok), 0, "length"
+        timings, before = None, None                    # this request's timings; the engine's `last` before it
         raw_ids = []                                    # every generated id (STRATA_DEBUG: dump raw model text)
         emb = getattr(self.embeddings, "path", None)
+        # Identity token: only a DONE line replaces engine.last, so a request that died, errored or was
+        # disconnected must not have the PREVIOUS request's decode figures recorded as its own.
+        engine_last0 = getattr(self.engine, "last", None)
         with self.status_lock:
             self.status["queued"] += 1
         try:
@@ -720,6 +816,9 @@ class Service:
                 with self.status_lock:
                     self.status.update(busy=True, phase="reading the prompt", prompt_tokens=len(ids), generated=0,
                                        started=time.time(), first_token=None, tool=None, tail="", max_tokens=max_new)
+                    self.last_request_at = time.time()
+                    self.rate.clear()               # the previous request's samples must not leak into this one
+                before = getattr(self.engine, "last", None)
                 last_print = time.time()
                 gen = self.engine.generate(ids, max_new, sampling, cancel, embeddings=emb) if emb else \
                     self.engine.generate(ids, max_new, sampling, cancel)
@@ -765,7 +864,9 @@ class Service:
                 Path(emb).unlink(missing_ok=True)
             with self.status_lock:
                 if self.status.get("busy"):
-                    last = dict(getattr(self.engine, "last", {}) or {})
+                    # only this request's DONE counts: same object means no DONE arrived (death, error, disconnect)
+                    last = dict(getattr(self.engine, "last", {}) or {}) \
+                        if getattr(self.engine, "last", None) is not engine_last0 else {}
                     started = self.status.get("started", time.time())
                     loaded = str((getattr(self.engine, "info", {}) or {}).get("cvec", 0)) not in ("0", "", "None")
                     hit_rate = round(last["hits"] / last["lookups"], 3) if last.get("lookups") else None
@@ -774,6 +875,7 @@ class Service:
                         if loaded else None,
                         "time": started, "duration_s": round(time.time() - started, 1), "finish": finish,
                         "prompt_tokens": len(ids), "reused": last.get("reused"), "output_tokens": n,
+                        "engine_generated": last.get("generated"),
                         "prompt_ms": last.get("prompt_ms"), "decode_ms": last.get("decode_ms"),
                         "decode_tok_s": round(last["generated"] / (last["decode_ms"] / 1000), 1)
                         if n and last.get("generated") and last.get("decode_ms") else None,
@@ -785,6 +887,11 @@ class Service:
                     t["output_tokens"] += n
                     t["prompt_ms"] += last.get("prompt_ms") or 0.0
                     t["decode_ms"] += last.get("decode_ms") or 0.0
+                    fresh = getattr(self.engine, "last", None)
+                    if fresh is not None and fresh is not before:      # the engine's clock for THIS request
+                        timings = request_timings(len(ids), n, last)
+                        self.last_timings = dict(timings, at=int(time.time())) if timings else None
+                    self.last_request_at = time.time()
                     now = time.time()
                     el = now - self.status.get("started", now)
                     ft = self.status.get("first_token")
@@ -797,7 +904,28 @@ class Service:
                 self.status["busy"] = False
         for ev in parser.finish():
             yield "event", ev
-        yield "done", {"finish": finish, "completion_tokens": n}
+        yield "done", {"finish": finish, "completion_tokens": n, "reused": (timings or {}).get("cache_n", 0),
+                       "timings": timings}
+
+
+def request_timings(prompt_tokens: int, generated: int, last: dict) -> dict | None:
+    """One request's `timings` in llama.cpp's names (what its clients show as speed), from the engine's own clock
+    (StrataEngine.last): prompt_n is what was read, cache_n what the conversation cache already held.  None when the
+    engine keeps no clock (MockEngine)."""
+    if last.get("prompt_ms") is None:
+        return None
+    cache_n = int(last.get("reused") or 0)
+    prompt_n, prompt_ms, decode_ms = max(0, prompt_tokens - cache_n), float(last["prompt_ms"]), float(last.get("decode_ms") or 0)
+    decoded = int(last.get("generated") or generated)          # the engine's count gives its rate, as /metrics does
+    return {"cache_n": cache_n, "prompt_n": prompt_n, "prompt_ms": round(prompt_ms, 1),
+            "prompt_per_token_ms": round(prompt_ms / prompt_n, 3) if prompt_n else None,
+            "prompt_per_second": round(prompt_n / (prompt_ms / 1000), 1) if prompt_n and prompt_ms > 0 else None,
+            "predicted_n": generated, "predicted_ms": round(decode_ms, 1),
+            "predicted_per_token_ms": round(decode_ms / decoded, 3) if decoded else None,
+            "predicted_per_second": round(decoded / (decode_ms / 1000), 1) if decoded and decode_ms > 0 else None,
+            # the speculative drafts, as llama.cpp names them (from PR #83, @mikicvi): only when the engine reported them
+            **({"draft_n": int(last["drafts_offered"]), "draft_n_accepted": int(last["drafts_accepted"])}
+               if last.get("drafts_offered") is not None else {})}
 
 
 def _debug_req(api, req, messages, tools, max_new, thinking, prompt_tokens):
@@ -948,7 +1076,11 @@ def openai_chunks(svc: Service, req: dict, ids, thinking, tools, max_new, cancel
             last = chunk({}, finish)
             pt = x.get("prompt_tokens", len(ids))     # after MCP rounds: the last round's prompt
             last["usage"] = {"prompt_tokens": pt, "completion_tokens": x["completion_tokens"],
-                             "total_tokens": pt + x["completion_tokens"]}
+                             "total_tokens": pt + x["completion_tokens"],
+                             # the part of the prompt the conversation cache already held (OpenAI's field)
+                             "prompt_tokens_details": {"cached_tokens": x.get("reused") or 0}}
+            if x.get("timings"):
+                last["timings"] = x["timings"]          # llama.cpp's field: the speed its clients show
             yield last
 
 
@@ -978,9 +1110,12 @@ def openai_collect(chunks) -> dict:
         msg["tool_calls"] = calls
     if mcp:
         msg["strata_mcp"] = mcp
-    return {"id": last["id"], "object": "chat.completion", "created": last["created"], "model": last["model"],
-            "choices": [{"index": 0, "message": msg, "finish_reason": last["choices"][0]["finish_reason"]}],
-            "usage": last["usage"]}
+    out = {"id": last["id"], "object": "chat.completion", "created": last["created"], "model": last["model"],
+           "choices": [{"index": 0, "message": msg, "finish_reason": last["choices"][0]["finish_reason"]}],
+           "usage": last["usage"]}
+    if last.get("timings"):
+        out["timings"] = last["timings"]
+    return out
 
 
 # ------------------------------------------------------------------------------------------------ Anthropic
@@ -1040,8 +1175,12 @@ def anthropic_events(svc: Service, req: dict, ids, thinking, tools, max_new, can
                 yield close()
             stop = "tool_use" if used_tool and x["finish"] == "stop" else \
                 {"stop": "end_turn", "length": "max_tokens", "cancel": "end_turn"}[x["finish"]]
+            # the final counts, Anthropic's way: input_tokens leaves out what the conversation cache already held,
+            # which is cache_read_input_tokens (message_start could only say the whole prompt)
+            reused = min(x.get("reused") or 0, len(ids))
             yield "message_delta", {"type": "message_delta", "delta": {"stop_reason": stop, "stop_sequence": None},
-                                    "usage": {"output_tokens": x["completion_tokens"]}}
+                                    "usage": {"input_tokens": len(ids) - reused, "cache_read_input_tokens": reused,
+                                              "output_tokens": x["completion_tokens"]}}
             yield "message_stop", {"type": "message_stop"}
 
 
@@ -1068,7 +1207,7 @@ def anthropic_collect(events) -> dict:
             b["input"] = json.loads(b.pop("_json") or "{}")
         elif name == "message_delta":
             msg["stop_reason"] = e["delta"]["stop_reason"]
-            msg["usage"]["output_tokens"] = e["usage"]["output_tokens"]
+            msg["usage"].update(e["usage"])
     msg["content"] = blocks
     return msg
 
@@ -1165,13 +1304,32 @@ def make_handler(svc: Service):
                 if s.get("busy"):
                     s["elapsed_s"] = round(now - s["started"], 1)
                     if s.get("first_token"):
-                        s["tokens_per_s"] = round(s["generated"] / max(1e-6, now - s["first_token"]), 1)
+                        s["tokens_per_s"] = round(svc._tok_s(), 1)
+                        s["tokens_per_s_mean"] = round(svc._tok_s_mean(), 1)
                 for k in ("started", "first_token"):
                     s.pop(k, None)
                 self._json(200, s)
-            elif path == "/v1/models":
+            elif path in ("/v1/models", "/models"):
                 if self._authorized():
-                    self._json(200, {"object": "list", "data": [{"id": svc.model, "object": "model"}]})
+                    loaded = not hasattr(svc.engine, "alive") or svc.engine.alive()
+                    model = {"id": svc.model, "object": "model", "status": {"value": "loaded"},
+                             "meta": {"n_ctx": svc.engine.max_context},
+                             "architecture": {"input_modalities": ["text", "image"] if svc.vision is not None else ["text"],
+                                              "output_modalities": ["text"]}}
+                    self._json(200, {"object": "list", "data": [model] if loaded else []})
+            elif path == "/props":
+                if self._authorized():
+                    self._props()
+            elif path == "/slots":
+                if self._authorized():
+                    loaded = not hasattr(svc.engine, "alive") or svc.engine.alive()
+                    with svc.status_lock:
+                        busy = bool(svc.status.get("busy"))
+                    slot = {"id": 0, "n_ctx": svc.engine.max_context, "is_processing": busy}
+                    self._json(200, [slot] if loaded else [])
+            elif path == "/v1/status":
+                if self._authorized():
+                    self._json(200, svc.v1_status())
             else:
                 self._json(404, {"error": {"message": "not found"}})
 
@@ -1195,6 +1353,31 @@ def make_handler(svc: Service):
             except EngineDied as e:                          # before the answer started (not streamed)
                 self._json(503, {"error": {"type": "server_error", "message": f"{e}; the next request restarts it"}})
 
+        def _props(self):
+            model = parse_qs(urlsplit(self.path).query).get("model", [svc.model])[0]
+            if model != svc.model:
+                self._json(404, {"error": {"message": "model not found"}})
+                return
+            if hasattr(svc.engine, "alive") and not svc.engine.alive():
+                self._json(503, {"error": {"message": "the engine is not running"}})
+                return
+            defaults = {**svc.sampling_defaults, **svc.shared}
+            names = {"repetition_penalty": "repeat_penalty", "penalty_last_n": "repeat_last_n"}
+            params = {names.get(k, k): v for k, v in defaults.items()
+                      if k in ("temperature", "top_p", "top_k", "min_p", "seed", "repetition_penalty",
+                               "presence_penalty", "frequency_penalty", "penalty_last_n")}
+            params["n_predict"] = svc.shared.get("max_tokens", -1)
+            props = {"default_generation_settings": {"n_ctx": svc.engine.max_context, "params": params},
+                     "total_slots": 1, "model_alias": svc.model, "chat_template": svc.template.source,
+                     "modalities": {"vision": svc.vision is not None}, "models_autoload": False,
+                     "is_sleeping": False}
+            if getattr(svc.engine, "model_path", None):
+                props["model_path"] = svc.engine.model_path
+            version = getattr(svc.engine, "info", {}).get("version")
+            if version:
+                props["build_info"] = "Strata " + str(version)
+            self._json(200, props)
+
         def _own_page(self, what) -> bool:
             """Only JSON (a form or a "simple" cross-site request can't send it without a CORS preflight, which this
             server never grants) and no foreign Origin: a web page elsewhere must not change settings or run tools."""
@@ -1463,7 +1646,8 @@ def main() -> int:
                     help="the mock engine's answer (default: a short greeting); given more than once, requests get "
                          "them in turn and the last one repeats")
     ap.add_argument("--port", type=int, default=8095)
-    ap.add_argument("--gpu", type=int, help="the GPU to run on, as nvidia-smi numbers them (also \"gpu\" in the config)")
+    ap.add_argument("--gpu", help="the GPU to run on, as nvidia-smi numbers them, or several for a layer split "
+                                  "(\"0,2\"; also \"gpu\" in the config)")
     ap.add_argument("--tokenizer", default=str(ROOT / "pack/full/tokenizer"),
                     help="pack tokenizer directory (falls back to a byte tokenizer if absent)")
     ap.add_argument("--open", action="store_true", help="open the local page in the browser once the model is ready")
@@ -1478,7 +1662,7 @@ def main() -> int:
     a = ap.parse_args()
     cfg = json.loads(Path(a.config).read_text(encoding="utf-8-sig")) if a.config else {}   # Notepad adds a BOM
     if a.gpu is not None:
-        cfg["gpu"] = a.gpu
+        cfg["gpu"] = int(a.gpu) if a.gpu.strip().isdigit() else a.gpu
     a.host = a.host or cfg.get("host") or "127.0.0.1"   # issue #26: the run scripts pass no --host, the config can
     try:                                                # before the minutes of loading: is the port free?
         Server((a.host, a.port), BaseHTTPRequestHandler).server_close()
@@ -1515,7 +1699,9 @@ def main() -> int:
             vision = Vision(cfg["vision"], log=open(cfg["log"], "a", encoding="utf-8") if cfg.get("log") else None,
                             env=env)
         print("loading the model (the first start takes a minute or two) ...", flush=True)
-        engine = StrataEngine(cfg["exe"], cfg["args"], cwd=cfg.get("cwd"), log=cfg.get("log"), env=env)
+        if len(gpu_list(cfg)) > 1:
+            print(f"[strata] layer split across GPUs {gpu_list(cfg)} ({cfg.get('layer_split') or 'auto'})", flush=True)
+        engine = StrataEngine(cfg["exe"], engine_args(cfg), cwd=cfg.get("cwd"), log=cfg.get("log"), env=env)
         warn_tight_ram(engine.info.get("arena_mib"))
     else:
         engine, vision, sampling_defaults = MockEngine(tok, a.script or [
@@ -1527,7 +1713,8 @@ def main() -> int:
                   sampling_defaults=sampling_defaults,
                   fit_max_tokens=a.fit_max_tokens or cfg.get("fit_max_tokens") is True)
     svc.api_key = a.api_key or cfg.get("api_key", "")
-    svc.gpu_index = cfg.get("gpu") or 0                 # the Monitor reads the card the engine runs on (issue #51)
+    svc.gpu_index = (gpu_list(cfg) or [0])[0]           # the Monitor reads the card the engine runs on (issue #51)
+    svc.gpu_indices = gpu_list(cfg)                     # ... or every card of a layer split (issue #112)
     if a.config:                                        # the Chat settings shared with other apps, from last time
         svc.shared_path = str(Path(a.config).with_suffix("")) + ".shared-settings.json"
         try:

serve/telemetry.py

@@ -169,14 +169,19 @@ class _CpuRamFallback:
 
 # ------------------------------------------------------------------------------------------------ the sampler
 class Telemetry:
-    def __init__(self, extra=None, gpu_index=0):
+    def __init__(self, extra=None, gpu_index=0, gpu_indices=None):
         """`extra()` -> dict of more series to record each second (the server's tok/s).  `gpu_index`: the card the
-        engine runs on, numbered as nvidia-smi and NVML number them (by PCI bus)."""
+        engine runs on, numbered as nvidia-smi and NVML number them (by PCI bus); `gpu_indices`: all of them when
+        the model is split across several (issue #112) - the gpu_* readings are then their total (memory, power,
+        PCIe traffic), mean (load) or hottest (temperature), and "gpus" has each card's own."""
         self.extra = extra
         self.lock = threading.Lock()
         self.now: dict = {}
         self.hist = collections.defaultdict(lambda: collections.deque(maxlen=HISTORY))
-        self.gpu = _Nvml(gpu_index)
+        idx = list(gpu_indices) if gpu_indices and len(gpu_indices) > 1 else [gpu_index]
+        self.gpus = [(i, _Nvml(i)) for i in idx]
+        self.gpus = [(i, g) for i, g in self.gpus if g.ok()] or self.gpus[:1]
+        self.gpu = self.gpus[0][1]
         try:
             import psutil  # noqa: F401
             self.ps = sys.modules["psutil"]
@@ -184,7 +189,8 @@ class Telemetry:
             self.ps = None
         self.fallback = _CpuRamFallback()
         self.static = {
-            "gpu_name": self.gpu.name() if self.gpu.ok() else None,
+            "gpu_name": " + ".join(g.name() or "?" for _, g in self.gpus) if self.gpu.ok() else None,
+            "gpu_count": len(self.gpus),
             "cpu_name": _cpu_name(),
             "cores": (self.ps.cpu_count(logical=False) if self.ps else None) or None,
             "threads": os.cpu_count(),
@@ -210,7 +216,21 @@ class Telemetry:
     def sample(self):
         s = {}
         if self.gpu.ok():
-            g = self.gpu.read()
+            reads = [(i, g.read()) for i, g in self.gpus]
+            g = dict(reads[0][1])
+            if len(reads) > 1:
+                def vals(k):
+                    return [r[k] for _, r in reads if r.get(k) is not None]
+                for k in ("mem_used", "mem_total", "power", "power_limit", "pcie_rx_mb", "pcie_tx_mb"):
+                    v = vals(k)
+                    g[k] = sum(v) if v else None
+                u = vals("util")
+                g["util"] = sum(u) / len(u) if u else None
+                t = vals("temp")
+                g["temp"] = max(t) if t else None
+                s["gpus"] = [{"index": i, "util": r.get("util"), "mem_used": r.get("mem_used"),
+                              "mem_total": r.get("mem_total"), "temp": r.get("temp"), "power": r.get("power")}
+                             for i, r in reads]
             s.update({f"gpu_{k}": v for k, v in g.items()})
         if self.ps:
             try:

serve/test_detok.py

@@ -0,0 +1,107 @@
+"""The streaming detokenizer (perf-review F-1): the incremental one emits exactly what the old re-decode did, token
+by token, and its cost per token does not grow with the reply.
+
+    python -m unittest serve.test_detok      (needs a pack's tokenizer; skipped without one)
+"""
+import os
+import random
+import sys
+import time
+import unittest
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(ROOT / "tools"))
+sys.path.insert(0, str(ROOT))
+
+from serve.server import Detokenizer  # noqa: E402
+
+
+def find_tokenizer():
+    cands = [os.environ.get("STRATA_TOKENIZER", "")]
+    cands += [str(p) for p in ROOT.glob("packs/*/tokenizer")] + [str(p) for p in ROOT.glob("pack/*/tokenizer")]
+    cands += [r"C:\Users\AI-Server\Desktop\Strata\Public\Engine\pack\full\tokenizer"]
+    for c in cands:
+        if c and (Path(c) / "vocab.json").exists():
+            return Path(c)
+    return None
+
+
+class OldDetokenizer:
+    """The pre-0.1.22 algorithm: re-decode every generated id, hold a trailing U+FFFD."""
+
+    def __init__(self, tok):
+        self.tok, self.ids, self.sent = tok, [], 0
+
+    def push(self, t):
+        self.ids.append(t)
+        text = self.tok.decode(self.ids)
+        if text.endswith("\ufffd"):
+            return ""
+        delta, self.sent = text[self.sent:], len(text)
+        return delta
+
+
+@unittest.skipIf(find_tokenizer() is None, "no pack tokenizer here")
+class Detok(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        import json
+        import strata_tokenizer as ST
+        t = find_tokenizer()
+        vocab = json.loads((t / "vocab.json").read_text(encoding="utf-8"))
+        toks = [None] * len(vocab)
+        for s, i in vocab.items():
+            toks[i] = s
+        cls.tok = ST.Tokenizer(toks, (t / "merges.txt").read_text(encoding="utf-8").split("\n"),
+                               json.loads((t / "token_type.json").read_text()))
+
+    def same_stream(self, ids):
+        """The same text as the old re-decode, never behind it (the old one held complete characters too while
+        the tail was an incomplete one; the new one holds only the incomplete bytes), and never ahead of what the
+        ids so far decode to."""
+        a, b = OldDetokenizer(self.tok), Detokenizer(self.tok)
+        self.assertIsNotNone(b.inc)
+        old = new = ""
+        for i, t in enumerate(ids):
+            old += a.push(t)
+            new += b.push(t)
+            self.assertTrue(new.startswith(old), f"token {i} ({t}): behind the old stream")
+            self.assertTrue(self.tok.decode(ids[:i + 1]).startswith(new), f"token {i} ({t}): ahead of the ids")
+        if not self.tok.decode(ids).endswith("�"):
+            self.assertEqual(old, new)
+
+    def test_text_with_split_characters(self):
+        text = ("Grüße aus Zürich! 你好，世界。日本語のテキスト 🌊🦊👩‍💻 and ASCII code: def f(x): return x**2\n" * 40)
+        self.same_stream(self.tok.encode(text))
+
+    def test_random_ids_and_broken_bytes(self):
+        rng = random.Random(5)
+        n = len(self.tok.tokens)
+        for _ in range(20):   # random ids: split and invalid UTF-8 sequences everywhere (no U+FFFD token itself)
+            ids = [rng.randrange(n) for _ in range(300)]
+            ids = [t for t in ids if "\ufffd" not in self.tok.decode([t], errors="strict" if False else "replace")
+                   or len(self.tok.token_bytes(t)) < 3]
+            self.same_stream(ids)
+
+    def test_constant_cost(self):
+        ids = self.tok.encode("The quick brown fox jumps over the lazy dog. " * 2000)[:16384]
+        d = Detokenizer(self.tok)
+        for t in ids[:1024]:
+            d.push(t)
+        t0 = time.perf_counter()
+        for t in ids[1024:1124]:
+            d.push(t)
+        early = time.perf_counter() - t0
+        for t in ids[1124:16000]:
+            d.push(t)
+        t0 = time.perf_counter()
+        for t in ids[16000:16100]:
+            d.push(t)
+        late = time.perf_counter() - t0
+        self.assertLess(late / 100, 0.0002, "a token after 16K tokens costs more than 0.2 ms")
+        self.assertLess(late, early * 5 + 0.002)
+
+
+if __name__ == "__main__":
+    unittest.main()

serve/test_server.py

@@ -7,6 +7,8 @@ from __future__ import annotations
 import json
 import os
 import sys
+import threading
+import time
 import unittest
 import urllib.error
 import urllib.request
@@ -14,7 +16,7 @@ from pathlib import Path
 
 sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
 from serve.frontend import ChatTemplate  # noqa: E402
-from serve.server import CTX_SLACK, ByteTokenizer, EngineDied, MockEngine, Service, StrataEngine, serve  # noqa: E402
+from serve.server import CTX_SLACK, ByteTokenizer, EngineDied, MockEngine, Service, StrataEngine, request_timings, serve  # noqa: E402
 
 ROOT = Path(__file__).resolve().parents[1]
 CTX = 4096
@@ -368,6 +370,97 @@ class EngineDeath(unittest.TestCase):
             httpd.server_close()
 
 
+class LiveRate(unittest.TestCase):
+    """The Monitor's Speed readout: live.tok_s is a rate, and a request that never got a DONE keeps no counters.
+
+    It used to be `generated / (now - first_token)` - the mean since the first token, whose first sample is
+    1/elapsed.  Against a paced engine that reads five-digit numbers for the first instant of every answer and
+    undershoots for the first second after that.  It is now the rate over the last RATE_WINDOW_S, with the mean
+    still available as `live.tok_s_mean` for anyone who wants it."""
+
+    PACE_S = 0.02                    # 50 tokens/s: a 30-token answer takes about 0.6 s
+    TOKENS = 30
+
+    def setUp(self):
+        self.tok = ByteTokenizer()
+        self.engine = MockEngine(self.tok, "x" * self.TOKENS, max_context=CTX, delay_s=self.PACE_S)
+        self.svc = Service(self.engine, self.tok, ChatTemplate(ROOT / "serve/chat_template.jinja"))
+        self.httpd = serve(self.svc, port=0)
+        self.base = f"http://127.0.0.1:{self.httpd.server_address[1]}"
+
+    def tearDown(self):
+        self.httpd.shutdown()
+        self.httpd.server_close()
+
+    def metrics(self):
+        with urllib.request.urlopen(self.base + "/metrics", timeout=10) as r:
+            return json.loads(r.read())
+
+    def test_the_live_number_is_a_rate(self):
+        live_samples, stop = [], threading.Event()
+
+        def poll():                                   # what the Monitor polls, at 10 ms
+            while not stop.is_set():
+                live = self.metrics()["live"]
+                if live["state"] == "generating" and live["tok_s"] is not None:
+                    live_samples.append((live["generated"], live["tok_s"], live["tok_s_mean"]))
+                time.sleep(0.01)
+
+        body = json.dumps({"model": "m", "max_tokens": self.TOKENS, "temperature": 0,
+                           "messages": [{"role": "user", "content": "hi"}]}).encode()
+        watcher = threading.Thread(target=poll, daemon=True)
+        watcher.start()
+        t0 = time.time()
+        try:
+            with urllib.request.urlopen(urllib.request.Request(self.base + "/v1/chat/completions", data=body,
+                                                               headers={"Content-Type": "application/json"}),
+                                        timeout=30) as r:
+                usage = json.loads(r.read())["usage"]
+        finally:
+            stop.set()
+            watcher.join(2)
+        true_rate = usage["completion_tokens"] / (time.time() - t0)
+        self.assertGreaterEqual(len(live_samples), 3, "too few live readings to judge the readout")
+        self.assertLess(max(s for _g, s, _m in live_samples), 4 * true_rate,
+                        f"live.tok_s peaked at {max(s for _g, s, _m in live_samples):.1f} tok/s "
+                        f"for a {true_rate:.1f} tok/s engine")
+        self.assertEqual(self.metrics()["live"]["state"], "idle")
+        self.assertIsNone(self.metrics()["live"]["tok_s"])
+
+    def test_a_request_without_a_done_keeps_no_engine_counters(self):
+        """An engine that dies mid-answer: the previous request's `last` must not become this row's decode rate."""
+        class HalfDead(MockEngine):
+            last = {"generated": 99, "prompt_tokens": 9, "prompt_ms": 10.0, "decode_ms": 100.0, "finish": "stop"}
+
+            def generate(self, ids, max_new, sampling, cancel, embeddings=None):
+                for i, t in enumerate(super().generate(ids, max_new, sampling, cancel, embeddings)):
+                    if i == 3:
+                        raise EngineDied("the engine stopped unexpectedly (exit code -9)")
+                    yield t
+
+        tok = ByteTokenizer()
+        svc = Service(HalfDead(tok, "x" * self.TOKENS, max_context=CTX), tok,
+                      ChatTemplate(ROOT / "serve/chat_template.jinja"))
+        httpd = serve(svc, port=0)
+        base = f"http://127.0.0.1:{httpd.server_address[1]}"
+        try:
+            body = json.dumps({"model": "m", "max_tokens": self.TOKENS, "stream": True,
+                               "messages": [{"role": "user", "content": "hi"}]}).encode()
+            with urllib.request.urlopen(urllib.request.Request(base + "/v1/chat/completions", data=body,
+                                                               headers={"Content-Type": "application/json"}),
+                                        timeout=30) as r:
+                text = r.read().decode()
+            self.assertIn('"error"', text)
+            row = svc.metrics()["requests"][0]
+            self.assertEqual(row["finish"], "error")
+            self.assertEqual(row["output_tokens"], 3)
+            self.assertIsNone(row["decode_tok_s"], "the previous request's counters were recorded as this one's")
+            self.assertIsNone(row["engine_generated"])
+        finally:
+            httpd.shutdown()
+            httpd.server_close()
+
+
 class SharedSettings(unittest.TestCase):
     """The web app's "Use for other apps too": POST /settings makes its Chat settings every client's defaults."""
 
@@ -489,7 +582,7 @@ class WebApp(unittest.TestCase):
         code, ctype, body = self.get("/")
         self.assertEqual(code, 200)
         self.assertIn("text/html", ctype)
-        self.assertIn(b"/web/app.js", body)
+        self.assertIn(b"\"web/app.js\"", body)   # relative since #82 (works behind a path-prefixed proxy)
         for path, want in (("/web/app.js", "javascript"), ("/web/app.css", "text/css"), ("/web/tokens.css", "text/css"),
                            ("/web/components.css", "text/css"), ("/web/sprite.svg", "image/svg+xml")):
             with self.subTest(path=path):
@@ -516,6 +609,86 @@ class WebApp(unittest.TestCase):
         self.assertEqual(m["live"]["state"], "idle")
         self.assertEqual(m["requests"][0]["output_tokens"], 5)
 
+    def test_model_discovery_and_props(self):
+        svc = self.svc
+        previous = svc.engine.max_context, svc.vision, svc.sampling_defaults, svc.shared
+        try:
+            svc.engine.max_context = 262144
+            svc.sampling_defaults = {"temperature": 1.0, "repetition_penalty": 1.1}
+            svc.shared = {"temperature": 0.7, "max_tokens": 4096}
+            for vision in (None, object()):
+                svc.vision = vision
+                for path in ("/models", "/v1/models"):
+                    code, _, body = self.get(path)
+                    self.assertEqual(code, 200)
+                    models = json.loads(body)["data"]
+                    self.assertEqual(len(models), 1)
+                    model = models[0]
+                    self.assertEqual(model["id"], svc.model)
+                    self.assertEqual(model["status"]["value"], "loaded")
+                    self.assertEqual(model["meta"]["n_ctx"], 262144)
+                    self.assertEqual(model["architecture"]["input_modalities"],
+                                     ["text", "image"] if vision else ["text"])
+                code, _, body = self.get("/props?model=" + svc.model + "&autoload=false")
+                self.assertEqual(code, 200)
+                props = json.loads(body)
+                self.assertEqual(props["default_generation_settings"]["n_ctx"], 262144)
+                self.assertEqual(props["default_generation_settings"]["params"],
+                                 {"temperature": 0.7, "repeat_penalty": 1.1, "n_predict": 4096})
+                self.assertEqual(props["chat_template"], (ROOT / "serve/chat_template.jinja").read_text(encoding="utf-8"))
+                self.assertEqual(props["modalities"]["vision"], vision is not None)
+                self.assertEqual(props["total_slots"], 1)
+                self.assertFalse(props["models_autoload"])
+            svc.shared = {}
+            props = json.loads(self.get("/props")[2])
+            self.assertEqual(props["default_generation_settings"]["params"]["n_predict"], -1)
+            self.assertEqual(self.get("/props?model=not-loaded&autoload=true")[0], 404)
+        finally:
+            svc.engine.max_context, svc.vision, svc.sampling_defaults, svc.shared = previous
+
+    def test_discovery_needs_the_api_key(self):
+        self.svc.api_key = "secret"
+        try:
+            for path in ("/models", "/v1/models", "/props", "/slots"):
+                self.assertEqual(self.get(path)[0], 401)
+                self.assertEqual(self.get(path, {"Authorization": "Bearer secret"})[0], 200)
+        finally:
+            self.svc.api_key = ""
+
+    def test_build_model_path_and_slot_status(self):
+        engine = self.svc.engine
+        engine.model_path = "models/example.gguf"
+        engine.info = {"version": "0.1.21"}
+        try:
+            props = json.loads(self.get("/props")[2])
+            self.assertEqual(props["model_path"], engine.model_path)
+            self.assertEqual(props["build_info"], "Strata 0.1.21")
+            for busy in (True, False):
+                with self.svc.status_lock:
+                    self.svc.status["busy"] = busy
+                code, _, body = self.get("/slots")
+                self.assertEqual(code, 200)
+                self.assertEqual(json.loads(body), [{"id": 0, "n_ctx": CTX, "is_processing": busy}])
+        finally:
+            with self.svc.status_lock:
+                self.svc.status["busy"] = False
+            del engine.model_path, engine.info
+        props = json.loads(self.get("/props")[2])
+        self.assertNotIn("build_info", props)
+        self.assertNotIn("model_path", props)
+
+    def test_discovery_does_not_restart_a_dead_engine(self):
+        self.svc.engine.alive = lambda: False
+        try:
+            for path in ("/models", "/v1/models"):
+                code, _, body = self.get(path)
+                self.assertEqual(code, 200)
+                self.assertEqual(json.loads(body)["data"], [])
+            self.assertEqual(self.get("/props")[0], 503)
+            self.assertEqual(json.loads(self.get("/slots")[2]), [])
+        finally:
+            del self.svc.engine.alive
+
     def test_metrics_need_the_key_when_one_is_set(self):
         self.svc.api_key = "secret"
         try:
@@ -526,5 +699,119 @@ class WebApp(unittest.TestCase):
             self.svc.api_key = ""
 
 
+class ClockedEngine(MockEngine):
+    """The mock engine with StrataEngine's clock: `last` as the engine's DONE line gives it, the conversation cache
+    holding the first REUSED tokens of every prompt."""
+    REUSED = 5
+
+    def generate(self, ids, max_new, sampling, cancel, embeddings=None):
+        n = 0
+        try:
+            for t in super().generate(ids, max_new, sampling, cancel, embeddings):
+                n += 1
+                yield t
+        finally:          # as StrataEngine reads its DONE line: also when the server closes the request at a stop token
+            self.last = {"generated": n, "prompt_tokens": len(ids), "prompt_ms": 40.0, "decode_ms": 20.0 * n,
+                         "finish": "stop", "reused": min(self.REUSED, len(ids)), "hits": 9, "lookups": 10}
+
+
+class UsageAndStatus(unittest.TestCase):
+    """What clients read besides the text: the part of the prompt the conversation cache held (OpenAI's
+    prompt_tokens_details.cached_tokens, Anthropic's cache_read_input_tokens), llama.cpp's timings, GET /v1/status."""
+
+    @classmethod
+    def setUpClass(cls):
+        tok = ByteTokenizer()
+        cls.engine = ClockedEngine(tok, "</think>\n\nok", max_context=CTX)
+        cls.svc = Service(cls.engine, tok, ChatTemplate(ROOT / "serve/chat_template.jinja"))
+        cls.httpd = serve(cls.svc, port=0)
+        cls.base = f"http://127.0.0.1:{cls.httpd.server_address[1]}"
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.httpd.shutdown()
+        cls.httpd.server_close()
+
+    def request(self, path, body=None):
+        req = urllib.request.Request(self.base + path, data=None if body is None else json.dumps(body).encode(),
+                                     headers={"Content-Type": "application/json", "anthropic-version": "2023-06-01"})
+        with urllib.request.urlopen(req, timeout=30) as r:
+            return r.status, r.read()
+
+    def chat(self, path, stream=False):
+        body = {"model": "x", "max_tokens": 20, "messages": [{"role": "user", "content": "hi"}], "stream": stream}
+        status, raw = self.request(path, body)
+        self.assertEqual(status, 200)
+        if not stream:
+            return json.loads(raw)
+        return [json.loads(line[6:]) for line in raw.decode().splitlines()
+                if line.startswith("data: {")]
+
+    def test_openai(self):
+        b = self.chat("/v1/chat/completions")
+        u, t = b["usage"], b["timings"]
+        self.assertEqual(u["prompt_tokens_details"]["cached_tokens"], ClockedEngine.REUSED)
+        self.assertEqual(t["cache_n"], ClockedEngine.REUSED)
+        self.assertEqual(t["prompt_n"] + t["cache_n"], u["prompt_tokens"])
+        self.assertEqual(t["predicted_n"], u["completion_tokens"])
+        self.assertAlmostEqual(t["prompt_per_second"], t["prompt_n"] / 0.040, delta=0.1)
+        self.assertAlmostEqual(t["predicted_per_second"], 50.0, delta=0.1)            # 20 ms a token
+
+    def test_openai_stream(self):
+        last = self.chat("/v1/chat/completions", stream=True)[-1]
+        self.assertEqual(last["usage"]["prompt_tokens_details"]["cached_tokens"], ClockedEngine.REUSED)
+        self.assertEqual(last["timings"]["cache_n"], ClockedEngine.REUSED)
+
+    def test_anthropic(self):
+        u = self.chat("/v1/messages")["usage"]
+        self.assertEqual(u["cache_read_input_tokens"], ClockedEngine.REUSED)
+        self.assertEqual(u["input_tokens"] + u["cache_read_input_tokens"], len(self.engine.last_prompt))
+        self.assertGreater(u["output_tokens"], 0)
+
+    def test_v1_status(self):
+        self.chat("/v1/chat/completions")
+        status, raw = self.request("/v1/status")
+        self.assertEqual(status, 200)
+        s = json.loads(raw)
+        self.assertEqual(s["model"], self.svc.model)
+        self.assertEqual(s["context"]["max_positions"], CTX)
+        self.assertEqual(s["concurrency"]["serving"], 1)
+        self.assertFalse(s["vision"]["available"])
+        self.assertEqual(s["activity"]["in_flight"], 0)
+        self.assertGreaterEqual(s["activity"]["requests"], 1)
+        self.assertEqual(s["last_timings"]["cache_n"], ClockedEngine.REUSED)
+        self.assertIn("at", s["last_timings"])
+
+    def test_no_clock(self):
+        """An engine without a clock (MockEngine): no timings, nothing cached."""
+        tok = ByteTokenizer()
+        svc = Service(MockEngine(tok, "</think>\n\nok", max_context=CTX), tok,
+                      ChatTemplate(ROOT / "serve/chat_template.jinja"))
+        httpd = serve(svc, port=0)
+        try:
+            data = json.dumps({"model": "x", "max_tokens": 5, "messages": [{"role": "user", "content": "hi"}]}).encode()
+            req = urllib.request.Request(f"http://127.0.0.1:{httpd.server_address[1]}/v1/chat/completions", data=data,
+                                         headers={"Content-Type": "application/json"})
+            with urllib.request.urlopen(req, timeout=30) as r:
+                b = json.loads(r.read())
+            self.assertNotIn("timings", b)
+            self.assertEqual(b["usage"]["prompt_tokens_details"]["cached_tokens"], 0)
+            self.assertIsNone(svc.v1_status()["last_timings"])
+        finally:
+            httpd.shutdown()
+            httpd.server_close()
+
+
+class TimingsDrafts(unittest.TestCase):
+    """`timings` carries the speculative draft counts (PR #83's fields) only when the engine reported them."""
+
+    def test_draft_fields(self):
+        base = {"prompt_ms": 100.0, "decode_ms": 200.0, "generated": 20, "reused": 4}
+        t = request_timings(24, 20, dict(base, drafts_offered=15, drafts_accepted=11))
+        self.assertEqual((t["draft_n"], t["draft_n_accepted"]), (15, 11))
+        self.assertEqual((t["prompt_n"], t["cache_n"]), (20, 4))
+        self.assertNotIn("draft_n", request_timings(24, 20, base))
+        self.assertIsNone(request_timings(24, 20, {}))
+
 if __name__ == "__main__":
     unittest.main()

serve/web/app.js

@@ -3,7 +3,7 @@
 "use strict";
 
 const $ = (id) => document.getElementById(id);
-const SPRITE = "/web/sprite.svg";
+const SPRITE = "web/sprite.svg";
 const icon = (name, cls = "st-icon") => `<svg class="${cls}" aria-hidden="true"><use href="${SPRITE}#i-${name}"/></svg>`;
 const esc = (s) => String(s).replace(/[&<>"']/g, (c) => ({"&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;"}[c]));
 const fmt = (n, d = 0) => (n == null || Number.isNaN(n) ? "–" : Number(n).toLocaleString(undefined, {maximumFractionDigits: d, minimumFractionDigits: d}));
@@ -98,7 +98,7 @@ $("api-key").onchange = () => { store.set("apikey", $("api-key").value.trim());
 let health = {model: "strata", images: false, max_context: 0};
 async function loadHealth() {
   try {
-    health = await (await fetch("/health")).json();
+    health = await (await fetch("health")).json();
     $("attach-btn").title = health.images ? "Attach a text file or a picture (or drop it here)"
                                           : "Attach a text file (or drop it here)";
     $("chat-empty-sub").textContent = `${health.model} runs on this PC. Nothing leaves it.`;
@@ -147,7 +147,7 @@ let lastMetrics = null, metricsFailures = 0, keyWarned = false, mcpTick = 0;
 let reqShowAll = false;   // the Monitor's request table: the last 12, or every one the server keeps (issue #35)
 async function poll() {
   try {
-    const r = await fetch(reqShowAll ? "/metrics?requests=all" : "/metrics", {headers: headers()});
+    const r = await fetch(reqShowAll ? "metrics?requests=all" : "metrics", {headers: headers()});
     if (r.status === 401) {
       setPill("error", "API key needed");
       if (!keyWarned) { keyWarned = true; toast("warn", "API key needed", "This server needs a key: add it under About > Settings.", 6000); }
@@ -228,12 +228,18 @@ function renderMonitor(live, hw, st, eng, h, last, requests, totals, kept) {
   const speed = live.state === "generating" ? live.tok_s : last ? last.decode_tok_s : null;
   setMetric("speed", speed == null ? null : fmt(speed, 1), "tok/s", live.state === "generating" ? "now" : last ? "last request" : "");
   spark("sp-speed", h.tok_s);
-  setMetric("gpu", hw.gpu_util == null ? null : fmt(hw.gpu_util), "%", st.gpu_name || "");
+  // a model split across several cards (issue #112): the cards show their total / mean / hottest, and each card's own
+  const per = (f) => (hw.gpus || []).map((g) => `GPU ${g.index} ${f(g)}`).join(" · ");
+  const multi = (hw.gpus || []).length > 1;
+  setMetric("gpu", hw.gpu_util == null ? null : fmt(hw.gpu_util), "%",
+            multi ? per((g) => (g.util == null ? "–" : `${fmt(g.util)}%`)) : st.gpu_name || "");
   spark("sp-gpu", h.gpu_util, 100);
   setMetric("vram", hw.gpu_mem_used == null ? null : gb(hw.gpu_mem_used), hw.gpu_mem_total ? `/ ${gb(hw.gpu_mem_total, 0)} GB` : "GB",
-            eng.expert_slots ? `${fmt(eng.expert_slots)} experts cached` : "");
+            multi ? per((g) => (g.mem_used == null ? "–" : `${gb(g.mem_used)} GB`))
+                  : eng.expert_slots ? `${fmt(eng.expert_slots)} experts cached` : "");
   spark("sp-vram", h.gpu_mem_used, hw.gpu_mem_total);
-  setMetric("temp", hw.gpu_temp == null ? null : fmt(hw.gpu_temp), "°C", "");
+  setMetric("temp", hw.gpu_temp == null ? null : fmt(hw.gpu_temp), "°C",
+            multi ? per((g) => (g.temp == null ? "–" : `${fmt(g.temp)}°`)) : "");
   spark("sp-temp", h.gpu_temp, 90);
   setMetric("power", hw.gpu_power == null ? null : fmt(hw.gpu_power), "W", hw.gpu_power_limit ? `of ${fmt(hw.gpu_power_limit)} W limit` : "");
   spark("sp-power", h.gpu_power, hw.gpu_power_limit);
@@ -347,7 +353,7 @@ $("req-all").addEventListener("click", () => { reqShowAll = !reqShowAll; if (las
 let mcpInfo = {servers: [], tools: 0}, mcpRetry = null;
 async function loadMcp() {
   try {
-    const r = await fetch("/mcp", {headers: headers()});
+    const r = await fetch("mcp", {headers: headers()});
     if (!r.ok) return;
     mcpInfo = await r.json();
   } catch (e) { return; /* an older server: no MCP */ }
@@ -679,7 +685,7 @@ async function send() {
   let firstAt = null, thinkStart = null, usage = null, frame = 0;
   const paint = () => { frame = 0; updateAssistant(el, m, true); scrollDown(); };
   try {
-    const r = await fetch("/v1/chat/completions", {method: "POST", headers: headers(true), body: JSON.stringify(body),
+    const r = await fetch("v1/chat/completions", {method: "POST", headers: headers(true), body: JSON.stringify(body),
                                                    signal: controller.signal});
     if (!r.ok) {
       let msg = `HTTP ${r.status}`;
@@ -881,7 +887,7 @@ function loadDrawer(s = settings) {
 let sharedOn = false;
 async function loadShared() {
   try {
-    const r = await fetch("/settings", {headers: headers()});
+    const r = await fetch("settings", {headers: headers()});
     if (r.ok) sharedOn = !!(await r.json()).shared;
   } catch (e) { /* an older server: the switch just stays off */ }
   $("s-share").setAttribute("aria-checked", String(sharedOn));
@@ -895,7 +901,7 @@ function sharedDefaults(s) {
   return d;
 }
 async function saveShared(on, s) {
-  const r = await fetch("/settings", {method: "POST", headers: headers(true),
+  const r = await fetch("settings", {method: "POST", headers: headers(true),
                                       body: JSON.stringify({defaults: on ? sharedDefaults(s) : null})});
   if (!r.ok) {
     let msg = `HTTP ${r.status}`;

serve/web/index.html

@@ -4,9 +4,9 @@
 <meta charset="utf-8">
 <meta name="viewport" content="width=device-width, initial-scale=1">
 <title>Strata</title>
-<link rel="stylesheet" href="/web/tokens.css">
-<link rel="stylesheet" href="/web/components.css">
-<link rel="stylesheet" href="/web/app.css">
+<link rel="stylesheet" href="web/tokens.css">
+<link rel="stylesheet" href="web/components.css">
+<link rel="stylesheet" href="web/app.css">
 <script>
   // the theme before the first paint: the saved choice, else the system's
   try {
@@ -21,17 +21,17 @@
   <span class="st-header__brand">Strata</span>
   <nav class="st-tabs" role="tablist" aria-label="Views">
     <button class="st-tab" role="tab" id="tab-btn-chat" aria-controls="view-chat" aria-selected="true" data-tab="chat">
-      <svg class="st-icon st-icon--sm"><use href="/web/sprite.svg#i-chat"/></svg>Chat</button>
+      <svg class="st-icon st-icon--sm"><use href="web/sprite.svg#i-chat"/></svg>Chat</button>
     <button class="st-tab" role="tab" id="tab-btn-monitor" aria-controls="view-monitor" aria-selected="false" data-tab="monitor">
-      <svg class="st-icon st-icon--sm"><use href="/web/sprite.svg#i-activity"/></svg>Monitor</button>
+      <svg class="st-icon st-icon--sm"><use href="web/sprite.svg#i-activity"/></svg>Monitor</button>
     <button class="st-tab" role="tab" id="tab-btn-about" aria-controls="view-about" aria-selected="false" data-tab="about">
-      <svg class="st-icon st-icon--sm"><use href="/web/sprite.svg#i-info"/></svg>About</button>
+      <svg class="st-icon st-icon--sm"><use href="web/sprite.svg#i-info"/></svg>About</button>
   </nav>
   <span class="st-header__spacer"></span>
   <span class="st-pill" id="pill" data-state="idle" role="status" aria-live="polite">
     <span class="st-pill__dot"></span><span id="pill-text">Connecting&hellip;</span></span>
   <button class="st-btn st-btn--icon" id="theme-btn" title="Light / dark" aria-label="Switch between light and dark">
-    <svg class="st-icon"><use id="theme-icon" href="/web/sprite.svg#i-moon"/></svg></button>
+    <svg class="st-icon"><use id="theme-icon" href="web/sprite.svg#i-moon"/></svg></button>
 </header>
 
 <main>
@@ -51,20 +51,20 @@
         <textarea id="input" rows="2" placeholder="Ask anything&hellip;" aria-label="Message"></textarea>
         <div class="st-composer__bar">
           <button type="button" class="st-btn st-btn--icon" id="attach-btn" title="Attach a text file (or drop it here)" aria-label="Attach a file">
-            <svg class="st-icon"><use href="/web/sprite.svg#i-attach"/></svg></button>
+            <svg class="st-icon"><use href="web/sprite.svg#i-attach"/></svg></button>
           <input type="file" id="file" multiple hidden>
           <button type="button" class="st-btn st-btn--icon" id="new-btn" title="New chat" aria-label="New chat">
-            <svg class="st-icon"><use href="/web/sprite.svg#i-new-chat"/></svg></button>
+            <svg class="st-icon"><use href="web/sprite.svg#i-new-chat"/></svg></button>
           <button type="button" class="st-btn st-btn--icon" id="export-btn" title="Save this chat as Markdown" aria-label="Save this chat">
-            <svg class="st-icon"><use href="/web/sprite.svg#i-download"/></svg></button>
+            <svg class="st-icon"><use href="web/sprite.svg#i-download"/></svg></button>
           <button type="button" class="st-btn st-btn--icon" id="sampling-btn" title="Sampling and thinking" aria-label="Sampling and thinking">
-            <svg class="st-icon"><use href="/web/sprite.svg#i-settings"/></svg></button>
+            <svg class="st-icon"><use href="web/sprite.svg#i-settings"/></svg></button>
           <span class="spacer"></span>
           <span class="composer-hint" id="composer-hint"></span>
           <button type="button" class="st-btn st-btn--secondary" id="stop-btn" hidden>
-            <svg class="st-icon st-icon--sm"><use href="/web/sprite.svg#i-stop"/></svg>Stop</button>
+            <svg class="st-icon st-icon--sm"><use href="web/sprite.svg#i-stop"/></svg>Stop</button>
           <button type="submit" class="st-btn st-btn--primary st-btn--send" id="send-btn" aria-label="Send">
-            <svg class="st-icon"><use href="/web/sprite.svg#i-send"/></svg></button>
+            <svg class="st-icon"><use href="web/sprite.svg#i-send"/></svg></button>
         </div>
       </form>
     </div>
@@ -169,7 +169,7 @@
 <aside class="st-drawer" id="drawer" data-open="false" aria-label="Sampling and thinking" aria-hidden="true">
   <div class="st-drawer__head"><span class="card-title">Sampling</span>
     <button class="st-btn st-btn--icon" id="drawer-close" aria-label="Close">
-      <svg class="st-icon"><use href="/web/sprite.svg#i-chevron"/></svg></button></div>
+      <svg class="st-icon"><use href="web/sprite.svg#i-chevron"/></svg></button></div>
   <div class="st-drawer__body">
     <div class="st-field">
       <label class="st-label" for="s-thinking">Thinking <output id="o-thinking"></output></label>
@@ -218,6 +218,6 @@
 </aside>
 <div class="scrim" id="scrim" hidden></div>
 <div class="st-toasts" id="toasts" aria-live="polite"></div>
-<script src="/web/app.js"></script>
+<script src="web/app.js"></script>
 </body>
 </html>

serve/web/tokens.css

@@ -3,8 +3,8 @@
    Values marked (derived) are not in the Gainsty source and were added for the app. */
 
 /* Outfit (SIL Open Font License 1.1, serve/web/fonts/OFL.txt): one variable font, weights 100-900 */
-@font-face{font-family:"Outfit";src:url("/fonts/outfit-latin-wght.woff2") format("woff2-variations");font-weight:100 900;font-display:swap;unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}
-@font-face{font-family:"Outfit";src:url("/fonts/outfit-latin-ext-wght.woff2") format("woff2-variations");font-weight:100 900;font-display:swap;unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}
+@font-face{font-family:"Outfit";src:url("../fonts/outfit-latin-wght.woff2") format("woff2-variations");font-weight:100 900;font-display:swap;unicode-range:U+0000-00FF,U+0131,U+0152-0153,U+02BB-02BC,U+02C6,U+02DA,U+02DC,U+0304,U+0308,U+0329,U+2000-206F,U+20AC,U+2122,U+2191,U+2193,U+2212,U+2215,U+FEFF,U+FFFD}
+@font-face{font-family:"Outfit";src:url("../fonts/outfit-latin-ext-wght.woff2") format("woff2-variations");font-weight:100 900;font-display:swap;unicode-range:U+0100-02BA,U+02BD-02C5,U+02C7-02CC,U+02CE-02D7,U+02DD-02FF,U+0304,U+0308,U+0329,U+1D00-1DBF,U+1E00-1E9F,U+1EF2-1EFF,U+2020,U+20A0-20AB,U+20AD-20C0,U+2113,U+2C60-2C7F,U+A720-A7FF}
 
 :root{
   /* type */

setup.py

@@ -36,6 +36,7 @@ import os
 import platform
 import re
 import shutil
+import struct
 import subprocess
 import sys
 import time
@@ -57,7 +58,7 @@ PREBUILT_ASSET = "strata-windows-x64.zip" if WIN else "strata-linux-x64.zip"
 # the CUDA libraries the ready-made engine loads (the same CUDA 13.0 it is built with), from NVIDIA's pip packages
 CUDA_WHEELS = ["nvidia-cublas==13.0.2.14", "nvidia-cuda-runtime==13.0.96"]
 MIN_DRIVER = 580                       # CUDA 13.0
-MIN_ENGINE = (0, 1, 20)                # v0.1.20: system-prompt checkpoint, PCIe probe, hit rate; v0.1.19: penalties
+MIN_ENGINE = (0, 1, 23)                # v0.1.23: image requests honor sampling, 8 GB cards start, batched verify window; v0.1.22: faster prompts (tensor-core attention), multi-GPU across images/steering/KV streaming; v0.1.21: multi-GPU layer split (--gpus); v0.1.20: system-prompt checkpoint, PCIe probe, hit rate; v0.1.19: penalties
 PY_PACKAGES = ["numpy", "jinja2", "regex", "pyyaml", "tqdm", "requests", "cmake", "ninja", "pillow", "psutil"]
 
 MODELS = {
@@ -274,6 +275,162 @@ def gpus():
 
 
 GPU_PICK = None                                         # --gpu N (issue #51); None: the card with the most VRAM
+SPLIT_MIN_VRAM_GB = 8                                   # a card sharing a model holds the dense weights and its own
+                                                        # prompt buffers too (docs/MULTI_GPU.md)
+
+
+def cc(g) -> str:
+    return f"{g['arch'][:-1]}.{g['arch'][-1]}"
+
+
+def gpu_problem(g, together=False):
+    """Why Strata cannot use this card, in plain words (None: it can)."""
+    if int(g["arch"]) < 80:
+        return (f"not supported - older than the RTX 30 series (compute capability {cc(g)}; Strata needs 8.0 or "
+                "newer)")
+    if together and g["vram_gb"] < SPLIT_MIN_VRAM_GB - 0.5:
+        return (f"not supported together with other GPUs - {g['vram_gb']:.0f} GB of VRAM (a card sharing the model "
+                f"needs {SPLIT_MIN_VRAM_GB} GB or more)")
+    return None
+
+
+def gpu_rank(g):
+    """The order cards share a model in: the newest generation first (it gets the first layers and most of the
+    work), then the most VRAM."""
+    return (-int(g["arch"]), -round(g["vram_gb"]), g["index"])
+
+
+def gpu_name(g) -> str:
+    return f"GPU {g['index']} ({g['name']}, {g['vram_gb']:.0f} GB)"
+
+
+def gpu_table(found) -> None:
+    say("  Your NVIDIA GPUs:")
+    for g in found:
+        p = gpu_problem(g)
+        say(f"    GPU {g['index']}: {g['name']}, {g['vram_gb']:.0f} GB VRAM - " + ("can be used" if p is None else p))
+
+
+def together_ok(found) -> list:
+    """The cards that can share one model, in the order they would (empty if fewer than two)."""
+    ok_ = sorted([g for g in found if gpu_problem(g, together=True) is None], key=gpu_rank)
+    return ok_ if len(ok_) >= 2 else []
+
+
+def parse_gpus(text, found) -> list:
+    """--gpus / --gpu with several: "0,2" or "all" (every card that can share the model)."""
+    if str(text).strip().lower() == "all":
+        sel = [g["index"] for g in together_ok(found)]
+        if not sel:
+            gpu_table(found)
+            fail("--gpus all: this PC does not have two GPUs Strata can use together")
+        return sel
+    try:
+        sel = [int(x) for x in str(text).split(",") if x.strip()]
+    except ValueError:
+        fail(f"--gpus takes GPU numbers as nvidia-smi numbers them, e.g. --gpus 0,2 (or --gpus all), not {text!r}")
+    if len(sel) < 2 or len(set(sel)) != len(sel):
+        fail("--gpus takes two or more different GPUs, e.g. --gpus 0,2 (one GPU: --gpu 0)")
+    return sel
+
+
+def check_gpus(sel, found, what="") -> None:
+    """Stops with a plain message when a chosen card is missing or cannot be used, and says what can."""
+    together = len(sel) > 1
+    for i in sel:
+        g = next((x for x in found if x["index"] == i), None)
+        p = "not found on this PC" if g is None else gpu_problem(g, together)
+        if p is None:
+            continue
+        say()
+        gpu_table(found)
+        can = together_ok(found)
+        single = [x for x in found if gpu_problem(x) is None]
+        ones = " or ".join(f"--gpu {x['index']}" for x in single)
+        both = "--gpus " + ",".join(str(x["index"]) for x in can) if can else ""
+        hint = ((f"use these together: {both}" + (f" (or one card: {ones})" if not together else "")) if can else
+                f"use one card: {ones}" if single else "Strata needs an NVIDIA RTX 30 series or newer card")
+        fail(f"GPU {i}{'' if g is None else ' (' + g['name'] + ')'} {what}cannot be used: {p}", hint)
+
+
+def engine_archs():
+    """The GPU generations the installed engine has code for: (archs, ptx), or None when there is none."""
+    info = ROOT / "engine" / "BUILD.json"
+    try:
+        meta = json.loads(info.read_text())
+    except (OSError, ValueError):
+        return None
+    return [int(x) for x in meta.get("archs", [])], bool(meta.get("ptx"))
+
+
+def engine_runs_on(g) -> bool:
+    ea = engine_archs()
+    if ea is None or not ea[0]:
+        return True
+    archs, ptx = ea
+    return int(g["arch"]) in archs or (ptx and int(g["arch"]) > max(archs))
+
+
+def choose_gpus(a, found) -> list:
+    """Which cards this install uses: --gpus / --gpu, or asked when two or more can share the model (the two best
+    together recommended), else the supported card with the most VRAM.  Returns their numbers, the main one first."""
+    if a.gpus:
+        sel = parse_gpus(a.gpus, found)
+        check_gpus(sel, found)
+        return sel
+    if a.gpu is not None:
+        check_gpus([a.gpu], found)
+        return [a.gpu]
+    single = sorted([g for g in found if gpu_problem(g) is None], key=lambda x: (-round(x["vram_gb"]), x["index"]))
+    if not single:
+        gpu_table(found)
+        fail("none of your GPUs can run Strata", "it needs an NVIDIA RTX 30 series or newer (compute capability 8.0+)")
+    can = together_ok(found)
+    if not can:
+        return [single[0]["index"]]
+    say()
+    say(f"  Strata can run the model on one GPU, or share it across {'these' if len(can) > 2 else 'both'}: then each"
+        " card holds the")
+    say("  experts of its own layers, so together they hold about twice as many, and prompts are read about 20%")
+    say("  faster (details: docs/MULTI_GPU.md). A much slower extra card can also make it slower.")
+    opts = [can[:2]] + ([can] if len(can) > 2 else []) + [[g] for g in single]
+    for i, o in enumerate(opts, 1):
+        label = (" + ".join(gpu_name(g) for g in o) + " together") if len(o) > 1 else gpu_name(o[0]) + " only"
+        say(f"  {i}) {label}" + ("   (recommended)" if i == 1 else ""))
+    for g in found:
+        if gpu_problem(g, together=True) is not None:
+            say(f"     (GPU {g['index']}, {g['name']}: {gpu_problem(g, together=True)})")
+    pick = opts[int(ask("Which GPUs?", [str(i) for i in range(1, len(opts) + 1)], "1", a.yes or a.check)) - 1]
+    return [g["index"] for g in pick]
+
+
+def offer_together(cfg_path: Path, cfg: dict, yes: bool) -> dict:
+    """Starting a model set up for one card on a PC with two or more that can share it: asked once (the answer is
+    saved in its config)."""
+    if isinstance(cfg.get("gpu"), list) or cfg.get("gpus_asked"):
+        return cfg
+    found = gpus()
+    can = together_ok(found)
+    if not can:
+        return cfg
+    pair = can[:2]
+    cfg["gpus_asked"] = True
+    say()
+    say("  This PC has " + " and ".join(gpu_name(g) for g in pair) + ": Strata can share the model across both.")
+    say("  Together they hold about twice the model's experts and read prompts about 20% faster (docs/MULTI_GPU.md).")
+    missing = [g for g in pair if not engine_runs_on(g)]
+    if missing:
+        say("  The installed engine has no code for " + ", ".join(g["name"] for g in missing) + ": to use them "
+            "together, run START-HERE.bat --setup --gpus " + ",".join(str(g["index"]) for g in pair))
+    elif ask("  Use both from now on? (you can change it later: START-HERE.bat --gpu N for one card)",
+             ["y", "n"], "y", yes) == "y":
+        cfg["gpu"] = [g["index"] for g in pair]
+        cfg["layer_split"] = cfg.get("layer_split") or "auto"
+        ok("from now on this model runs on " + " + ".join(gpu_name(g) for g in pair))
+    else:
+        ok("staying on one GPU (START-HERE.bat --gpus " + ",".join(str(g["index"]) for g in pair) + " switches)")
+    cfg_path.write_text(json.dumps(cfg, indent=1), encoding="utf-8")
+    return cfg
 
 
 def gpu_info(pick=None):
@@ -316,7 +473,9 @@ def find_vcvars():
     vswhere = Path(os.environ.get("ProgramFiles(x86)", r"C:\Program Files (x86)")) / "Microsoft Visual Studio/Installer/vswhere.exe"
     if not vswhere.exists():
         return None
-    p = out([str(vswhere), "-latest", "-products", "*", "-requires",
+    # CUDA 13 accepts Visual Studio 2019 and 2022 only: a newer one (2026 = version 18) installed next to them
+    # must not be picked ("unsupported Microsoft Visual Studio version"); with only a newer one there is none
+    p = out([str(vswhere), "-latest", "-products", "*", "-version", "[16.0,18.0)", "-requires",
              "Microsoft.VisualStudio.Component.VC.Tools.x86.x64", "-property", "installationPath"]).strip()
     v = Path(p) / "VC/Auxiliary/Build/vcvars64.bat" if p else None
     return v if v and v.exists() else None
@@ -400,12 +559,34 @@ def download(url, dst: Path, what=None):
             warn(f"download interrupted ({e}); retrying in 10 s ...")
             time.sleep(10)
     if total and part.stat().st_size != total:
-        fail(f"could not finish downloading {dst.name}", "check your internet connection and run it again")
+        fail(f"could not finish downloading {dst.name}: {part.stat().st_size:,} bytes on disk, the server says {total:,}",
+             "check your internet connection and run it again (the download resumes where it stopped)")
     part.replace(dst)
     mark(dst)
     ok(f"{what or dst.name} downloaded")
 
 
+def check_shards(shards):
+    """Every shard present and whole, or setup stops naming the file and the numbers.  Whole means as long as
+    its own tensor directory says (the header is read, the data is not): a truncated copy (--gguf-dir, a .part
+    renamed by hand, a download finished by an older setup) otherwise passes as a model file and the engine
+    fails much later, at the first tensor that runs past the end."""
+    sys.path.insert(0, str(ROOT / "tools"))
+    from gguf_reader import GGUFFile
+    for s in shards:
+        if not s.exists():
+            fail(f"missing {s}")
+        try:
+            g = GGUFFile(s)
+        except (ValueError, struct.error) as e:
+            fail(f"{s.name} is not a whole GGUF shard ({e})", "delete it and run setup again")
+        need = g.data_start + max((t.offset + (t.expected_bytes() or 0) for t in g.tensors), default=0)
+        have = s.stat().st_size
+        if have < need:
+            fail(f"{s.name} is short: {have:,} of {need:,} bytes ({need - have:,} missing)",
+                 "delete it and run setup again (or copy the whole file into --gguf-dir)")
+
+
 def get_llama_cpp():
     """llama.cpp at the pinned commit (ggml for the build, gguf-py for the tools, mtmd for images), as a zip: no git."""
     llama = ROOT / "third_party" / "llama.cpp"
@@ -512,9 +693,11 @@ def get_prebuilt(url_base, gpu, vision, updating=False) -> Path | None:
         shutil.rmtree(tmp, ignore_errors=True)
         return None
     archs = [int(a) for a in meta.get("archs", [])]
-    arch = int(gpu["arch"])
-    if arch not in archs and not (meta.get("ptx") and arch > max(archs)):
-        warn(f"the ready-made engine is built for {', '.join(str(a) for a in archs)}; your GPU is {arch}" + ("" if updating else ": compiling instead"))
+    miss = [int(x) for x in gpu.get("archs", [gpu["arch"]])
+            if int(x) not in archs and not (meta.get("ptx") and int(x) > max(archs))]
+    if miss:
+        warn(f"the ready-made engine is built for {', '.join(str(a) for a in archs)}; your GPU is "
+             f"{', '.join(str(x) for x in miss)}" + ("" if updating else ": compiling instead"))
         shutil.rmtree(tmp, ignore_errors=True)
         return None
     for p in tmp.iterdir():
@@ -568,6 +751,7 @@ def update_installed_engine(url_base) -> None:
         try:                                           # a failed compile must not stop the model from starting
             if gpu is None:
                 raise RuntimeError("no NVIDIA GPU found")
+            gpu = {**gpu, "archs": sorted({int(gpu["arch"]), *(int(x) for x in meta.get("archs", []))})}
             build_engine(gpu, vision, False, get_llama_cpp())
         except (Exception, SystemExit) as e:
             warn(f"could not compile the updated engine{'' if isinstance(e, SystemExit) else f' ({e})'}: starting the installed one")
@@ -589,7 +773,7 @@ def update_installed_engine(url_base) -> None:
 def install_build_tools(gpu, yes):
     """The compiler and the CUDA toolkit, installed for the user (asks once).  Returns (nvcc, vcvars)."""
     nvcc, cuda_v = find_nvcc()
-    need_cuda = (12, 8) if int(gpu["arch"]) >= 120 else (12, 0)
+    need_cuda = (12, 8) if max(int(x) for x in gpu.get("archs", [gpu["arch"]])) >= 120 else (12, 0)
     vcvars = find_vcvars() if WIN else None
     have_cc = vcvars is not None if WIN else shutil.which("g++") is not None
     missing = []
@@ -701,23 +885,25 @@ def build_engine(gpu, vision, yes, llama) -> Path:
         ok("engine already built for this PC")
         return eng
     nvcc, vcvars = install_build_tools(gpu, yes)
+    archs = sorted({int(x) for x in gpu.get("archs", [gpu["arch"]])})    # every card the model runs on
+    cuda_archs = ";".join(str(x) for x in archs)
     if not engine_ok:
         say("  The engine's source changed: compiling it again (only what changed, a few minutes) ..."
             if local and (eng / EXE).exists() else "  Compiling the Strata engine for your GPU (10-20 minutes, once) ...")
         cmake_build(ROOT, ROOT / "build", "strata",
-                    ["-DSTRATA_ENABLE_CUDA=ON", "-DSTRATA_BUILD_TESTS=OFF", f"-DCMAKE_CUDA_ARCHITECTURES={gpu['arch']}",
+                    ["-DSTRATA_ENABLE_CUDA=ON", "-DSTRATA_BUILD_TESTS=OFF", f"-DCMAKE_CUDA_ARCHITECTURES={cuda_archs}",
                      f"-DCMAKE_CUDA_COMPILER={nvcc}", f"-DSTRATA_GGML_DIR={llama}"], vcvars, "build-strata.bat")
         shutil.copy2(ROOT / "build" / EXE, eng / EXE)
     if not vision_ok:
         say("  Compiling the image encoder" + (" with CUDA (10-20 minutes, once) ..." if vision == "gpu" else " ..."))
         defs = [f"-DLLAMA_DIR={llama}", f"-DSTRATA_VISION_CUDA={'ON' if vision == 'gpu' else 'OFF'}"]
         if vision == "gpu":
-            defs += [f"-DCMAKE_CUDA_ARCHITECTURES={gpu['arch']}", f"-DCMAKE_CUDA_COMPILER={nvcc}"]
+            defs += [f"-DCMAKE_CUDA_ARCHITECTURES={cuda_archs}", f"-DCMAKE_CUDA_COMPILER={nvcc}"]
         cmake_build(ROOT / "tools" / "vision", ROOT / "build-vision", "strata-vision", defs, vcvars, "build-vision.bat")
         shutil.copy2(ROOT / "build-vision" / "bin" / VEXE, eng / VEXE)
     bindir = Path(nvcc).parent                            # the toolkit's own libraries (bin, bin/x64, lib64)
     dirs = [str(d) for d in (bindir, bindir / "x64", bindir.parent / "lib64") if d.is_dir()]
-    stamp.write_text(json.dumps({"source": "local", "version": source_version(), "archs": [int(gpu["arch"])],
+    stamp.write_text(json.dumps({"source": "local", "version": source_version(), "archs": archs,
                                  "vision": vision,
                                  "cuda_dirs": dirs, "src": src, "vision_src": vsrc if want_vision else None}, indent=1))
     ok(f"engine compiled: {eng / EXE}")
@@ -910,7 +1096,8 @@ def choices_from_config(cfg_path: Path) -> dict:
             "kv": val("--kv") if val("--kv") in ("int8", "q4_0") else None,
             "vision": ("gpu" if vis.get("gpu") else "cpu") if isinstance(vis, dict) else "none",
             "esp": ("on" if Path(esp_path).name == ESP_VECTOR.name else esp_path) if esp_path else "off",
-            "host": cfg.get("host"), "api_key": cfg.get("api_key"), "port": cfg.get("port"), "gpu": cfg.get("gpu")}
+            "host": cfg.get("host"), "api_key": cfg.get("api_key"), "port": cfg.get("port"), "gpu": cfg.get("gpu"),
+            "layer_split": cfg.get("layer_split")}
 
 
 def find_in(roots: list, rel: str):
@@ -957,7 +1144,9 @@ def is_wsl() -> bool:
 def hardware_key(cfg: dict) -> str:
     """What a calibration is valid for: this GPU, CPU and RAM, and the model with its context and images setting
     (the context's KV cache and the image encoder take VRAM from the expert cache)."""
-    g = gpu_info(cfg.get("gpu")) or {}
+    sel = cfg.get("gpu")
+    gl = [gpu_info(i) or {} for i in sel] if isinstance(sel, list) else [gpu_info(sel) or {}]
+    g = {"name": " + ".join(x.get("name", "?") for x in gl), "vram_gb": sum(x.get("vram_gb", 0) for x in gl)}
     a = cfg.get("args", [])
     ctx = a[a.index("--max-context") + 1] if "--max-context" in a else "?"
     return "|".join([g.get("name", "?"), f"{g.get('vram_gb', 0):.0f}GB", cpu_info()[0], f"{ram_gb():.0f}GB",
@@ -1023,7 +1212,8 @@ def upgrade_config(cfg_path: Path, cfg: dict) -> dict:
     return cfg
 
 
-def start(cfg_path: Path, port: int | None, gpu: int | None = None, open_browser=True) -> int:
+def start(cfg_path: Path, port: int | None, gpu: int | list | None = None, open_browser=True, yes=False,
+          layer_split=None) -> int:
     cfg = upgrade_config(cfg_path, json.loads(cfg_path.read_text(encoding="utf-8-sig")))
     missing = [p for p in [cfg["exe"], *[a for a in cfg["args"] if a.endswith(".gguf")]] if not Path(p).exists()]
     if missing:
@@ -1031,9 +1221,28 @@ def start(cfg_path: Path, port: int | None, gpu: int | None = None, open_browser
     cfg_path.touch()                                     # the most recently used model
     cmd = [sys.executable, str(ROOT / "serve" / "server.py"), "--engine", "strata", "--config", str(cfg_path),
            "--port", str(port or cfg.get("port", 8080))]
-    if gpu is not None:
-        gpu_info(gpu)                                  # stops with the list of GPUs if there is no such one
+    found = gpus()
+    if isinstance(gpu, list):                          # --gpus: saved, this model runs on these cards from now on
+        check_gpus(gpu, found)
+        cfg["gpu"], cfg["gpus_asked"] = gpu, True
+        cfg["layer_split"] = layer_split or cfg.get("layer_split") or "auto"
+        cfg_path.write_text(json.dumps(cfg, indent=1), encoding="utf-8")
+        gpu = None
+    elif gpu is not None:                              # --gpu N: this start only, on that card
+        check_gpus([gpu], found)
         cmd += ["--gpu", str(gpu)]
+    else:
+        cfg = offer_together(cfg_path, cfg, yes)
+    use = gpu if gpu is not None else cfg.get("gpu")
+    if isinstance(use, list):
+        check_gpus(use, found, "(chosen for this model) ")
+        byid = {g["index"]: g for g in found}
+        ok("GPUs: " + " + ".join(gpu_name(byid[i]) for i in use) + f" together (layers split {cfg.get('layer_split') or 'auto'})")
+    elif found:
+        g = next((x for x in found if x["index"] == use), None) if use is not None else max(
+            found, key=lambda x: (round(x["vram_gb"]), -x["index"]))
+        if g is not None:
+            ok("GPU: " + gpu_name(g))
     if open_browser:
         cmd.append("--open")
     gb = 0.0
@@ -1083,8 +1292,13 @@ def main() -> int:
                     help="EXPERIMENTAL, off by default: the control vector in data/experimental-speed-projection "
                          "(or another GGUF) as a projection on layers 4-44; see docs/DETAILS.md")
     ap.add_argument("--port", type=int, help="the server's port (default: the one the install was set up with, 8080 for a new one)")
-    ap.add_argument("--gpu", type=int, help="the GPU to use, numbered as nvidia-smi numbers them (default: the one with the "
-                                            "most VRAM; with --setup it is saved, when starting it overrides the saved one)")
+    ap.add_argument("--gpu", help="one GPU, numbered as nvidia-smi numbers them (default: asked when several can be "
+                                  "used; with --setup it is saved, when starting it is for that start only)")
+    ap.add_argument("--gpus", help="several GPUs sharing one model, as nvidia-smi numbers them: \"0,2\", or \"all\" "
+                                   "(every card that can); the first is the main one. Saved, also when starting "
+                                   "(see docs/MULTI_GPU.md)")
+    ap.add_argument("--layer-split", help="with --gpus: where each later GPU's layers start (\"18\", \"16,32\"); "
+                                          "default auto, placed from each GPU's free VRAM")
     ap.add_argument("--host", help="where the server listens: 127.0.0.1 = this PC only (default), 0.0.0.0 = also other "
                                    "devices on your network (issue #26; set --api-key too)")
     ap.add_argument("--api-key", help="require this key from clients (recommended with --host 0.0.0.0)")
@@ -1103,6 +1317,13 @@ def main() -> int:
                     help="tune the engine's settings for this PC (about 5-10 minutes), then start the model")
     ap.add_argument("--skip-build", action="store_true", help=argparse.SUPPRESS)
     a = ap.parse_args()
+    if a.gpu is not None:                              # --gpu 0,2 means --gpus 0,2 (a user tried it: issue report)
+        if "," in a.gpu:
+            a.gpus, a.gpu = a.gpus or a.gpu, None
+        elif a.gpu.strip().isdigit():
+            a.gpu = int(a.gpu)
+        else:
+            ap.error(f"--gpu takes a GPU number as nvidia-smi numbers them, e.g. --gpu 1 (or --gpus 0,2), not {a.gpu!r}")
     say("Strata - Qwen3.8-Flash-Next on a normal PC (NVIDIA GPU + system RAM + CPU)")
     data, elsewhere = data_folder(a.data_dir)          # the model files: in the data folder, found from any copy
     roots = [data, *elsewhere]
@@ -1125,10 +1346,17 @@ def main() -> int:
                 a.experimental_speed_projection = a.experimental_speed_projection or ch["esp"]
                 a.host, a.api_key = a.host or ch["host"], a.api_key or ch["api_key"]
                 a.port = a.port or ch["port"]
-                a.gpu = a.gpu if a.gpu is not None else ch.get("gpu")
+                if isinstance(ch.get("gpu"), list):     # a layer split: set up across the same cards again
+                    a.gpus = a.gpus or ",".join(str(g) for g in ch["gpu"])
+                    a.layer_split = a.layer_split or ch.get("layer_split")
+                else:
+                    a.gpu = a.gpu if a.gpu is not None else ch.get("gpu")
                 a.yes = True
     global GPU_PICK
-    GPU_PICK = a.gpu
+    # starting an installed model: --gpus 0,2 (or all) saves those cards for it and starts on them (it used to start
+    # on the first one alone unless given with --setup), --gpu N runs this start on one card; neither: the saved
+    # choice, and asked once when the PC has cards that could share the model
+    run_gpu = (parse_gpus(a.gpus, gpus()) if a.gpus else None) or a.gpu
     port = a.port or 8080                              # a new install's port (issue #32: --port for an existing one)
     if have and a.calibrate and not (a.setup or a.model or a.family or a.check):
         if not a.build:
@@ -1140,35 +1368,38 @@ def main() -> int:
                 say(f"  {i}) {json.loads(c.read_text(encoding='utf-8-sig')).get('model_name', c.stem)}")
             pick_cfg = have[int(ask("Tune which one?", [str(i) for i in range(1, len(have) + 1)], "1", a.yes)) - 1]
         calibrate_config(pick_cfg)
-        return 0 if a.no_start else start(pick_cfg, a.port, a.gpu)
+        return 0 if a.no_start else start(pick_cfg, a.port, run_gpu, yes=a.yes, layer_split=a.layer_split)
     if have and not (a.setup or a.model or a.family or a.check or a.no_start):
         if not a.build:
             update_installed_engine(a.prebuilt)
         if len(have) == 1:
-            return start(have[0], a.port, a.gpu)
+            return start(have[0], a.port, run_gpu, yes=a.yes, layer_split=a.layer_split)
         say()
         for i, c in enumerate(have, 1):
             say(f"  {i}) {json.loads(c.read_text(encoding='utf-8-sig')).get('model_name', c.stem)}")
         say(f"  {len(have) + 1}) install another model / change settings")
         pick = int(ask("Which one?", [str(i) for i in range(1, len(have) + 2)], "1", a.yes))
         if pick <= len(have):
-            return start(have[pick - 1], a.port, a.gpu)
+            return start(have[pick - 1], a.port, run_gpu, yes=a.yes, layer_split=a.layer_split)
 
     # ---- 1. the PC
     step(1, "checking your PC")
-    gpu = gpu_info()
-    if gpu is None:
+    found = gpus()
+    if not found:
         fail("no NVIDIA GPU found (nvidia-smi did not answer)",
              "install the NVIDIA driver from https://www.nvidia.com/drivers and restart the PC")
-    ok(f"GPU: {gpu['name']}, {gpu['vram_gb']:.1f} GB VRAM, compute capability {gpu['arch'][:-1]}.{gpu['arch'][-1]}, "
-       f"driver {gpu['driver']}")
-    if gpu["count"] > 1:
-        say(f"       {gpu['count']} NVIDIA GPUs: Strata uses GPU {gpu['index']}"
-            + (" (the one with the most VRAM)" if a.gpu is None else "") + " - choose another with --gpu N:")
-        for x in gpus():
-            say(f"         {x['index']}: {x['name']}, {x['vram_gb']:.0f} GB")
-    if int(gpu["arch"]) < 80:
-        fail("this GPU is older than the RTX 30 series (compute capability 8.0 is required)")
+    if len(found) > 1 or gpu_problem(found[0]) is not None:
+        gpu_table(found)
+    sel = choose_gpus(a, found)                        # asked when two or more cards can share the model
+    multi = sel if len(sel) > 1 else []
+    a.gpu = sel[0]                                     # the main GPU: the checks and the sizing below are its
+    GPU_PICK = a.gpu
+    gpu = gpu_info(a.gpu)
+    chosen = [gpu_info(i) for i in sel]
+    gpu["archs"] = sorted({x["arch"] for x in chosen})  # the engine needs code for every one of them
+    if multi:
+        ok("GPUs: " + " + ".join(gpu_name(x) for x in chosen) + " together (the model's layers are split across them)")
+    ok(f"GPU: {gpu['name']}, {gpu['vram_gb']:.1f} GB VRAM, compute capability {cc(gpu)}, driver {gpu['driver']}")
     if driver_major(gpu) < MIN_DRIVER:
         fail(f"the NVIDIA driver is too old ({gpu['driver']}; {MIN_DRIVER} or newer is needed)",
              "update it with the NVIDIA App or from https://www.nvidia.com/drivers, restart, and run this again")
@@ -1225,11 +1456,21 @@ def main() -> int:
     rec = str(names.index("IQ3_XXS") + 1) if ram >= 60 and "IQ3_XXS" in names else "1"
     model = a.model or names[int(ask("Which size?", [str(i) for i in range(1, len(names) + 1)], rec, a.yes)) - 1]
     if ram < MODELS[model]["ram_gb"] - 4:
-        fail(f"{model} needs about {MODELS[model]['ram_gb']} GB of RAM; this PC has {ram:.0f} GB",
-             "choose Q2_0 or IQ2_XS, or add RAM")
+        # #125: a warning and a question, not a stop: the user may accept paging (asked, "no" by default, so an
+        # unattended --yes install still stops here)
+        need_gb, arena = MODELS[model]["ram_gb"], MODELS[model]["arena_gb"]
+        warn(f"{model} needs about {need_gb} GB of RAM and this PC has {ram:.0f} GB: its experts alone are "
+             f"{arena:.0f} GB and must stay in RAM, so Windows/Linux will page part of them from disk. Expect it "
+             "to be much slower, and it may not start at all.")
+        say("       A smaller size (Q2_0 or IQ2_XS) fits; more RAM fixes it.")
+        if ask("  Install it anyway?", ["y", "n"], "n", a.yes) != "y":
+            fail(f"{model} needs about {need_gb} GB of RAM; this PC has {ram:.0f} GB",
+                 "choose Q2_0 or IQ2_XS, or add RAM")
+        warn(f"installing {model} with {ram:.0f} GB of RAM, as you chose")
     ok(f"size: {model}")
     tag = fam["tag"] + model                           # names of the pack, config and start script
-    rec_ctx = 32768 if gpu["vram_gb"] < 14 else 65536 if gpu["vram_gb"] < 20 else 131072
+    small = min(x["vram_gb"] for x in chosen)         # each card keeps its layers' KV of the whole context
+    rec_ctx = 32768 if small < 14 else 65536 if small < 20 else 131072
     if a.context:
         ctx = a.context
     else:
@@ -1335,9 +1576,7 @@ def main() -> int:
                 except OSError:
                     pass
             download(fam["hf"].format(q=model) + s.name, s)
-    for s in shards:
-        if not s.exists():
-            fail(f"missing {s}")
+    check_shards(shards)
     ok("model files present")
     mmproj = Path(a.models_dir) / fam["mmproj"]
     if not mmproj.exists():
@@ -1415,6 +1654,11 @@ def main() -> int:
            "lib_dirs": lib_dirs, "port": port}
     if gpu["count"] > 1 or a.gpu is not None:
         cfg["gpu"] = gpu["index"]                      # the engine is told this card (issue #51)
+        cfg["gpus_asked"] = True                       # chosen at setup: not asked again at start
+    if multi:                                          # a layer split across these cards (the server adds the flag)
+        cfg["gpu"] = multi
+        cfg["layer_split"] = a.layer_split or "auto"
+        ok(f"layer split across GPUs {multi} ({cfg['layer_split']})")
     if a.host:
         cfg["host"] = a.host
     if a.api_key:

src/artifact/gguf_reader_test.cpp

@@ -0,0 +1,93 @@
+// src/artifact/gguf_reader_test.cpp - the reader refuses what it cannot arbitrate (synthetic GGUF, no model,
+// no GPU).
+//
+//   1. two tensors with distinct names: the file opens and find() sees both;
+//   2. the same name twice: refused at open, and the error names the tensor and the file - GGUF has no
+//      index to say which of the two a name means, and find() is first-match, so the alternative is a
+//      lookup that silently picks by position.
+//
+// The fixture is a minimal GGUF v3 written here (header, no metadata, two F32[8] tensors, 32-byte
+// alignment), so the test needs neither gguf-py nor a shard.  The file is closed before it is removed:
+// on Windows an open mapping keeps it.
+#include "strata/artifact/gguf_reader.hpp"
+
+#include <cstdint>
+#include <cstdio>
+#include <cstring>
+#include <filesystem>
+#include <fstream>
+#include <string>
+#include <vector>
+
+namespace {
+int g_fail = 0;
+void check(bool ok, const char* what) {
+    std::printf("  %-66s %s\n", what, ok ? "ok" : "FAIL");
+    if (!ok) ++g_fail;
+}
+
+template <class T> void put(std::vector<uint8_t>& b, T v) {
+    const auto n = b.size();
+    b.resize(n + sizeof(T));
+    std::memcpy(b.data() + n, &v, sizeof(T));
+}
+void put_str(std::vector<uint8_t>& b, const std::string& s) {
+    put<uint64_t>(b, s.size());
+    b.insert(b.end(), s.begin(), s.end());
+}
+
+// A GGUF v3 file of F32[8] tensors named `names`, laid out 32 bytes apart from a 32-byte-aligned data start.
+std::filesystem::path write_gguf(const std::vector<std::string>& names) {
+    std::vector<uint8_t> b;
+    put<uint32_t>(b, 0x46554747u);   // "GGUF"
+    put<uint32_t>(b, 3);
+    put<uint64_t>(b, names.size());  // n_tensors
+    put<uint64_t>(b, 0);             // n_kv
+    for (size_t i = 0; i < names.size(); ++i) {
+        put_str(b, names[i]);
+        put<uint32_t>(b, 1);         // n_dims
+        put<uint64_t>(b, 8);
+        put<uint32_t>(b, 0);         // F32
+        put<uint64_t>(b, 32 * i);    // offset from data_start
+    }
+    b.resize((b.size() + 31) / 32 * 32 + 32 * names.size(), 0);
+    const auto path = std::filesystem::temp_directory_path() / "strata_gguf_reader_test.gguf";
+    std::ofstream(path, std::ios::binary).write(reinterpret_cast<const char*>(b.data()), (std::streamsize)b.size());
+    return path;
+}
+}  // namespace
+
+int main() {
+    std::printf("gguf_reader_test\n");
+    {
+        const auto path = write_gguf({"blk.0.attn_q.weight", "blk.0.attn_k.weight"});
+        std::string err;
+        size_t n = 0;
+        bool both = false;
+        try {
+            strata::GgufFile g(path.string());
+            n = g.tensors().size();
+            both = g.find("blk.0.attn_q.weight") && g.find("blk.0.attn_k.weight");
+        } catch (const std::exception& e) {
+            err = e.what();
+        }
+        std::filesystem::remove(path);
+        check(err.empty() && n == 2 && both, "distinct names: the file opens and find() sees both tensors");
+    }
+    {
+        const auto path = write_gguf({"blk.0.attn_q.weight", "blk.0.attn_q.weight"});
+        std::string err;
+        try {
+            strata::GgufFile g(path.string());
+        } catch (const std::exception& e) {
+            err = e.what();
+        }
+        std::filesystem::remove(path);
+        check(!err.empty(), "the same name twice: refused at open");
+        check(err.find("blk.0.attn_q.weight") != std::string::npos, "  the error names the tensor");
+        check(err.find(path.filename().string()) != std::string::npos, "  the error names the file");
+        if (!err.empty()) std::printf("  (%s)\n", err.c_str());
+    }
+    std::printf(g_fail ? "gguf_reader_test: %d FAILED\n" : "gguf_reader_test: all passed\n", g_fail);
+    return g_fail ? 1 : 0;
+}

src/core/device.cu

@@ -30,7 +30,7 @@ DeviceInfo device_info(int ordinal) {
     int count = 0;
     check(cudaGetDeviceCount(&count), "cudaGetDeviceCount");
     if (count == 0) {
-        throw CudaError("no CUDA device is present; Strata targets sm_120 (RTX 5000 series)", -1);
+        throw CudaError("no CUDA device is present; Strata needs an NVIDIA GPU of compute capability 8.0 or newer", -1);
     }
     if (ordinal < 0 || ordinal >= count) {
         throw CudaError("device ordinal " + std::to_string(ordinal) + " is out of range (have " +
@@ -56,13 +56,14 @@ DeviceInfo device_info(int ordinal) {
     check(cudaDriverGetVersion(&d.driver_version), "cudaDriverGetVersion");
     check(cudaRuntimeGetVersion(&d.runtime_version), "cudaRuntimeGetVersion");
 
-    // The engine is written against sm_120.  Compiling for it is enforced by CMake; RUNNING on something else
-    // is caught here, because a binary can be carried to a machine with an older card and would otherwise
-    // silently take whatever path the driver chose.
-    if (d.cc_major != 12) {
+    // The kernels need sm_80 or newer (tf32 mma in the attention scorer, bf16 math) - the same floor the
+    // arch guard in CMakeLists enforces at build time (RTX 30 / 40 / 50).  Anything older is caught here,
+    // because a binary can be carried to a machine with an older card and would otherwise silently take
+    // whatever path the driver chose.  Pre-Blackwell is untested by the author; trust, then verify.
+    if (d.cc_major < 8) {
         throw CudaError("device " + d.name + " reports compute capability " + std::to_string(d.cc_major) +
                             "." + std::to_string(d.cc_minor) +
-                            "; Strata targets sm_120 (RTX 5000 series / Blackwell) only",
+                            "; Strata needs an NVIDIA GPU of compute capability 8.0 or newer (RTX 30 / 40 / 50)",
                         -1);
     }
     return d;

src/core/expert_cache.cpp

@@ -265,6 +265,32 @@ bool ExpertCache::fill_slot_blocking(int32_t slot, const uint8_t* host_blob, std
     return true;
 }
 
+bool ExpertCache::fill_slot_queued(int32_t slot, const uint8_t* host_blob, std::string& err, int64_t bytes) {
+    const size_t n = (size_t) (bytes > 0 && bytes <= blob_ ? bytes : blob_);
+    uint8_t* dst = device_slot(slot);
+    if (dst == nullptr || host_blob == nullptr) {
+        err = dst == nullptr ? "ExpertCache::fill_slot_queued: slot outside the arena"
+                             : "ExpertCache::fill_slot_queued: the host blob is null";
+        return false;
+    }
+    const cudaError_t e = cudaMemcpyAsync(dst, host_blob, n, cudaMemcpyHostToDevice, (cudaStream_t) 0);
+    if (e != cudaSuccess) {
+        err = std::string("ExpertCache::fill_slot_queued: ") + cudaGetErrorString(e);
+        return false;
+    }
+    ++fills_;
+    return true;
+}
+
+bool ExpertCache::sync_queued(std::string& err) {
+    const cudaError_t e = cudaStreamSynchronize((cudaStream_t) 0);
+    if (e != cudaSuccess) {
+        err = std::string("ExpertCache::sync_queued: ") + cudaGetErrorString(e);
+        return false;
+    }
+    return true;
+}
+
 bool ExpertCache::verify_slot(int32_t slot, const uint8_t* host_blob, std::string& err, int64_t bytes) {
     const int64_t nb = bytes > 0 && bytes <= blob_ ? bytes : blob_;
     const uint8_t* src = device_slot(slot);

src/core/expert_source.cpp

@@ -1,5 +1,6 @@
 // src/core/expert_source.cpp - the adapter.  See the header for the three clauses of the contract.
 #include "strata/core/expert_source.hpp"
+#include "strata/core/remote_experts.hpp"
 #include "strata/kernels/cpu/expert_layout.hpp"
 
 #include "strata/core/pinned.hpp"
@@ -199,6 +200,22 @@ void expert_pool_dispatch(void* user, const float* x_f, const int32_t* ids, cons
     const bool use_hits = graph_hits || (d.hits_ready() && d.decided);
     int64_t njobs = 0;
 
+    if (d.remote_count > 0) {
+        int32_t kind[32];
+        if (k > 32) {
+            d.failed = true; d.fail = "remote experts: routing width exceeds 32"; return;
+        }
+        for (int64_t i = 0; i < k; ++i)
+            kind[i] = use_hits && ids[i] >= 0 && ids[i] < d.n_expert && (graph_hits
+                ? d.host_res[(size_t) d.layers * (size_t) d.n_expert + (size_t) ids[i]] >= 0
+                : d.is_hit[(size_t) i] != 0) ? 0 : -1;
+        static thread_local std::string remote_error;
+        for (int r = 0; r < d.remote_count; ++r)
+            if (!d.remote[r]->begin(d.layers, x_f, ids, 1, k, kind, d.host_res, remote_error)) {
+                d.failed = true; d.fail = remote_error.c_str(); d.fail_layer = d.layers; return;
+            }
+    }
+
     for (int64_t i = 0; i < k; ++i) {
         const int64_t e = ids[i];
         if (e < 0 || e >= d.n_expert) {
@@ -231,6 +248,13 @@ void expert_pool_dispatch(void* user, const float* x_f, const int32_t* ids, cons
         }
         if (graph_hits) ++d.cache_refused;   // token graph: a miss (nothing is admitted during a token)
 
+        bool remote_owns = false;
+        for (int r = 0; r < d.remote_count; ++r) remote_owns |= d.remote[r]->owns(i);
+        if (remote_owns) {
+            std::memset(out + (size_t) i * (size_t) n_embd, 0, (size_t) n_embd * sizeof(float));
+            continue;
+        }
+
         // `njobs` indexes the JOB ARRAY and `i` indexes the OUTPUT - they are the same only when nothing is a
         // hit, and using one for the other is how a hit's row would get two experts summed into it.
         ExpertJob& j = d.jobs[(size_t) njobs++];
@@ -244,6 +268,13 @@ void expert_pool_dispatch(void* user, const float* x_f, const int32_t* ids, cons
     // Plan v0.3 P4: rows of every expert across all threads (bitwise the same as `run`).
     if (d.split_rows) d.pool->run_split(d.jobs.data(), (int) njobs);
     else d.pool->run(d.jobs.data(), (int) njobs);
+    if (d.remote_count > 0) {
+        static thread_local std::string remote_error;
+        for (int r = 0; r < d.remote_count; ++r)
+            if (!d.remote[r]->finish(out, remote_error)) {
+                d.failed = true; d.fail = remote_error.c_str(); d.fail_layer = d.layers; return;
+            }
+    }
     ++d.layers;
     d.experts += k;
 }
@@ -366,6 +397,15 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
                        d.host_res[(size_t) d.layers * (size_t) d.n_expert + (size_t) e] >= 0) ? 0 : -1;
         }
     }
+    if (d.remote_count > 0) {
+        static thread_local std::string remote_error;
+        for (int r = 0; r < d.remote_count; ++r) {
+            if (!d.remote[r]->begin(d.layers, x_f, ids, n_tok, k, kind, d.host_res, remote_error)) {
+                d.failed = true; d.fail = remote_error.c_str(); d.fail_layer = d.layers; return;
+            }
+            for (int64_t i = 0; i < n; ++i) if (d.remote[r]->owns(i)) kind[i] = 2;
+        }
+    }
     const auto c1 = std::chrono::steady_clock::now();
     if (native && lay.fmt[(size_t) d.layers].gu_type == 42)   // a native Q2_0 pack: the Q2_0 kernels' activations
         for (int64_t t = 0; t < n_tok; ++t) act_quant_any(x_f + (size_t) t * H, H, d.act_multi[(size_t) t]);
@@ -388,7 +428,7 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
                 d.fail_expert = e;
                 return;
             }
-            if (kind[i] >= 0) {             // the GPU computes this entry (a VRAM hit or a PCIe read)
+            if (kind[i] >= 0) {             // CUDA0, PCIe, or a remote result staged into this row below
                 if (kind[i] == 0) ++d.cache_hits;
                 std::memset(row, 0, (size_t) H * sizeof(float));
                 continue;
@@ -421,6 +461,13 @@ void expert_pool_dispatch_multi(ExpertDispatch& d, const float* x_f, const int32
     pt("run", njobs);
     if (native) d.pool->run_split_multi_native(lay.fmt[(size_t) d.layers], d.jobs_multi.data(), njobs);
     else d.pool->run_split_multi(d.jobs_multi.data(), njobs);
+    if (d.remote_count > 0) {
+        static thread_local std::string remote_error;
+        for (int r = 0; r < d.remote_count; ++r)
+            if (!d.remote[r]->finish(out, remote_error)) {
+                d.failed = true; d.fail = remote_error.c_str(); d.fail_layer = d.layers; return;
+            }
+    }
     const auto c4 = std::chrono::steady_clock::now();
     pt("ran");
     auto ms = [](auto a, auto b) { return std::chrono::duration<double, std::milli>(b - a).count(); };
@@ -606,6 +653,8 @@ LoadStats load_experts_gguf(const std::string& gguf, uint8_t* dst, const strata:
     for (auto& t : pool) t.join();
     if (bad) {
         st.seconds = -1.0;
+        st.ok = false;
+        st.error = "short read or unreadable shard while reading the experts from the GGUF";
         return st;
     }
     st.bytes = lay.total;
@@ -616,7 +665,7 @@ LoadStats load_experts_gguf(const std::string& gguf, uint8_t* dst, const strata:
 ArenaExpertSource::~ArenaExpertSource() { close(); }
 
 bool ArenaExpertSource::open(const std::string& pack_dir, int64_t n_layers, int64_t n_expert, int threads,
-                             std::string& err) {
+                             std::string& err, uint64_t max_pinned_bytes) {
     close();
     const std::string path = pack_dir + "/experts.bin";
     // plan v0.3 P6: the layout (canonical, or a native pack's per-layer blobs) was loaded by the driver
@@ -657,7 +706,7 @@ bool ArenaExpertSource::open(const std::string& pack_dir, int64_t n_layers, int6
         lbytes.push_back(lay.blob_bytes(l) * (uint64_t) n_expert);
     }
     bounds.push_back(want);
-    PinnedArena* a = new PinnedArena(want + (uint64_t) blob, bounds);
+    PinnedArena* a = new PinnedArena(want + (uint64_t) blob, bounds, max_pinned_bytes);
     if (!a->valid()) {
         delete a;
         err = "ArenaExpertSource: the arena could not be reserved (" + std::to_string(want) + " B)";
@@ -665,6 +714,11 @@ bool ArenaExpertSource::open(const std::string& pack_dir, int64_t n_layers, int6
     }
     const LoadStats st = from_gguf ? load_experts_gguf(gguf_, a->data(), lay, threads)
                                    : load_experts_ranges(path, a->data(), loff, lbytes, threads, /*chunk=*/8u << 20);
+    if (!st.ok) {
+        delete a;
+        err = "ArenaExpertSource: the expert load was refused: " + (st.error.empty() ? std::string("unknown") : st.error);
+        return false;
+    }
     if (st.bytes != want) {
         delete a;
         err = "ArenaExpertSource: the load read " + std::to_string(st.bytes) + " B of " + std::to_string(want);
@@ -693,6 +747,9 @@ bool ArenaExpertSource::open(const std::string& pack_dir, int64_t n_layers, int6
     reads_ = 0;
     note_ = a->note;
     gib_per_s_ = st.gib_per_second();
+    load_seconds_ = st.seconds;
+    load_read_s_ = st.read_seconds;
+    load_copy_s_ = st.copy_seconds;
     return true;
 }
 

src/core/load_main.cpp

@@ -65,7 +65,10 @@ int main(int argc, char** argv) {
 
     const strata::core::LoadStats st =
         strata::core::load_experts(path, dst, blob_bytes, blobs_per_layer, layers, threads, chunk);
-    if (st.seconds < 0) return 1;
+    if (!st.ok) {
+        std::fprintf(stderr, "loading the experts failed: %s\n", st.error.empty() ? "unknown" : st.error.c_str());
+        return 1;
+    }
 
     std::printf("loaded %.3f GiB in %.3f s with %d threads and %llu MiB chunks  ->  %.2f GiB/s\n",
                 (double) st.bytes / (1024.0 * 1024 * 1024), st.seconds, threads,

src/core/mtp.cpp

@@ -1,5 +1,6 @@
 // src/core/mtp.cpp - see include/strata/core/mtp.hpp.
 #include "strata/core/mtp.hpp"
+#include "strata/core/on_device.hpp"
 
 #include "strata/core/native_head.hpp"
 #include "strata/kernels/bf16_gemv.hpp"
@@ -68,13 +69,31 @@ strata::kernels::QsaShapes shapes_of(const ModelGeometry& g) {
     return s;
 }
 
+// 64-bit seek/tell on a `FILE*`: `fseek`/`ftell` take a 32-bit `long` on Windows and would wrap past 2 GiB.
+#if defined(_WIN32)
+#define STRATA_FILE_SEEK64(f, o, w) _fseeki64((f), (long long) (o), (w))
+#define STRATA_FILE_TELL64(f) _ftelli64(f)
+#else
+#define STRATA_FILE_SEEK64(f, o, w) fseeko((f), (off_t) (o), (w))
+#define STRATA_FILE_TELL64(f) ftello(f)
+#endif
+
 bool read_file(const std::string& path, std::vector<uint8_t>& out) {
-    std::ifstream f(path, std::ios::binary | std::ios::ate);
-    if (!f) return false;
-    const std::streamsize n = f.tellg();
-    f.seekg(0);
+    // Loader fix (0.1.15+loaderfix.2): `ifstream::read` reaches the disk as 4095-byte reads on MSVC - the same
+    // split `load_experts_ranges` had - so the drafter's 111 MiB dense blob paid 4 KiB per operation on a cold
+    // start.  `fread` passes a request bigger than the stream buffer straight to `ReadFile()`.
+    FILE* f = std::fopen(path.c_str(), "rb");
+    if (f == nullptr) return false;
+    // A `FILE*` has no destructor and the short-read path below returns early: the guard closes on every path.
+    struct Closer {
+        FILE* f;
+        ~Closer() { if (f != nullptr) std::fclose(f); }
+    } closer{f};
+    if (STRATA_FILE_SEEK64(f, 0, SEEK_END) != 0) return false;
+    const long long n = (long long) STRATA_FILE_TELL64(f);
+    if (n < 0 || STRATA_FILE_SEEK64(f, 0, SEEK_SET) != 0) return false;
     out.resize((size_t) n);
-    return (bool) f.read((char*) out.data(), n);
+    return n == 0 || std::fread(out.data(), 1, (size_t) n, f) == (size_t) n;
 }
 
 }  // namespace
@@ -82,6 +101,8 @@ bool read_file(const std::string& path, std::vector<uint8_t>& out) {
 MtpDrafter::~MtpDrafter() {
     if (cs_) cudaStreamSynchronize(cs_);
     for (auto& e : prefill_exec_) if (e) cudaGraphExecDestroy(e);
+    for (auto& e : prefill_dev_exec_) if (e) cudaGraphExecDestroy(e);
+    if (pf_dev_) cudaFree(pf_dev_);
     for (auto& e : round_exec_) if (e) cudaGraphExecDestroy(e);
     for (auto& e : step_exec_) if (e) cudaGraphExecDestroy(e);
     if (cs_) cudaStreamDestroy(cs_);
@@ -111,11 +132,15 @@ const void* MtpDrafter::q8(const char* name) const {
 
 bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, SessionState& ss, int max_t, std::string& err,
                       int64_t window) {
+    cudaGetDevice(&device_);   // a layer split's last stage on another GPU: the drafter lives there
     g_ = &g;
     ss_ = &ss;
     max_t_ = max_t;
     rt_dir_ = rt_dir;
     if (max_t < 1 || max_t > strata::kernels::kVerifyMaxT) { err = "mtp: max_t out of range"; return false; }
+    // Loader fix (0.1.15+loaderfix.2): the two reads below are the whole “drafter files” cost; reporting
+    // them apart from the rest of the stage is what makes the next regression visible.
+    const auto t_files = std::chrono::steady_clock::now();
     // ---- the index and the dense weights
     {
         std::ifstream idx(rt_dir + "/dense.txt");
@@ -131,20 +156,34 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Session
         }
         std::vector<uint8_t> blob;
         if (!read_file(rt_dir + "/dense.bin", blob)) { err = "mtp: cannot read dense.bin"; return false; }
-        if (cudaMalloc((void**) &dense_, blob.size()) != cudaSuccess) { err = "mtp: dense weights do not fit"; return false; }
+        const cudaError_t alloc = cudaMalloc((void**) &dense_, blob.size());
+        if (alloc != cudaSuccess) {
+            size_t free_bytes = 0, total_bytes = 0;
+            const cudaError_t info = cudaMemGetInfo(&free_bytes, &total_bytes);
+            err = "mtp: dense weights allocation failed (" + std::string(cudaGetErrorString(alloc)) +
+                  "), requested " + std::to_string(blob.size() >> 20) + " MiB, CUDA0 free " +
+                  (info == cudaSuccess ? std::to_string(free_bytes >> 20) + " MiB" : "unknown");
+            return false;
+        }
         cudaMemcpy(dense_, blob.data(), blob.size(), cudaMemcpyHostToDevice);
         vram_ += blob.size();
     }
     // ---- the 512 routed experts, one blob each
     {
         const uint64_t bytes = (uint64_t) g.n_expert * strata::kernels::cpu::BLOB;
-        std::ifstream f(rt_dir + "/experts.bin", std::ios::binary);
-        if (!f) { err = "mtp: cannot open experts.bin"; return false; }
+        // Loader fix (0.1.15+loaderfix.2): each 64 MiB read below reached the disk as ~16k 4095-byte reads under
+        // MSVC's `basic_filebuf::xsgetn`, which is what made 675 MiB of drafter experts take minutes.
+        FILE* f = std::fopen((rt_dir + "/experts.bin").c_str(), "rb");
+        if (f == nullptr) { err = "mtp: cannot open experts.bin"; return false; }
+        struct Closer {
+            FILE* f;
+            ~Closer() { if (f != nullptr) std::fclose(f); }
+        } closer{f};
         if (cudaMalloc((void**) &experts_, bytes) != cudaSuccess) { err = "mtp: the 512 experts do not fit in VRAM"; return false; }
         std::vector<uint8_t> chunk(64u << 20);
         for (uint64_t off = 0; off < bytes;) {
             const uint64_t n = std::min<uint64_t>(chunk.size(), bytes - off);
-            if (!f.read((char*) chunk.data(), (std::streamsize) n)) { err = "mtp: experts.bin is truncated"; return false; }
+            if (std::fread(chunk.data(), 1, (size_t) n, f) != (size_t) n) { err = "mtp: experts.bin is truncated"; return false; }
             cudaMemcpy(experts_ + off, chunk.data(), n, cudaMemcpyHostToDevice);
             off += n;
         }
@@ -236,13 +275,17 @@ bool MtpDrafter::load(const std::string& rt_dir, const ModelGeometry& g, Session
         cudaMemcpy(ident_, id.data(), id.size() * 4, cudaMemcpyHostToDevice);
     }
     if (cudaStreamCreateWithFlags(&cs_, cudaStreamNonBlocking) != cudaSuccess) { err = "mtp: stream"; return false; }
-    std::fprintf(stderr, "strata mtp: draft layer loaded, %.0f MiB of VRAM (experts %.0f, dense %.0f)\n",
+    const double files_s = std::chrono::duration<double>(std::chrono::steady_clock::now() - t_files).count();
+    std::fprintf(stderr, "strata mtp: draft layer loaded, %.0f MiB of VRAM (experts %.0f, dense %.0f), files read in %.2f s (%.0f MiB/s)\n",
                  (double) vram_ / 1048576.0, (double) g.n_expert * strata::kernels::cpu::BLOB / 1048576.0,
-                 (double) tensors_.back().off / 1048576.0);
+                 (double) tensors_.back().off / 1048576.0, files_s,
+                 files_s > 0 ? ((double) g.n_expert * strata::kernels::cpu::BLOB + (double) tensors_.back().off) /
+                                   1048576.0 / files_s : 0.0);
     return true;
 }
 
 bool MtpDrafter::bind(const WeightTable& wt, const NativeHead* head, const float* window_R, std::string& err) {
+    const OnDevice on_device(device_);
     wt_ = &wt;
     head_ = head;
     window_R_ = window_R;
@@ -469,6 +512,13 @@ bool MtpDrafter::capture_prefill(int T, std::string& err) {
     return finish_capture(cs_, ok, prefill_exec_[T], "prefill", err);
 }
 
+bool MtpDrafter::capture_prefill_dev(int T, std::string& err) {
+    if (prefill_dev_exec_[T]) return true;
+    if (cudaStreamBeginCapture(cs_, cudaStreamCaptureModeThreadLocal) != cudaSuccess) { err = "mtp: begin capture"; return false; }
+    const bool ok = record_forward(T, -1, cs_, err);   // K/V only, rows [0, T); tok_/step_/pos_ filled before launch
+    return finish_capture(cs_, ok, prefill_dev_exec_[T], "prefill (device inputs)", err);
+}
+
 bool MtpDrafter::capture_round(int T, std::string& err) {
     if (round_exec_[T]) return true;
     using namespace strata::kernels;
@@ -511,6 +561,7 @@ bool MtpDrafter::capture_step(int j, std::string& err) {
 }
 
 void MtpDrafter::kv_restore(int64_t upto) {
+    const OnDevice on_device(device_);
     if (st_.kv_mode != 2 || upto <= 0) return;
     // the ring's blocks below `upto`, from the host copy: a checkpoint resume may have left later cells in them
     const strata::kernels::QsaShapes s = shapes_of(*g_);
@@ -520,10 +571,69 @@ void MtpDrafter::kv_restore(int64_t upto) {
 }
 
 bool MtpDrafter::prefill(const float* R_rows, const int32_t* next_tokens, int64_t n, int64_t cell0, std::string& err) {
+    const OnDevice on_device(device_);
     const Clock::time_point t0 = Clock::now();
     const int64_t HCN = g_->hc * g_->n_embd;
     // cells the window can never reach again need no K/V
     const int64_t first_needed = (window_ > 0 && prompt_len_ > 0) ? prompt_len_ - window_ - 64 : 0;
+    // E-4: every group's token / step / position records uploaded at once; each group is then device copies and a
+    // graph on the one stream, with a single sync at the end (a group of <= max_t rows used to be staged in mapped
+    // memory and synced before the next: ~5,500 host round trips on a 32K prompt).  The same work in the same
+    // order.  STRATA_MTP_PREFILL_SYNC=1 keeps the old loop.
+    static const bool per_group_sync = std::getenv("STRATA_MTP_PREFILL_SYNC") != nullptr;
+    const int64_t NHp = g_->n_head, per_row = 1 + 4 + NHp;
+    if (!per_group_sync && n > 0) {
+        if (pf_cap_ < n * per_row) {
+            if (pf_dev_) cudaFree(pf_dev_);
+            pf_dev_ = nullptr;
+            pf_cap_ = 0;
+            if (cudaMalloc((void**) &pf_dev_, (size_t) (n * per_row) * sizeof(int32_t)) != cudaSuccess) {
+                err = "mtp prefill: the input records do not fit";
+                return false;
+            }
+            pf_cap_ = n * per_row;
+        }
+        std::vector<int32_t> rec((size_t) (n * per_row));
+        int32_t* tk = rec.data();
+        int32_t* stp = tk + n;
+        int32_t* ps = stp + 4 * n;
+        for (int64_t i = 0; i < n; ++i) {
+            const int64_t cell = cell0 + i;
+            tk[i] = next_tokens[i];
+            stp[i * 4 + 0] = (int32_t) cell;
+            stp[i * 4 + 1] = (int32_t) (cell + 1);
+            stp[i * 4 + 2] = (int32_t) ((cell + 1) / 4);
+            stp[i * 4 + 3] = (int32_t) (cell + 1);
+            for (int64_t h = 0; h < NHp; ++h) ps[i * NHp + h] = (int32_t) cell;
+        }
+        if (cudaMemcpyAsync(pf_dev_, rec.data(), rec.size() * sizeof(int32_t), cudaMemcpyHostToDevice, cs_) != cudaSuccess) {
+            err = "mtp prefill: the input records upload failed";
+            return false;
+        }
+        const int32_t* d_tk = pf_dev_;
+        const int32_t* d_stp = d_tk + n;
+        const int32_t* d_ps = d_stp + 4 * n;
+        for (int64_t c = 0; c < n; c += max_t_) {
+            const int T = (int) std::min<int64_t>(max_t_, n - c);
+            if (cell0 + c + T <= first_needed) continue;
+            if (!capture_prefill_dev(T, err)) return false;
+            if (cudaMemcpyAsync(tok_, d_tk + c, (size_t) T * 4, cudaMemcpyDeviceToDevice, cs_) != cudaSuccess ||
+                cudaMemcpyAsync(step_, d_stp + c * 4, (size_t) T * 16, cudaMemcpyDeviceToDevice, cs_) != cudaSuccess ||
+                cudaMemcpyAsync(pos_, d_ps + c * NHp, (size_t) (T * NHp) * 4, cudaMemcpyDeviceToDevice, cs_) != cudaSuccess ||
+                cudaMemcpyAsync(Rin_, R_rows + (size_t) c * HCN, (size_t) T * HCN * sizeof(float), cudaMemcpyDeviceToDevice,
+                                cs_) != cudaSuccess ||
+                cudaGraphLaunch(prefill_dev_exec_[T], cs_) != cudaSuccess) {
+                err = std::string("mtp prefill: ") + cudaGetErrorString(cudaGetLastError());
+                return false;
+            }
+        }
+        if (cudaStreamSynchronize(cs_) != cudaSuccess) {
+            err = std::string("mtp prefill: ") + cudaGetErrorString(cudaGetLastError());
+            return false;
+        }
+        ms_prefill += ms_since(t0);
+        return true;
+    }
     for (int64_t c = 0; c < n; c += max_t_) {
         const int T = (int) std::min<int64_t>(max_t_, n - c);
         if (cell0 + c + T <= first_needed) continue;
@@ -551,6 +661,7 @@ bool MtpDrafter::prefill(const float* R_rows, const int32_t* next_tokens, int64_
 
 bool MtpDrafter::draft(int T, const int32_t* tokens, int64_t p, int a, int32_t* drafts, std::string& err,
                        float* probs, float min_p, int* n_drafts) {
+    const OnDevice on_device(device_);
     if (T < 1 || T > max_t_ || a < 0 || a >= T) { err = "mtp: draft arguments out of range"; return false; }
     if (!capture_round(T, err)) return false;
     const Clock::time_point t0 = Clock::now();
@@ -601,6 +712,7 @@ bool MtpDrafter::draft(int T, const int32_t* tokens, int64_t p, int a, int32_t*
 
 bool MtpDrafter::draft_first(int T, const float* R_row, int32_t token, int64_t cell, int32_t* drafts, std::string& err,
                              float* probs, float min_p, int* n_drafts) {
+    const OnDevice on_device(device_);
     // row 0 is the real pair; rows 1.. repeat it and only write cells the next round overwrites
     const int64_t HCN = g_->hc * g_->n_embd;
     for (int t = 0; t < T; ++t)

src/core/pinned.cu

@@ -12,6 +12,14 @@
 #include <mutex>
 #include <thread>
 
+// Loader fix: `fseek`/`ftell` are 32-bit on Windows by default (and the pack is 42.9 GB), and the 64-bit
+// spelling is not the same on the two platforms the engine builds for.
+#ifdef _WIN32
+#define STRATA_FSEEK64(f, o) _fseeki64((f), (long long) (o), SEEK_SET)
+#else
+#define STRATA_FSEEK64(f, o) fseeko((f), (off_t) (o), SEEK_SET)
+#endif
+
 #ifdef _WIN32
 #include <windows.h>
 #else
@@ -129,23 +137,27 @@ PinnedArena::PinnedArena(uint64_t bytes, uint64_t slice) : PinnedArena(bytes, un
     if (slice_bytes) slice_bytes = slice;   // sliced registration: record the uniform size
 }
 
-PinnedArena::PinnedArena(uint64_t bytes, const std::vector<uint64_t>& bounds) : capacity(bytes) {
+PinnedArena::PinnedArena(uint64_t bytes, const std::vector<uint64_t>& bounds,
+                         uint64_t max_pinned_bytes) : capacity(bytes) {
     if (bytes == 0) return;
     base = reserve(bytes, backing, note);
 
     // Register with CUDA BEFORE any page is touched: cudaHostRegister pins what is resident now, and a region
     // that has already been faulted in page by page is far more expensive to register and may fail outright.
     if (base) {
-        const cudaError_t e = cudaHostRegister(base, (size_t) bytes, cudaHostRegisterPortable | cudaHostRegisterMapped);
-        if (e == cudaSuccess) {
+        const bool capped = max_pinned_bytes > 0 && max_pinned_bytes < bytes && bounds.size() >= 2;
+        const cudaError_t e = capped ? cudaSuccess :
+            cudaHostRegister(base, (size_t) bytes, cudaHostRegisterPortable | cudaHostRegisterMapped);
+        if (!capped && e == cudaSuccess) {
             note = "cudaHostRegister PORTABLE ok; " + note;
             registered_bytes = bytes;
-        } else if (bounds.size() >= 2 && clear_error()) {
+        } else if (bounds.size() >= 2 && (capped || clear_error())) {
             // Plan v0.3 P5: the whole range is refused, so pin it slice by slice from the start.  The rest stays
             // resident through the working-set lock below.  (P6: slices may differ in size, one per layer.)
             slice_bytes = 1;   // sliced; the uniform constructor records the size
             for (size_t i = 0; i + 1 < bounds.size(); ++i) {
                 const uint64_t off = bounds[i], n = bounds[i + 1] - bounds[i];
+                if (capped && (off > max_pinned_bytes || n > max_pinned_bytes - off)) break;
                 if (cudaHostRegister((uint8_t*) base + off, (size_t) n, cudaHostRegisterPortable | cudaHostRegisterMapped) != cudaSuccess) {
                     (void) cudaGetLastError();
                     break;
@@ -154,7 +166,9 @@ PinnedArena::PinnedArena(uint64_t bytes, const std::vector<uint64_t>& bounds) :
                 registered_bytes = off + n;
                 ++registered_slices;
             }
-            note = "cudaHostRegister of the whole arena FAILED (" + std::string(cudaGetErrorString(e)) + "); " +
+            note = (capped ? "cudaHostRegister limited to " + std::to_string(max_pinned_bytes >> 30) +
+                             " GiB for CUDA1; " :
+                             "cudaHostRegister of the whole arena FAILED (" + std::string(cudaGetErrorString(e)) + "); ") +
                    std::to_string(registered_slices) + " slices pinned (" + std::to_string(registered_bytes >> 30) +
                    " GiB); " + note;
             if (registered_bytes < bytes) {
@@ -228,11 +242,33 @@ LoadStats load_experts_ranges(const std::string& path, uint8_t* dst, const std::
     const auto t0 = std::chrono::steady_clock::now();
     std::vector<uint64_t> layer_hash((size_t) layers, 1469598103934665603ull);
     std::atomic<uint64_t> next_layer{0};
+    std::atomic<uint64_t> read_ns_sum{0};   // summed over the threads: see LoadStats::read_seconds
+    std::atomic<uint64_t> copy_ns_sum{0};
     std::mutex err_mu;
     std::string err;
 
     auto worker = [&]() {
         std::vector<uint8_t> buf((size_t) chunk);
+        // One handle per thread, seeked once per layer: a shared handle would need a lock around the seek and
+        // would serialise the very thing the threads are here to parallelise.  `fread` on a `FILE*` rather than
+        // `std::ifstream`: see the header - MSVC's `basic_filebuf::xsgetn` splits any request larger than
+        // `_INTERNAL_BUFSIZ - 1` into 4095-byte freads, which turned one 8 MiB chunk into ~2048 4 KiB reads.
+        // `fread` sees a request bigger than the stream buffer and passes it to `_read()`/
+        // `ReadFile()` unchanged, so the chunk size reaches the disk.  Buffered, not `FILE_FLAG_NO_BUFFERING`:
+        // the cache should still hold what it can.
+        FILE* f = std::fopen(path.c_str(), "rb");
+        if (f == nullptr) {
+            std::lock_guard<std::mutex> g(err_mu);
+            err = "cannot open " + path;
+            return;
+        }
+        // A `FILE*` has no destructor that closes it, and this function has early returns below (open, seek and
+        // short-read failures), so the guard is what keeps the closing correct on every path.
+        struct Closer {
+            FILE* f;
+            ~Closer() { if (f != nullptr) std::fclose(f); }
+        } closer{f};
+        uint64_t read_ns = 0, copy_ns = 0;
         for (;;) {
             const uint64_t L = next_layer.fetch_add(1);
             if (L >= layers) break;
@@ -240,30 +276,39 @@ LoadStats load_experts_ranges(const std::string& path, uint8_t* dst, const std::
             uint64_t remaining = layer_bytes[(size_t) L];
             uint64_t pos = 0;
             uint64_t h = 1469598103934665603ull;
-            // one handle per thread, seeked once per layer: a shared handle would need a lock around the seek
-            // and would serialise the very thing the threads are here to parallelise
-            std::ifstream f(path, std::ios::binary);
-            if (!f) {
+            // 64-bit seek: the pack is 42.9 GB, so the 32-bit `fseek` would wrap past 4 GiB
+            if (STRATA_FSEEK64(f, off) != 0) {
                 std::lock_guard<std::mutex> g(err_mu);
-                err = "cannot open " + path;
+                err = "seek to " + std::to_string(off) + " B failed in layer " + std::to_string(L);
                 return;
             }
-            f.seekg((std::streamoff) off);
             while (remaining > 0) {
                 const uint64_t n = remaining < chunk ? remaining : chunk;
-                f.read((char*) buf.data(), (std::streamsize) n);
-                if ((uint64_t) f.gcount() != n) {
+                const auto t_read = std::chrono::steady_clock::now();
+                const size_t got = std::fread(buf.data(), 1, (size_t) n, f);
+                read_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(
+                               std::chrono::steady_clock::now() - t_read).count();
+                // A short read is EOF or an I/O error, never a silent zero fill: say WHERE and HOW SHORT, and
+                // keep going no further - the caller turns this into a refused load, not a wrong answer.
+                if (got != (size_t) n) {
                     std::lock_guard<std::mutex> g(err_mu);
-                    err = "short read in layer " + std::to_string(L);
+                    err = "short read in layer " + std::to_string(L) + ": got " + std::to_string(got) + " of "
+                          + std::to_string(n) + " B at offset " + std::to_string(off + pos)
+                          + (std::ferror(f) != 0 ? " (ferror set)" : "");
                     return;
                 }
+                const auto t_copy = std::chrono::steady_clock::now();
                 std::memcpy(dst + off + pos, buf.data(), (size_t) n);
                 h = fnv1a64(buf.data(), n, h);
+                copy_ns += (uint64_t) std::chrono::duration_cast<std::chrono::nanoseconds>(
+                               std::chrono::steady_clock::now() - t_copy).count();
                 pos += n;
                 remaining -= n;
             }
             layer_hash[(size_t) L] = h;
         }
+        read_ns_sum.fetch_add(read_ns);
+        copy_ns_sum.fetch_add(copy_ns);
     };
 
     std::vector<std::thread> pool;
@@ -274,8 +319,12 @@ LoadStats load_experts_ranges(const std::string& path, uint8_t* dst, const std::
     if (!err.empty()) {
         std::fprintf(stderr, "load_experts: %s\n", err.c_str());
         st.seconds = -1.0;
+        st.ok = false;
+        st.error = err;
         return st;
     }
+    st.read_seconds = (double) read_ns_sum.load() / 1e9;
+    st.copy_seconds = (double) copy_ns_sum.load() / 1e9;
     st.layer_checksums = std::move(layer_hash);
     st.seconds = std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count();
     return st;

src/core/remote_experts.cpp

@@ -0,0 +1,325 @@
+#include "strata/core/remote_experts.hpp"
+
+#include "strata/kernels/cpu/expert_layout.hpp"
+#include "strata/kernels/iq_kernels.hpp"
+#include "strata/kernels/quantize_act.hpp"
+#include "strata/kernels/s2_expert_grouped.hpp"
+
+#include <algorithm>
+#include <chrono>
+#include <cstdlib>
+#include <cstring>
+
+namespace strata::core {
+namespace {
+constexpr int64_t H = strata::kernels::cpu::H;
+constexpr int64_t FF = strata::kernels::cpu::FF;
+constexpr int64_t CAP = strata::kernels::cpu::MAXT * 10;
+
+// Small enough to copy as one pinned buffer per layer. The kernels read the
+// individual arrays through pointers into the same device allocation.
+struct RemoteMeta {
+    unsigned long long ptr[CAP];
+    int32_t start[CAP + 1];
+    int32_t dst[CAP];
+    int32_t tok[CAP];
+    int32_t count;
+};
+
+struct DeviceScope {
+    int previous = -1;
+    bool ok = false;
+    cudaError_t status = cudaSuccess;
+    const char* failed_step = nullptr;
+    explicit DeviceScope(int device) {
+        status = cudaGetDevice(&previous);
+        if (status != cudaSuccess) {
+            previous = -1;
+            failed_step = "cudaGetDevice";
+            return;
+        }
+        status = cudaSetDevice(device);
+        if (status != cudaSuccess) {
+            failed_step = "cudaSetDevice";
+            return;
+        }
+        ok = true;
+    }
+    ~DeviceScope() { if (previous >= 0) cudaSetDevice(previous); }
+    std::string error(int device) const {
+        return std::string("CUDA") + std::to_string(device) + " experts: " +
+               (failed_step ? failed_step : "device switch") +
+               "(" + std::to_string(device) + ") failed: " + cudaGetErrorString(status) +
+               " (CUDA error " + std::to_string((int) status) + ")";
+    }
+};
+
+bool check(cudaError_t result, const char* what, std::string& err, int device) {
+    if (result == cudaSuccess) return true;
+    err = "CUDA" + std::to_string(device) + " experts: " + what + ": " + cudaGetErrorString(result);
+    return false;
+}
+} // namespace
+
+RemoteExperts::~RemoteExperts() { close(); }
+
+bool RemoteExperts::preflight(int device, double& free_gib, std::string& err) {
+    int count = 0;
+    if (!check(cudaGetDeviceCount(&count), "cudaGetDeviceCount", err, device)) return false;
+    if (device < 1 || device >= count) {
+        err = "CUDA" + std::to_string(device) + " experts: CUDA device is not visible";
+        return false;
+    }
+    // The layer waits for this GPU on the CPU pool's critical path: spin instead of sleeping, whose wake-up
+    // costs more than a small expert batch takes (measured: ~0.3 ms per round trip on Windows).  Only possible
+    // before the device's context exists, so first thing; STRATA_REMOTE_SPIN=0 keeps the driver's default.
+    const char* spin = std::getenv("STRATA_REMOTE_SPIN");
+    if (!(spin && spin[0] == '0')) cudaInitDevice(device, cudaDeviceScheduleSpin | cudaDeviceMapHost, 0);
+    cudaGetLastError();
+    DeviceScope scope(device);
+    if (!scope.ok) { err = scope.error(device); return false; }
+    size_t free_bytes = 0, total_bytes = 0;
+    if (!check(cudaMemGetInfo(&free_bytes, &total_bytes), "preflight free memory", err, device)) return false;
+    free_gib = (double) free_bytes / 1073741824.0;
+    return true;
+}
+
+void RemoteExperts::close() {
+    if (device_ < 0) return;
+    DeviceScope scope(device_);
+    if (scope.ok) {
+        if (stream_) cudaStreamSynchronize(stream_);
+        cache_.close();
+        if (d_x_) cudaFree(d_x_);
+        if (d_out_) cudaFree(d_out_);
+        if (d_q8_) cudaFree(d_q8_);
+        if (d_scales_) cudaFree(d_scales_);
+        if (d_scratch_) cudaFree(d_scratch_);
+        if (d_meta_) cudaFree(d_meta_);
+        if (h_x_) cudaFreeHost(h_x_);
+        if (h_out_) cudaFreeHost(h_out_);
+        if (h_meta_) cudaFreeHost(h_meta_);
+        if (stream_) cudaStreamDestroy(stream_);
+    }
+    device_ = -1;
+    stream_ = nullptr;
+    h_x_ = h_out_ = d_x_ = d_out_ = nullptr;
+    h_meta_ = d_meta_ = nullptr;
+    d_q8_ = nullptr;
+    d_scales_ = nullptr;
+    d_scratch_ = nullptr;
+    d_start_ = d_dst_ = d_tok_ = d_count_ = nullptr;
+    d_ptr_ = nullptr;
+    original_row_.clear();
+    layers_present_.clear();
+}
+
+bool RemoteExperts::open(int device, int slots, int64_t layers, int64_t experts,
+                         const std::vector<std::pair<int32_t, int32_t>>& ranked,
+                         const ExpertCache& primary, ExpertSource& source,
+                         std::vector<uint8_t>& claimed, std::string& err) {
+    close();
+    int count = 0;
+    if (!check(cudaGetDeviceCount(&count), "cudaGetDeviceCount", err, device)) return false;
+    if (device < 1 || device >= count || slots <= 0 || ranked.empty() ||
+        layers <= 0 || experts <= 0 || claimed.size() != (size_t) layers * (size_t) experts) {
+        err = "CUDA" + std::to_string(device) + " experts: need the device, ranked experts and positive slot count";
+        return false;
+    }
+    DeviceScope scope(device);
+    if (!scope.ok) { err = scope.error(device); return false; }
+    device_ = device;
+    n_expert_ = experts;
+    const auto& lay = strata::kernels::cpu::expert_layout();
+    std::vector<std::pair<int32_t, int32_t>> selected;
+    selected.reserve((size_t) slots);
+    std::vector<uint8_t> picked(claimed.size(), 0);
+    for (const auto& pair : ranked) {
+        if (pair.first < 0 || pair.first >= layers || pair.second < 0 || pair.second >= experts) continue;
+        const size_t index = (size_t) pair.first * (size_t) experts + (size_t) pair.second;
+        if (primary.slot_of(pair.first, pair.second) < 0 && !claimed[index] && !picked[index]) {
+            selected.push_back(pair);
+            picked[index] = 1;
+            if ((int) selected.size() >= slots) break;
+        }
+    }
+    if (selected.empty()) { err = "CUDA" + std::to_string(device) + " experts: no unclaimed experts remain"; close(); return false; }
+    std::vector<int64_t> sizes;
+    if (lay.native) {
+        sizes.reserve(selected.size());
+        for (const auto& pair : selected) sizes.push_back((int64_t) lay.blob_bytes(pair.first));
+    }
+    size_t free_bytes = 0, total_bytes = 0;
+    if (!check(cudaMemGetInfo(&free_bytes, &total_bytes), "free memory", err, device)) { close(); return false; }
+    uint64_t needed = 0;
+    for (const auto& pair : selected)
+        needed += lay.native ? (lay.blob_bytes(pair.first) + 255) / 256 * 256 : lay.max_blob;
+    // Leave room for the CUDA context, staging and later driver allocations, especially under WDDM.
+    if (needed + (512ull << 20) > free_bytes) {
+        err = "CUDA" + std::to_string(device) + " experts: slots leave less than 512 MiB free; reduce --expert-cache-device" + std::to_string(device);
+        close(); return false;
+    }
+    const bool cache_ok = lay.native ? cache_.open_sized(sizes, layers, experts, err)
+                                     : cache_.open((int64_t) selected.size(), layers, experts,
+                                                   (int64_t) lay.max_blob, err);
+    if (!cache_ok) { err = "CUDA" + std::to_string(device) + " experts: " + err; close(); return false; }
+    for (const auto& pair : selected) {
+        const int32_t slot = cache_.admit(pair.first, pair.second);
+        const uint8_t* blob = source.blob(pair.first, pair.second);
+        if (slot < 0 || !blob || !cache_.fill_slot_blocking(slot, blob, err, (int64_t) lay.blob_bytes(pair.first))) {
+            err = "CUDA" + std::to_string(device) + " experts: " +
+                  (err.empty() ? "cache fill failed" : err);
+            close(); return false;
+        }
+    }
+    const auto& first = selected.front();
+    if (!cache_.verify_slot(cache_.slot_of(first.first, first.second), source.blob(first.first, first.second),
+                            err, (int64_t) lay.blob_bytes(first.first))) {
+        err = "CUDA" + std::to_string(device) + " experts: " + err;
+        close(); return false;
+    }
+
+    const size_t scratch = std::max<size_t>(
+        (size_t) strata::kernels::moe_hit_grouped_scratch_bytes(CAP, H, FF),
+        strata::kernels::native_expert_scratch_bytes(CAP, FF));
+    const bool allocated =
+        check(cudaStreamCreateWithFlags(&stream_, cudaStreamNonBlocking), "stream", err, device) &&
+        check(cudaHostAlloc((void**) &h_x_, (size_t) CAP * H * sizeof(float), cudaHostAllocPortable | cudaHostAllocMapped), "input staging", err, device) &&
+        check(cudaHostAlloc((void**) &h_out_, (size_t) CAP * H * sizeof(float), cudaHostAllocPortable | cudaHostAllocMapped), "result staging", err, device) &&
+        check(cudaHostAlloc(&h_meta_, sizeof(RemoteMeta), cudaHostAllocPortable), "metadata staging", err, device) &&
+        check(cudaMalloc((void**) &d_x_, (size_t) CAP * H * sizeof(float)), "input", err, device) &&
+        check(cudaMalloc((void**) &d_out_, (size_t) CAP * H * sizeof(float)), "result", err, device) &&
+        check(cudaMalloc((void**) &d_q8_, (size_t) CAP * (H / 32) * 36), "activation", err, device) &&
+        check(cudaMalloc((void**) &d_scales_, (size_t) CAP * (H / 32) * sizeof(float)), "activation scales", err, device) &&
+        check(cudaMalloc(&d_scratch_, scratch), "scratch", err, device) &&
+        check(cudaMalloc(&d_meta_, sizeof(RemoteMeta)), "group metadata", err, device);
+    if (!allocated) { close(); return false; }
+    // Zero-copy: the helper reads its input from, and writes its compact rows into, the pinned host buffers
+    // directly - two copies fewer per layer, each of which is a PCIe round trip.  STRATA_REMOTE_ZEROCOPY=0 copies.
+    const char* zc = std::getenv("STRATA_REMOTE_ZEROCOPY");
+    zero_copy_ = !(zc && zc[0] == '0') &&
+                 cudaHostGetDevicePointer((void**) &z_x_, h_x_, 0) == cudaSuccess &&
+                 cudaHostGetDevicePointer((void**) &z_out_, h_out_, 0) == cudaSuccess;
+    cudaGetLastError();
+    auto* meta = (RemoteMeta*) d_meta_;
+    d_ptr_ = meta->ptr;
+    d_start_ = meta->start;
+    d_dst_ = meta->dst;
+    d_tok_ = meta->tok;
+    d_count_ = &meta->count;
+    owned_.resize(CAP);
+    layers_present_.assign((size_t) layers, 0);
+    group_of_.resize(CAP);
+    group_id_.reserve(CAP);
+    ptr_.reserve(CAP);
+    start_.reserve(CAP + 1);
+    dst_.reserve(CAP);
+    tok_.reserve(CAP);
+    original_row_.reserve(CAP);
+    computed_ = 0;
+    launched_layers_ = 0;
+    returned_bytes_ = full_row_bytes_ = 0;
+    for (const auto& pair : selected) {
+        layers_present_[(size_t) pair.first] = 1;
+        claimed[(size_t) pair.first * (size_t) experts + (size_t) pair.second] = 1;
+    }
+    return true;
+}
+
+bool RemoteExperts::begin(int64_t layer, const float* x, const int32_t* ids, int64_t n_tok,
+                          int64_t k, const int32_t* kind, const int32_t* primary_res,
+                          std::string& err) {
+    // cumulative host time in here (staging and launches), reported per request by the driver
+    struct Timer { double& acc; std::chrono::steady_clock::time_point t0 = std::chrono::steady_clock::now();
+        ~Timer() { acc += std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - t0).count(); } } timer{ms_begin_};
+    const int64_t n = n_tok * k;
+    if (n <= 0 || n > CAP || n_tok > strata::kernels::cpu::MAXT || k != 10 || layer < 0 ||
+        (size_t) layer >= layers_present_.size() || device_ < 0) {
+        err = "CUDA" + std::to_string(device_) + " experts: invalid layer, routing width or window size";
+        return false;
+    }
+    std::fill(owned_.begin(), owned_.begin() + n, 0);
+    group_id_.clear(); ptr_.clear(); start_.clear(); dst_.clear(); tok_.clear(); original_row_.clear();
+    if (!layers_present_[(size_t) layer]) return true;
+    std::fill(group_of_.begin(), group_of_.begin() + n, -1);
+    for (int64_t i = 0; i < n; ++i) {
+        const int32_t e = ids[i];
+        if ((kind && kind[i] != -1) || e < 0 || e >= n_expert_ ||
+            (primary_res && primary_res[(size_t) layer * (size_t) n_expert_ + (size_t) e] >= 0)) continue;
+        const int32_t slot = cache_.slot_of(layer, e);
+        if (slot < 0) continue;
+        owned_[(size_t) i] = 1;
+        int32_t group = -1;
+        for (size_t g = 0; g < group_id_.size(); ++g)
+            if (group_id_[g] == e) { group = (int32_t) g; break; }
+        if (group < 0) {
+            group = (int32_t) group_id_.size();
+            group_id_.push_back(e);
+            ptr_.push_back((unsigned long long) cache_.device_slot(slot));
+        }
+        group_of_[(size_t) i] = group;
+        ++computed_;
+    }
+    if (group_id_.empty()) return true;
+    for (size_t g = 0; g < group_id_.size(); ++g) {
+        start_.push_back((int32_t) dst_.size());
+        for (int64_t i = 0; i < n; ++i) if (group_of_[(size_t) i] == (int32_t) g) {
+            // The grouped kernels read the activation from `tok`, so their
+            // output row can instead be packed densely for the USB4 return.
+            original_row_.push_back((int32_t) i);
+            dst_.push_back((int32_t) dst_.size());
+            tok_.push_back((int32_t) (i / k));
+        }
+    }
+    start_.push_back((int32_t) dst_.size());
+    // Private pinned buffers survive until this GPU has consumed them. The CPU
+    // pool can write other output rows without a cross-device race.
+    std::memcpy(h_x_, x, (size_t) n_tok * H * sizeof(float));
+    auto* meta = (RemoteMeta*) h_meta_;
+    std::memcpy(meta->ptr, ptr_.data(), ptr_.size() * sizeof(ptr_[0]));
+    std::memcpy(meta->start, start_.data(), start_.size() * sizeof(start_[0]));
+    std::memcpy(meta->dst, dst_.data(), dst_.size() * sizeof(dst_[0]));
+    std::memcpy(meta->tok, tok_.data(), tok_.size() * sizeof(tok_[0]));
+    meta->count = (int32_t) group_id_.size();
+    DeviceScope scope(device_);
+    if (!scope.ok) { err = scope.error(device_); return false; }
+    groups_ = (int32_t) group_id_.size();
+    const cudaStream_t s = stream_;
+    const bool staged =
+        (zero_copy_ || check(cudaMemcpyAsync(d_x_, h_x_, (size_t) n_tok * H * sizeof(float), cudaMemcpyHostToDevice, s), "copy input", err, device_)) &&
+        check(cudaMemcpyAsync(d_meta_, h_meta_, sizeof(RemoteMeta), cudaMemcpyHostToDevice, s), "copy group metadata", err, device_);
+    if (!staged) return false;
+    const auto& lay = strata::kernels::cpu::expert_layout();
+    if (lay.native) {
+        strata::kernels::quantize_q8_1_rows(zero_copy_ ? z_x_ : d_x_, n_tok, H, d_q8_, s);
+        const auto& fmt = lay.fmt[(size_t) layer];
+        auto L = strata::kernels::native_expert_layout(fmt.gu_type, fmt.d_type, fmt.n_embd, fmt.n_ff);
+        strata::kernels::native_expert_grouped(L, d_ptr_, d_start_, d_count_, d_dst_, d_tok_,
+                                               groups_, (int64_t) dst_.size(), d_q8_, d_scratch_, zero_copy_ ? z_out_ : d_out_, s);
+    } else {
+        strata::kernels::quantize_q8_0_scaled(zero_copy_ ? z_x_ : d_x_, d_q8_, d_scales_, n_tok * H, s);
+        strata::kernels::moe_grouped_s2(d_ptr_, d_start_, d_count_, d_dst_, d_tok_,
+                                        groups_, (int64_t) dst_.size(), d_q8_, d_scales_, d_scratch_, zero_copy_ ? z_out_ : d_out_, s);
+    }
+    const uint64_t compact_bytes = (uint64_t) dst_.size() * H * sizeof(float);
+    if (!zero_copy_ && !check(cudaMemcpyAsync(h_out_, d_out_, (size_t) compact_bytes, cudaMemcpyDeviceToHost, s),
+               "copy results", err, device_)) return false;
+    ++launched_layers_;
+    returned_bytes_ += compact_bytes;
+    full_row_bytes_ += (uint64_t) n * H * sizeof(float);
+    return true;
+}
+
+bool RemoteExperts::finish(float* out, std::string& err) {
+    if (group_id_.empty()) return true;
+    DeviceScope scope(device_);
+    if (!scope.ok) { err = scope.error(device_); return false; }
+    const auto w0 = std::chrono::steady_clock::now();
+    if (!check(cudaStreamSynchronize(stream_), "finish", err, device_)) return false;
+    ms_wait_ += std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - w0).count();
+    for (size_t i = 0; i < original_row_.size(); ++i)
+        std::memcpy(out + (size_t) original_row_[i] * H, h_out_ + i * H, (size_t) H * sizeof(float));
+    return true;
+}
+
+} // namespace strata::core

src/core/verify.cpp

@@ -5,9 +5,12 @@
 #endif
 
 #include "strata/core/native_head.hpp"
+#include "strata/core/on_device.hpp"
 #include "strata/kernels/iq_kernels.hpp"
 #include "strata/kernels/cpu/expert_layout.hpp"
 #include "strata/kernels/bf16_gemv.hpp"
+#include "strata/kernels/native_router.hpp"
+#include "strata/kernels/native_moe.hpp"
 #include "strata/kernels/cpu/expert.hpp"
 #include "strata/kernels/elementwise.hpp"
 #include "strata/kernels/fused_gr.hpp"
@@ -32,7 +35,11 @@
 #include "strata/kernels/shared_expert.hpp"
 #include "strata/kernels/verify_kernels.hpp"
 
+#include <algorithm>
 #include <atomic>
+#include <map>
+#include <string>
+#include <vector>
 #include <chrono>
 #include <cmath>
 #include <cstdio>
@@ -127,6 +134,7 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
                     const NativeHead* head, int max_t, std::string& err) {
     g_diag_verifier.store(this);
     diag_verify_fn().store(&diag_active_verifier);
+    cudaGetDevice(&device_);   // a layer split's stage on another GPU: its streams, graphs and buffers live there
     wt_ = &wt;
     g_ = &g;
     ss_ = &ss;
@@ -153,6 +161,12 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
         err = "verify: geometry differs from the artifact's";
         return false;
     }
+    if (le_ < 0) le_ = g.n_layers;
+    if (lb_ < 0 || lb_ >= le_ || le_ > g.n_layers || (lb_ > 0 && hand_in_ == nullptr) ||
+        (le_ < g.n_layers && hand_out_ == nullptr)) {
+        err = "verify: the stage's layer range or its hand-off buffers are wrong";
+        return false;
+    }
     const WeightRef* wo = wt.find("output.weight");
     if (wo == nullptr) { err = "verify: output.weight is missing"; return false; }
     n_vocab_ = wo->ne1;
@@ -174,7 +188,7 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
     // ---- mapped staging
     bool ok = mapped(T * 4, (void**) &h_tok_, (void**) &m_tok_) &&
               mapped(T * strata::kernels::kStepCount * 4, (void**) &h_step_, (void**) &m_step_) &&
-              mapped(T * NH * 4, (void**) &h_pos_, (void**) &m_pos_) &&
+              mapped(T * (NH + NKV + IQ) * 4, (void**) &h_pos_, (void**) &m_pos_) &&
               mapped((2 + T) * 4 + 16, (void**) &h_commit_, (void**) &m_commit_) &&
               mapped(T * N * 4, (void**) &h_ple_, (void**) &m_ple_) &&
               mapped(T * 4 + 16, (void**) &h_out_, (void**) &m_out_) &&
@@ -213,7 +227,7 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
     // ---- the device arena: the same sequence counted, then carved
     auto carve = [&](Bump& b) {
         tok_ = b.take<int32_t>(T); step_ = b.take<int32_t>(T * strata::kernels::kStepCount);
-        pos_ = b.take<int32_t>(T * NH); commit_ = b.take<int32_t>(2 + T);
+        pos_ = b.take<int32_t>(T * (NH + NKV + IQ)); commit_ = b.take<int32_t>(2 + T);
         ple_ = b.take<float>(T * N); emb_ = b.take<float>(T * N); R_ = b.take<float>(T * HC * N);
         mixed_ = b.take<float>(T * N); bo_ = b.take<float>(T * N);
         inj_ = b.take<float>(T * HC); inj2_ = b.take<float>(T * HC);
@@ -252,6 +266,12 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
         return false;
     }
     cudaMemset(arena_, 0, count.used);
+    prof_on_ = std::getenv("STRATA_VERIFY_PROFILE") != nullptr;
+    if (prof_on_) {
+        const size_t np = (size_t) g.n_layers * kProfPer + 4;
+        if (cudaMalloc((void**) &prof_, np * 8) != cudaSuccess) { prof_on_ = false; prof_ = nullptr; cudaGetLastError(); }
+        else { cudaMemset(prof_, 0, np * 8); prof_h_.assign(np, 0); }
+    }
     Bump real;
     real.base = (uint8_t*) arena_;
     carve(real);
@@ -266,6 +286,21 @@ bool Verifier::init(const WeightTable& wt, const ModelGeometry& g, SessionState&
         err = "verify: stream create failed";
         return false;
     }
+    // E-6: a layer whose routed experts are all resident is planned on the device (STRATA_VERIFY_DEVICE_PLAN=1: on;
+    // exact, but neutral on RIBPC 1-2 GPUs: off by default)
+    {
+        const char* v = std::getenv("STRATA_VERIFY_DEVICE_PLAN");
+        device_plan_ = v != nullptr && std::atoi(v) != 0;
+    }
+    if (device_plan_) {
+        bool ok2 = cudaMalloc((void**) &skip_, 64) == cudaSuccess && cudaMemset(skip_, 0, 64) == cudaSuccess;
+        if (ok2 && hits.slot_off != nullptr && hits.n_slots > 0) {
+            ok2 = cudaMalloc((void**) &slot_off_d_, (size_t) hits.n_slots * sizeof(unsigned long long)) == cudaSuccess &&
+                  cudaMemcpy(slot_off_d_, hits.slot_off, (size_t) hits.n_slots * sizeof(unsigned long long),
+                             cudaMemcpyHostToDevice) == cudaSuccess;
+        }
+        if (!ok2) { cudaGetLastError(); device_plan_ = false; }
+    }
     std::fprintf(stderr, "strata verify: window up to %d tokens, %.1f MiB of device buffers\n", max_t,
                  (double) count.used / 1048576.0);
     return true;
@@ -297,20 +332,33 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
                                 (uint64_t) g.ssm_conv_channels * (g.ssm_d_conv - 1);
     const int64_t HS = (int64_t) NG_HIST * NG_HC_DIM;
     const int64_t TS = (s.idx_block - 1) * ID;
-    const bool ple_on = ss.ple.ready();
+    const bool ple_on = ss.ple.ready() && ple_stage();
     auto Rt = [&](int t) { return R_ + (size_t) t * HC * N; };
     const int G = (split_ && T >= 2) ? 2 : 1;
+    static const bool dec_batch = [] { const char* v = std::getenv("STRATA_DEC_BATCH"); return v == nullptr || std::atoi(v) != 0; }();
+    auto stamp = [&](int64_t l, int i, int grp) { if (prof_on_ && grp == 0) gpu_stamp(prof_, (int) (l * kProfPer + i), cs); };
     const int tb_[2] = {0, (T + 1) / 2}, te_[2] = {G == 2 ? (T + 1) / 2 : T, T};
     groups_[T] = G;
 
     // ---- the window's inputs, from mapped staging
     copy_i32_from_mapped(tok_, m_tok_, T, cs);
     copy_i32_from_mapped(step_, m_step_, (int64_t) T * kStepCount, cs);
-    copy_i32_from_mapped(pos_, m_pos_, (int64_t) T * NH, cs);
+    copy_i32_from_mapped(pos_, m_pos_, (int64_t) MT * (NH + NKV + IQ), cs);
+    // per-ROW positions of the K rows [t][NKV] and the indexer query rows [t][IQ] (for batched RoPE)
+    const int32_t* pos_k = pos_ + MT * NH;
+    const int32_t* pos_i = pos_ + MT * (NH + NKV);
     if (ple_on) copy_from_mapped(ple_, m_ple_, (int64_t) T * N, cs);
 
-    // ---- the embeddings, broadcast to the hc streams
-    if (const NativeEmbed* ne = native_embed()) {       // plan v0.3 P6: the GGUF-form table
+    // ---- the embeddings, broadcast to the hc streams - or, in a later stage of a layer split, the previous stage's
+    // residual, pending write and inject (see set_stage)
+    const int64_t HB = Verifier::handoff_floats(g);
+    if (lb_ > 0) {
+        for (int t = 0; t < T; ++t) {
+            copy_from_mapped(Rt(t), hand_in_ + (size_t) t * HB, HC * N, cs);
+            copy_from_mapped(bo_ + (size_t) t * N, hand_in_ + (size_t) t * HB + HC * N, N, cs);
+            copy_from_mapped(inj2_ + (size_t) t * HC, hand_in_ + (size_t) t * HB + HC * N + N, HC, cs);
+        }
+    } else if (const NativeEmbed* ne = native_embed()) {       // plan v0.3 P6: the GGUF-form table
         ne->gather_dev(tok_, T, emb_, cs);
         broadcast_streams(emb_, R_, N, (int) HC, T, cs);
     } else {
@@ -342,6 +390,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
     // ---------------------------------------------------------------- pre(l, group): up to the ring
     auto pre = [&](int64_t l, int grp) -> bool {
         const int tb = tb_[grp], te = te_[grp], n = te - tb;
+        stamp(l, 0, grp);
         const LayerView v(wt, l);
         const char* pfx[2] = {"hc_attn_", "hc_ffn_"};
         const WeightRef *wn[2], *wd[2], *wu[2], *wi[2];
@@ -384,9 +433,11 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
                 a.eps = EPS; a.lo = lo_ + t * g.hc_lr; a.rs = rs_ + t * HC;
                 a.inject_out = inj_out + t * HC; a.mixed = mixed_ + t * N;
             }
-            fused_gr_read_multi(fa, n, xn_ + (size_t) tb * HC * N, cs);
+            fused_gr_read_multi(fa, n, xn_ + (size_t) tb * HC * N, cs, (prof_on_ && grp == 0) ? prof_ : nullptr,
+                                (int) (l * kProfPer + (half == 0 ? 27 : 30)));
         };
         gr_read_group(0, pending, inj2_, inj_);
+        stamp(l, 1, grp);
         float* xm = mixed_ + tb * N;
         try {
             if (!is_qsa_layer(g, l)) {
@@ -409,14 +460,19 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
                 float* beta = beta_L_ + (size_t) gi * MT * HV;
                 native_quantize_q8_1(xm, xq_, (int) N, n, cs);
                 native_mmvq(wqkv->native_type, wqkv->native_data, xq_, qkv + (size_t) tb * C, (int) N, (int) C, n, cs);
+                stamp(l, 2, grp);
                 gdn_conv_l2_multi(conv, qkv, (const float*) wc->data, hb, (int) C, (int) (2 * HK), EPS, n, cs, tb);
+                stamp(l, 3, grp);
                 gdn_ab_multi(xm, (const uint16_t*) wa->data, (const uint16_t*) wb->data, (const float*) wdt->data,
                              (const float*) wsa->data, gate + (size_t) tb * HV, beta + (size_t) tb * HV, (int) N, (int) HV,
                              n, cs);
+                stamp(l, 4, grp);
                 native_mmvq(wg->native_type, wg->native_data, xq_, z_ + (size_t) tb * ZV, (int) N, (int) ZV, n, cs);
+                stamp(l, 5, grp);
                 // the recurrence from the untouched state over tokens [0, te); outputs only for this group's
                 gdn_step_norm_multi(state, hb, (int) C, gate, beta, z_, (const float*) wnm->data, EPS, y_, (int) HK,
                                     (int) HV, te, nullptr, cs, tb);
+                stamp(l, 6, grp);
                 native_quantize_q8_1(y_ + (size_t) tb * ZV, xq_, (int) ZV, n, cs);
                 native_mmvq(wout->native_type, wout->native_data, xq_, bo_ + tb * N, (int) ZV, (int) N, n, cs);
             } else {
@@ -439,16 +495,23 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
                     else rope_neox_apply(data, data, rows, cols, (int) s.n_rot, st.cos_tab, st.sin_tab, pos, cs);
                 };
                 float* idx_raw = idx_raw_L_ + (size_t) qi * MT * ID;
+                // the per-token GEMVs / norms / RoPEs / copies of this layer as one launch over the
+                // window's rows each - row-wise identical arithmetic (STRATA_DEC_BATCH=0: token by token)
+                const bool qb = dec_batch && n > 1 && native_qsa_enabled() && native_rope_enabled() && !st.kv_q4;
                 native_quantize_q8_1(xm, xq_, (int) N, n, cs);
-                for (int t = tb; t < te; ++t)
+                if (qb) bf16_gemv_fp32_mmvf_multi(mixed_ + tb * N, N, (const uint16_t*) wik->data, idx_raw + tb * ID, ID, N, ID, n, cs);
+                else for (int t = tb; t < te; ++t)
                     bf16_gemv_fp32_mmvf(mixed_ + t * N, (const uint16_t*) wik->data, idx_raw + t * ID, (int) N, (int) ID, cs);
+                stamp(l, 7, grp);
                 native_mmvq(wk->native_type, wk->native_data, xq_, kcur_ + tb * NKV * HD, (int) N, (int) (NKV * HD), n, cs);
                 native_mmvq(wv->native_type, wv->native_data, xq_, vcur_ + tb * NKV * HD, (int) N, (int) (NKV * HD), n, cs);
-                for (int t = tb; t < te; ++t) norm_rope(kcur_ + t * NKV * HD, wkn, (int) NKV, (int) HD, pos_ + t * NH);
+                if (qb) norm_rope(kcur_ + tb * NKV * HD, wkn, (int) (n * NKV), (int) HD, pos_k + tb * NKV);
+                else for (int t = tb; t < te; ++t) norm_rope(kcur_ + t * NKV * HD, wkn, (int) NKV, (int) HD, pos_ + t * NH);
                 if (st.kv_q4) {   // Q4_0 KV (kv_q4.hpp): K and V rotated before they are stored
                     fwht256_inplace_cuda(kcur_ + tb * NKV * HD, (int64_t) n * NKV, cs);
                     fwht256_inplace_cuda(vcur_ + tb * NKV * HD, (int64_t) n * NKV, cs);
                 }
+                stamp(l, 8, grp);
                 if (grp == 0) copy_from_mapped(tail_snap_ + (size_t) qi * TS, st.idx_tail, TS, cs);
                 for (int t = tb; t < te; ++t) {
                     const int32_t* step_t = step_ + t * kStepCount;
@@ -467,8 +530,20 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
                     native_qsa_indexer_append(idx_raw + t * ID, step_ + t * kStepCount + kStepPos, 0,
                                               (const float*) wikn->data, EPS, ib, s, st.max_cells,
                                               (float) qsa_freq_base(), cs);
+                stamp(l, 9, grp);
                 native_mmvq(wq->native_type, wq->native_data, xq_, qfull_ + tb * NH * 2 * HD, (int) N, (int) (NH * 2 * HD),
                             n, cs);
+                if (qb) {
+                    if (cudaMemcpy2DAsync(qcur_ + tb * NH * HD, (size_t) HD * 4, qfull_ + tb * NH * 2 * HD, (size_t) HD * 2 * 4,
+                                          (size_t) HD * 4, (size_t) (n * NH), cudaMemcpyDeviceToDevice, cs) != cudaSuccess) {
+                        err = "verify: the q/gate split failed";
+                        return false;
+                    }
+                    norm_rope(qcur_ + tb * NH * HD, wqn, (int) (n * NH), (int) HD, pos_ + tb * NH);
+                    bf16_gemv_fp32_mmvf_multi(mixed_ + tb * N, N, (const uint16_t*) wiq->data, qidx_ + tb * IQ * ID, IQ * ID,
+                                              N, IQ * ID, n, cs);
+                    norm_rope(qidx_ + tb * IQ * ID, wiqn, (int) (n * IQ), (int) ID, pos_i + tb * IQ);
+                } else {
                 for (int t = tb; t < te; ++t) {
                     float* qc = qcur_ + t * NH * HD;
                     if (cudaMemcpy2DAsync(qc, (size_t) HD * 4, qfull_ + t * NH * 2 * HD, (size_t) HD * 2 * 4,
@@ -484,16 +559,24 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
                     bf16_gemv_fp32_mmvf(mixed_ + t * N, (const uint16_t*) wiq->data, qx, (int) N, (int) (IQ * ID), cs);
                     norm_rope(qx, wiqn, (int) IQ, (int) ID, pos_ + t * NH);
                 }
+                }
+                stamp(l, 10, grp);
                 qsa_block_scores(st.idx_pooled, st.idx_dead, qidx_ + tb * IQ * ID, step_ + tb * kStepCount, n, max_blocks_,
                                  s, scores_ + (size_t) tb * max_blocks_, cs);
                 qsa_block_topk(scores_ + (size_t) tb * max_blocks_, step_ + tb * kStepCount, n, max_blocks_, cap_, s,
                                sel_ + (size_t) tb * cap_, cs);
+                stamp(l, 11, grp);
                 // KV streaming: the n selections' blocks resident (device-side, inside the graph)
                 qsa_kv_resolve(st, *g_, sel_ + (size_t) tb * cap_, step_ + tb * kStepCount, n, cap_, cs);
+                stamp(l, 12, grp);
                 const QsaAttnPools pools = qsa_attn_pools(st);
                 qsa_decode_attn_batch(qcur_ + tb * NH * HD, pools, sel_ + (size_t) tb * cap_, step_ + tb * kStepCount, cap_,
                                       s, attn_scratch_ + (size_t) tb * attn_scratch_floats_, attn_ + tb * NH * HD, n, cs);
+                stamp(l, 13, grp);
                 if (st.kv_q4) fwht256_inplace_cuda(attn_ + tb * NH * HD, (int64_t) n * NH, cs);   // back: H^-1 = H
+                if (qb) native_qsa_gate_apply(attn_ + tb * NH * HD, qfull_ + tb * NH * 2 * HD, attn32_ + tb * NH * HD,
+                                              (int) (n * NH), (int) HD, cs);
+                else
                 for (int t = tb; t < te; ++t) {
                     if (native_qsa_enabled())
                         native_qsa_gate_apply(attn_ + t * NH * HD, qfull_ + t * NH * 2 * HD, attn32_ + t * NH * HD,
@@ -501,6 +584,7 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
                     else
                         qsa_gate_apply_f32(attn_ + t * NH * HD, qfull_ + t * NH * 2 * HD, s, attn32_ + t * NH * HD, cs);
                 }
+                stamp(l, 14, grp);
                 native_quantize_q8_1(attn32_ + tb * NH * HD, xq_, (int) (NH * HD), n, cs);
                 native_mmvq(wo->native_type, wo->native_data, xq_, bo_ + tb * N, (int) (NH * HD), (int) N, n, cs);
             }
@@ -508,14 +592,31 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
             err = "verify layer " + std::to_string(l) + ": " + e.what();
             return false;
         }
+        stamp(l, 16, grp);
         gr_read_group(1, true, inj_, inj2_);
+        // the window's rows routed in 2 launches (one router GEMV reading the weight once, one
+        // top-10) instead of 2 per token; every row's arithmetic is the single-token call's (STRATA_DEC_BATCH=0: old)
+        const WeightRef* w_router = v.get("ffn_gate_inp.weight");
+        if (dec_batch && n > 1 && w_router != nullptr && native_router_enabled() && NE == 512 && K == 10) {
+            try {
+                bf16_gemv_fp32_mmvf_multi(mixed_ + tb * N, N, (const uint16_t*) w_router->data, logits_ + tb * NE, NE, N,
+                                          NE, n, cs);
+                native_router_top10_multi(logits_ + tb * NE, ids_ + tb * K, w_ + tb * K, n, cs);
+            } catch (const std::exception& e) { err = "verify router: " + std::string(e.what()); return false; }
+        } else
         for (int t = tb; t < te; ++t) {
             MoEBuffers mb = ss.moe;
             mb.logits = logits_ + t * NE; mb.ids = ids_ + t * K; mb.weights = w_ + t * K;
             if (!moe_route(wt, g, l, K, mb, mixed_ + t * N, cs, err, nullptr)) return false;
         }
+        if (device_plan_)   // E-6: every routed expert resident: this group's plan without the host
+            resident_plan(ids_ + tb * K, n * (int) K, (int) K, hits_.d_res + l * g.n_expert, (int) g.n_expert,
+                          hits_.cache_base, slot_off_d_, (long long) hits_.blob,
+                          plan_ + (size_t) grp * (size_t) (plan_i32_ + 16), (long long) max_t_ * K, skip_ + grp,
+                          (uint32_t) ((l - lb_) * G + grp + 1), cs);
         doorbell_publish(xm, ids_ + tb * K, w_ + tb * K, (int64_t) n * N, (int64_t) n * K, m_x_ + tb * N,
                          m_ids_ + tb * K, m_w_ + tb * K, m_seq_, cs);
+        stamp(l, 17, grp);
         {
             const WeightRef *wgi = need(v, "ffn_gate_inp_shexp.weight", err), *wsg = need(v, "ffn_gate_shexp.weight", err),
                             *wsu = need(v, "ffn_up_shexp.weight", err), *wsd = need(v, "ffn_down_shexp.weight", err);
@@ -528,7 +629,8 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
             nsw.up_type = wsu->native_type; nsw.up_data = wsu->native_data;
             nsw.down_type = wsd->native_type; nsw.down_data = wsd->native_data;
             nsw.q8_1 = xq_;
-            for (int t = tb; t < te; ++t) f32_to_bf16_bulk(mixed_ + t * N, sh_bf16_ + t * N, N, cs);
+            if (dec_batch) f32_to_bf16_bulk(mixed_ + tb * N, sh_bf16_ + tb * N, (int64_t) n * N, cs);   // contiguous rows
+            else for (int t = tb; t < te; ++t) f32_to_bf16_bulk(mixed_ + t * N, sh_bf16_ + t * N, N, cs);
             try {
                 shared_expert_multi(n, xm, sh_bf16_ + tb * N, nsw, (const uint16_t*) wgi->data, sh_gate_ + (size_t) tb * g.n_ff,
                                     sh_up_ + (size_t) tb * g.n_ff, sh_g_ + tb, shared_ + tb * N, N, g.n_ff, cs);
@@ -541,17 +643,24 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
             quantize_q8_1_rows(xm, n, N, nat_xq_ + (size_t) tb * (N / 32) * 36, cs);
         else
             quantize_q8_0_scaled(xm, hit_xq_ + (size_t) tb * (N / 32) * 34, hit_xs_ + (size_t) tb * (N / 32), (int64_t) n * N, cs);
+        stamp(l, 18, grp);
         return true;
     };
 
     // ---------------------------------------------------------------- post(l, group): experts, combine
     auto post = [&](int64_t l, int grp) -> bool {
         const int tb = tb_[grp], te = te_[grp], n = te - tb;
-        const uint32_t ring = (uint32_t) (l * G + grp + 1);
-        wait_flag_ge(m_flagA_, ring, cs);                      // the pool published this group's GPU plan
+        const uint32_t ring = (uint32_t) ((l - lb_) * G + grp + 1);
         const int64_t cap = (int64_t) n * K, capx = (int64_t) max_t_ * K;
         int32_t* pl = plan_ + (size_t) grp * (size_t) (plan_i32_ + 16);
-        copy_i32_from_mapped(pl, m_plan_ + (size_t) grp * (size_t) plan_i32_, plan_i32_, cs);
+        if (device_plan_) {   // E-6: skipped when the device planned this group (all its experts resident)
+            wait_flag_ge_or(m_flagA_, ring, skip_ + grp, cs);
+            copy_i32_from_mapped_unless(pl, m_plan_ + (size_t) grp * (size_t) plan_i32_, plan_i32_, skip_ + grp, ring, cs);
+        } else {
+            wait_flag_ge(m_flagA_, ring, cs);                  // the pool published this group's GPU plan
+            copy_i32_from_mapped(pl, m_plan_ + (size_t) grp * (size_t) plan_i32_, plan_i32_, cs);
+        }
+        stamp(l, 19, grp);
         const int32_t* p_counts = pl;
         const int32_t* p_start = pl + 4;
         const int32_t* p_dst = p_start + capx + 1;
@@ -576,22 +685,43 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
             }
         };
         grouped(p_ptr, p_start, p_counts);
-        wait_flag_ge(m_flagB_, ring, cs);                      // the PCIe share is in staging (DMA) or mapped
+        stamp(l, 20, grp);
+        if (device_plan_) wait_flag_ge_or(m_flagB_, ring, skip_ + grp, cs);
+        else wait_flag_ge(m_flagB_, ring, cs);                 // the PCIe share is in staging (DMA) or mapped
         if (sink_.pcie_mode == 2) {                            // stage it with a copy kernel, then point at staging
             const int64_t per = G == 2 ? kStagingBlobs / 2 : kStagingBlobs;
             uint8_t* stage = staging_ + (size_t) (grp * per) * lay.max_blob;
             fetch_blobs(p_ptr2, p_counts + 2, stage, (int64_t) lay.blob_bytes(l), (int) per, cs);
             rebase_ptrs((unsigned long long*) p_ptr2, p_counts + 2, stage, (int64_t) lay.blob_bytes(l), cs);
         }
+        stamp(l, 21, grp);
         grouped(p_ptr2, p_start2, p_counts + 2);
-        wait_flag_ge(m_flag_, ring, cs);                       // the CPU's share is in the mapped rows
-        copy_from_mapped(parts_ + (size_t) tb * K * N, m_ymiss_ + (size_t) tb * K * N, (int64_t) n * K * N, cs);
+        stamp(l, 22, grp);
+        if (device_plan_) {   // no CPU share when the device planned the group: its rows are zeros
+            wait_flag_ge_or(m_flag_, ring, skip_ + grp, cs);
+            copy_or_zero_from_mapped(parts_ + (size_t) tb * K * N, m_ymiss_ + (size_t) tb * K * N, (long long) n * K * N,
+                                     skip_ + grp, ring, cs);
+        } else {
+            wait_flag_ge(m_flag_, ring, cs);               // the CPU's share is in the mapped rows
+            stamp(l, 23, grp);
+            if (dec_batch)   // only the CPU rows cross PCIe (p_dst[0, counts[1]) = the GPU's own rows)
+                copy_rows_from_mapped(parts_ + (size_t) tb * K * N, m_ymiss_ + (size_t) tb * K * N, (int64_t) n * K, N,
+                                      p_dst, p_counts + 1, cs);
+            else
+                copy_from_mapped(parts_ + (size_t) tb * K * N, m_ymiss_ + (size_t) tb * K * N, (int64_t) n * K * N, cs);
+        }
         moe_hit_add(parts_ + (size_t) tb * K * N, hit_out, p_dst, p_counts + 1, cap, N, cs);
+        if (dec_batch && n > 1 && native_moe_combine_enabled()) {   // one launch for the window's rows
+            try {
+                native_moe_combine_multi(parts_ + (size_t) tb * K * N, w_ + tb * K, shared_ + tb * N, bo_ + tb * N, N, K, n, cs);
+            } catch (const std::exception& e) { err = "verify combine: " + std::string(e.what()); return false; }
+        } else
         for (int t = tb; t < te; ++t) {
             MoEBuffers mb = ss.moe;
             mb.weights = w_ + t * K; mb.shared = shared_ + t * N;
             if (!moe_combine_parts(g, l, K, mb, parts_ + (size_t) t * K * N, bo_ + t * N, cs, err)) return false;
         }
+        stamp(l, 24, grp);
         if (l == g.n_layers - 1) {
             for (int t = tb; t < te; ++t) gr_write(Rt(t), bo_ + t * N, inj2_ + t * HC, gs, Rt(t), cs);
             if (cvec().covers(l)) cvec_apply(Rt(tb), l, n, HC * N, nullptr, 0, nullptr, 0, false, cs);
@@ -602,14 +732,23 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
     };
 
     for (int grp = 0; grp < G; ++grp)
-        if (!pre(0, grp)) return false;
-    for (int64_t l = 0; l < g.n_layers; ++l)
+        if (!pre(lb_, grp)) return false;
+    for (int64_t l = lb_; l < le_; ++l)
         for (int grp = 0; grp < G; ++grp) {
             if (!post(l, grp)) return false;
-            if (l + 1 < g.n_layers && !pre(l + 1, grp)) return false;
+            if (l + 1 < le_ && !pre(l + 1, grp)) return false;
         }
+    if (le_ < g.n_layers) {   // a layer split's earlier stage: hand the residual on, no head
+        for (int t = 0; t < T; ++t) {
+            copy_from_mapped(hand_out_ + (size_t) t * HB, Rt(t), HC * N, cs);
+            copy_from_mapped(hand_out_ + (size_t) t * HB + HC * N, bo_ + (size_t) t * N, N, cs);
+            copy_from_mapped(hand_out_ + (size_t) t * HB + HC * N + N, inj2_ + (size_t) t * HC, HC, cs);
+        }
+        return true;
+    }
 
     // ---- the head, T columns, and the argmax of each
+    stamp(g.n_layers, 0, 0);
     {
         const WeightRef *hn = wt.find("output_hc_norm.weight"), *hd = wt.find("output_hc_down.weight"),
                         *hu = wt.find("output_hc_up.weight");
@@ -641,9 +780,36 @@ bool Verifier::record_window(int T, cudaStream_t cs, std::string& err) {
         sp.temperature = 0.0f;
         sample_tokens(head_logits_, T, (int) n_vocab_, nullptr, 0, sp, m_out_, cs);
     }
+    stamp(g.n_layers, 1, 0);
     return true;
 }
 
+std::string Verifier::profile_report() {
+    if (!prof_on_ || prof_windows_ == 0) return std::string();
+    static const char* names[kProfPer] = {"-", "hc-read0", "q8+qkv/q-idx gemv", "conv", "ab", "z", "rec", "q8+kv-idx",
+                                          "k/v+norm-rope", "kv+idx append", "q+q-idx", "scores+topk", "kv-resolve",
+                                          "attention", "gate", "", "out-proj", "hc-read1+router", "shared+quant",
+                                          "waitA", "VRAM hits", "waitB", "PCIe grp", "waitCPU", "copy+combine",
+                                          "(gap)", "head", "  hc0 norm", "  hc0 down", "  hc0 up", "", ""};
+    std::string out;
+    char b[80];
+    double total = 0;
+    for (int k = 0; k < 2; ++k) {
+        out += k == 0 ? " GDN layers:" : " | QSA layers:";
+        for (int i = 0; i < kProfPer; ++i) {
+            if (prof_sum_[k][i] <= 0) continue;
+            total += prof_sum_[k][i];
+            std::snprintf(b, sizeof b, " %s %.2f", names[i], prof_sum_[k][i] / 1e6 / (double) prof_windows_);
+            out += b;
+        }
+    }
+    std::snprintf(b, sizeof b, " | total %.2f ms/window over %lld windows", total / 1e6 / (double) prof_windows_, (long long) prof_windows_);
+    out += b;
+    for (auto& r : prof_sum_) for (double& d : r) d = 0;
+    prof_windows_ = 0;
+    return out;
+}
+
 bool Verifier::capture(int T, std::string& err) {
     if (exec_[T] != nullptr) return true;
     if (cudaStreamBeginCapture(cs_, cudaStreamCaptureModeThreadLocal) != cudaSuccess) {
@@ -663,6 +829,33 @@ bool Verifier::capture(int T, std::string& err) {
         err = std::string("verify: end capture: ") + cudaGetErrorString(ce);
         return false;
     }
+    if (std::getenv("STRATA_VERIFY_NODES") != nullptr) {   // what the window graph holds
+        size_t nn = 0;
+        cudaGraphGetNodes(graph, nullptr, &nn);
+        std::vector<cudaGraphNode_t> nodes(nn);
+        cudaGraphGetNodes(graph, nodes.data(), &nn);
+        std::map<std::string, int> kinds;
+        for (cudaGraphNode_t nd : nodes) {
+            cudaGraphNodeType ty;
+            cudaGraphNodeGetType(nd, &ty);
+            std::string name = "type" + std::to_string((int) ty);
+            if (ty == cudaGraphNodeTypeKernel) {
+                cudaKernelNodeParams kp{};
+                if (cudaGraphKernelNodeGetParams(nd, &kp) == cudaSuccess) {
+                    const char* fn = nullptr;
+                    if (cudaFuncGetName(&fn, kp.func) == cudaSuccess && fn) name = fn;
+                }
+            } else if (ty == cudaGraphNodeTypeMemcpy) name = "memcpy";
+            else if (ty == cudaGraphNodeTypeMemset) name = "memset";
+            ++kinds[name];
+        }
+        std::vector<std::pair<int, std::string>> v;
+        for (auto& [k2, c] : kinds) v.push_back({c, k2});
+        std::sort(v.rbegin(), v.rend());
+        std::fprintf(stderr, "strata verify: the %d-token window graph has %zu nodes:", T, nn);
+        for (size_t i = 0; i < v.size() && i < 40; ++i) std::fprintf(stderr, " %d x %.60s;", v[i].first, v[i].second.c_str());
+        std::fprintf(stderr, "\n");
+    }
     const cudaError_t ie = cudaGraphInstantiate(&exec_[T], graph, 0);
     cudaGraphDestroy(graph);
     if (ie != cudaSuccess) {
@@ -695,7 +888,8 @@ bool Verifier::capture_commit(std::string& err) {
     try {
         copy_i32_from_mapped(commit_, m_commit_, 2 + MT, cs_);
         int64_t qsa_index = 0, gdn_index = 0;
-        for (int64_t l = 0; l < g.n_layers && ok; ++l) {
+        for (int64_t l = 0; l < lb_; ++l) (is_qsa_layer(g, l) ? qsa_index : gdn_index) += 1;
+        for (int64_t l = lb_; l < le_ && ok; ++l) {
             const LayerView v(*wt_, l);
             if (!is_qsa_layer(g, l)) {
                 const WeightRef* wnm = need(v, "ssm_norm.weight", err);
@@ -721,7 +915,7 @@ bool Verifier::capture_commit(std::string& err) {
                 ++qsa_index;
             }
         }
-        if (ok && ss.ple.ready()) copy_indexed(ss.ple.hist, hist_snap_, HS, commit_ + 1, HS, cs_);
+        if (ok && ss.ple.ready() && ple_stage()) copy_indexed(ss.ple.hist, hist_snap_, HS, commit_ + 1, HS, cs_);
     } catch (const std::exception& e) {
         err = std::string("verify commit: ") + e.what();
         ok = false;
@@ -744,6 +938,7 @@ bool Verifier::capture_commit(std::string& err) {
 bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool, void* user, int32_t* out,
                    std::string& err) {
     using namespace strata::kernels;
+    const OnDevice on_device(device_);
     if (T < 1 || T > max_t_) { err = "verify: window size out of range"; return false; }
     const ModelGeometry& g = *g_;
     SessionState& ss = *ss_;
@@ -756,8 +951,12 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool,
         h_tok_[t] = tokens[t];
         qsa_step_fill(h_step_ + t * kStepCount, pos0 + t, s);
         for (int64_t h = 0; h < g.n_head; ++h) h_pos_[t * g.n_head + h] = (int32_t) (pos0 + t);
+        int32_t* pk = h_pos_ + (size_t) max_t_ * g.n_head;
+        int32_t* pi = pk + (size_t) max_t_ * g.n_head_kv;
+        for (int64_t h = 0; h < g.n_head_kv; ++h) pk[t * g.n_head_kv + h] = (int32_t) (pos0 + t);
+        for (int64_t h = 0; h < g.idx_q_heads; ++h) pi[t * g.idx_q_heads + h] = (int32_t) (pos0 + t);
     }
-    if (ss.ple.ready()) {
+    if (ss.ple.ready() && ple_stage()) {
         uint32_t rows[kVerifyMaxT * PLE_N_HEADS];
         int32_t prev[2] = {ss.ple_prev[0], ss.ple_prev[1]};
         for (int t = 0; t < T; ++t) {
@@ -785,8 +984,8 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool,
     volatile uint32_t* const flag = h_flag_;
     const int G = groups_[T] > 0 ? groups_[T] : 1;
     const int gtb[2] = {0, (T + 1) / 2}, gte[2] = {G == 2 ? (T + 1) / 2 : T, T};
-    for (int64_t k = 0; k < g.n_layers * G; ++k) {
-        const int64_t l = k / G;
+    for (int64_t k = 0; k < (le_ - lb_) * G; ++k) {
+        const int64_t l = lb_ + k / G;
         const int grp = (int) (k % G);
         const uint32_t want = (uint32_t) (k + 1);
         const Clock::time_point a = Clock::now();
@@ -840,10 +1039,36 @@ bool Verifier::run(int T, const int32_t* tokens, int64_t pos0, PoolMultiFn pool,
     if (se != cudaSuccess) { err = std::string("verify: ") + cudaGetErrorString(se); return false; }
     progress_at("verify window: waiting for the expert copies", (int64_t) T);
     cudaStreamSynchronize(copy_);   // no host function of this window may raise flag B in the next one
+    if (prof_on_ && G == 1) {       // the window's GPU stage stamps
+        cudaMemcpy(prof_h_.data(), prof_, prof_h_.size() * 8, cudaMemcpyDeviceToHost);
+        const int64_t L = g.n_layers;
+        auto at = [&](int64_t l, int i) { return prof_h_[(size_t) (l * kProfPer + i)]; };
+        for (int64_t l = 0; l < L; ++l) {
+            const int kind = is_qsa_layer(g, l) ? 1 : 0;
+            unsigned long long prev = at(l, 0);
+            for (int i = 1; i <= 24; ++i) {
+                const unsigned long long x = at(l, i);
+                if (x == 0 || x < prev) continue;
+                prof_sum_[kind][i] += (double) (x - prev);
+                prev = x;
+            }
+            if (l + 1 < L) prof_sum_[kind][25] += (double) (at(l + 1, 0) - at(l, 24));
+            prof_sum_[kind][27] += (double) (at(l, 27) - at(l, 0));    // hc-read0: norm
+            prof_sum_[kind][28] += (double) (at(l, 28) - at(l, 27));   //           down
+            prof_sum_[kind][29] += (double) (at(l, 1) - at(l, 28));    //           up
+            prof_sum_[kind][1] -= (double) (at(l, 1) - at(l, 0));      // (hc-read0 shown split)
+        }
+        prof_sum_[0][26] += (double) (at(L, 1) - at(L, 0));
+        ++prof_windows_;
+    }
     // ---- a sampled or penalized request: the head's sampling again, host-side so its parameters are this call's
     // own (a captured kernel would replay the same draws forever).  Row t's draw is Philox(seed, pos0 + t): tied to
     // the POSITION it samples, not to how the text was cut into windows, so a seed replays the same text whatever
     // the drafts were. Exact: a rejected row's draw is discarded, and no kept decision depends on a reused draw.
+    if (le_ < g.n_layers) {   // a layer split's earlier stage: the hand-off is written (synced above)
+        ++windows;
+        return next_ == nullptr || next_->run(T, tokens, pos0, pool, next_user_, out, err);
+    }
     const bool sampled = !sampling_.greedy && sampling_.temperature > 0.0f;
     if (head_sampling_ && (sampled || hist_d_ != nullptr)) {
         SamplerParams sp = sampling_;
@@ -934,6 +1159,7 @@ void Verifier::publish_plan(void* ctx) {
 }
 
 bool Verifier::commit(int n_keep, std::string& err) {
+    const OnDevice on_device(device_);
     if (n_keep < 1 || n_keep > last_t_) { err = "verify: commit count out of range"; return false; }
     const Clock::time_point t0 = Clock::now();
     h_commit_[0] = n_keep;
@@ -944,12 +1170,13 @@ bool Verifier::commit(int n_keep, std::string& err) {
     if (le != cudaSuccess) { err = std::string("verify: commit launch: ") + cudaGetErrorString(le); return false; }
     const cudaError_t se = cudaStreamSynchronize(cs_);
     if (se != cudaSuccess) { err = std::string("verify: commit: ") + cudaGetErrorString(se); return false; }
-    for (int t = 0; t < n_keep; ++t) {
-        ss_->ple_prev[0] = ss_->ple_prev[1];
-        ss_->ple_prev[1] = last_tokens_[t];
-    }
+    if (ple_stage())   // stages that share one session must advance it once
+        for (int t = 0; t < n_keep; ++t) {
+            ss_->ple_prev[0] = ss_->ple_prev[1];
+            ss_->ple_prev[1] = last_tokens_[t];
+        }
     ms_commit += ms_since(t0);
-    return true;
+    return next_ == nullptr || next_->commit(n_keep, err);
 }
 
 }  // namespace strata::core

src/kernels/cpu/iq_avx512.cpp

@@ -6,7 +6,8 @@
 // `maddubs` and a `madd`.  The arithmetic is ggml's (ggml-cpu/quants.c, the `_generic` references): integer sums
 // per block, times d_x * d_y * the format's constant - only the order of the float additions differs.
 //
-// Formats: IQ2_XXS (16), IQ2_XS (17), IQ3_XXS (18), IQ3_S (21), IQ2_S (22).  IQ1_M stays on ggml-cpu.
+// Formats: IQ2_XXS (16), IQ2_XS (17), IQ3_XXS (18), IQ3_S (21), IQ2_S (22).  IQ1_M stays on ggml-cpu (no shipped
+// model has IQ1_M expert rows: the 'Coder IQ1_M' pack's gate/up are IQ2_S / IQ3_XXS / IQ3_S).
 #include "strata/kernels/cpu/iq_avx512.hpp"
 
 #define GGML_COMMON_DECL_CPP
@@ -16,6 +17,7 @@
 #include <immintrin.h>
 
 #include <cmath>
+#include <cstdlib>
 #include <cstring>
 
 namespace strata::kernels::cpu {
@@ -26,6 +28,16 @@ inline uint32_t u32(const uint8_t* p) { uint32_t v; std::memcpy(&v, p, 4); retur
 inline uint16_t u16(const uint8_t* p) { uint16_t v; std::memcpy(&v, p, 2); return v; }
 inline uint64_t u64(const uint8_t* p) { uint64_t v; std::memcpy(&v, p, 8); return v; }
 
+// E-2: the IQ3 grids by one AVX-512 gather (the same words, the same results) - measured 3-5% SLOWER than the scalar
+// lookups on a Ryzen 5 7600 (Zen 4 gathers are microcoded), so opt-in: STRATA_IQ_GATHER=1
+const bool gather = std::getenv("STRATA_IQ_GATHER") != nullptr;
+// E-2: software prefetch distance in bytes (STRATA_IQ_PREFETCH; 0 = off): -2-3% gate/up time on IQ3_S decode
+// (the rows are read at ~30 GB/s by 6 cores, near what DDR5 with 4 KB pages gives: the pool is memory-bound)
+const int prefetch_ahead = [] {
+    const char* v = std::getenv("STRATA_IQ_PREFETCH");
+    return v ? std::atoi(v) : 2048;
+}();
+
 // lanes 0-7 -> s0, 8-15 -> s1, 16-23 -> s2, 24-31 -> s3 (int16 lanes of a maddubs result: two values each)
 inline __m512i scales4(int s0, int s1, int s2, int s3) {
     const __m512i idx = _mm512_set_epi16(3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1,
@@ -98,7 +110,9 @@ template <> struct Fmt<18> {   // IQ3_XXS: d, qs[64] grid bytes, 8 x u32 (4 x 7-
     static constexpr float K = 0.25f;
     static inline void decode(const uint8_t* b, int j, __m512i& g, __mmask64& m, __m512i& sc) {
         const uint8_t* q = b + 2 + 16 * j;
-        g = _mm512_set_epi32((int) iq3xxs_grid[q[15]], (int) iq3xxs_grid[q[14]], (int) iq3xxs_grid[q[13]], (int) iq3xxs_grid[q[12]],
+        // E-2: one gather of the 16 grid words (the scalar lookups assembled with set_epi32 cost ~30 uops)
+        g = gather ? _mm512_i32gather_epi32(_mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*) q)), iq3xxs_grid, 4)
+                   : _mm512_set_epi32((int) iq3xxs_grid[q[15]], (int) iq3xxs_grid[q[14]], (int) iq3xxs_grid[q[13]], (int) iq3xxs_grid[q[12]],
                              (int) iq3xxs_grid[q[11]], (int) iq3xxs_grid[q[10]], (int) iq3xxs_grid[q[9]], (int) iq3xxs_grid[q[8]],
                              (int) iq3xxs_grid[q[7]], (int) iq3xxs_grid[q[6]], (int) iq3xxs_grid[q[5]], (int) iq3xxs_grid[q[4]],
                              (int) iq3xxs_grid[q[3]], (int) iq3xxs_grid[q[2]], (int) iq3xxs_grid[q[1]], (int) iq3xxs_grid[q[0]]);
@@ -119,12 +133,18 @@ template <> struct Fmt<21> {   // IQ3_S: d, qs[64], qh[8], signs[32], scales[4]
     static inline void decode(const uint8_t* b, int j, __m512i& g, __mmask64& m, __m512i& sc) {
         const uint8_t* q = b + 2 + 16 * j;
         const uint32_t h0 = b[66 + 2 * j], h1 = b[66 + 2 * j + 1];
+        if (gather) {   // E-2: the 9-bit indices (the high bit from qh under a mask), one gather
+            __m512i idx = _mm512_cvtepu8_epi32(_mm_loadu_si128((const __m128i*) q));
+            idx = _mm512_mask_add_epi32(idx, (__mmask16) (h0 | (h1 << 8)), idx, _mm512_set1_epi32(256));
+            g = _mm512_i32gather_epi32(idx, iq3s_grid, 4);
+        } else {
 #define G3(k, h, kk) (int) iq3s_grid[q[k] | (((h >> kk) & 1) << 8)]
         g = _mm512_set_epi32(G3(15, h1, 7), G3(14, h1, 6), G3(13, h1, 5), G3(12, h1, 4),
                              G3(11, h1, 3), G3(10, h1, 2), G3(9, h1, 1), G3(8, h1, 0),
                              G3(7, h0, 7), G3(6, h0, 6), G3(5, h0, 5), G3(4, h0, 4),
                              G3(3, h0, 3), G3(2, h0, 2), G3(1, h0, 1), G3(0, h0, 0));
 #undef G3
+        }
         m = _cvtu64_mask64(u64(b + 74 + 8 * j));
         const uint8_t s = b[106 + j];
         const int sa = 2 * (s & 15) + 1, sb = 2 * (s >> 4) + 1;
@@ -139,6 +159,13 @@ inline void row_dot(const uint8_t* row, int nblocks, const block_q8_K* const* y,
     const __m512i zero = _mm512_setzero_si512();
     for (int i = 0; i < nblocks; ++i) {
         const uint8_t* blk = row + (size_t) i * Fmt<TY>::bytes;
+        // E-2: the weight rows stream from DRAM (4 KB pages when large pages are refused): ask for the bytes a few
+        // blocks ahead (the next row's start included) before the decode needs them
+        if (prefetch_ahead > 0) {
+            const uint8_t* ahead = blk + (size_t) prefetch_ahead;
+            _mm_prefetch((const char*) ahead, _MM_HINT_T0);
+            _mm_prefetch((const char*) ahead + 64, _MM_HINT_T0);
+        }
         __m512i acci[NT];
         for (int t = 0; t < NT; ++t) acci[t] = _mm512_setzero_si512();
         for (int j = 0; j < 4; ++j) {

src/kernels/cuda/cvec.cu

@@ -11,9 +11,33 @@ namespace {
 constexpr int THREADS = 256;
 constexpr int MAXK = 16;   // n_embd up to 4096, held in registers between the dot and the update
 
-Cvec g_cvec;
-int* g_on = nullptr;
+Cvec g_cvec;                 // the description; its device pointers are the uploading device's
 bool g_on_host = false;
+// the tables on every device that holds them (a layer split applies the vector on several)
+constexpr int kDevices = 64;
+struct DevTables { float* dir = nullptr; float* s = nullptr; int* on = nullptr; };
+DevTables g_dev[kDevices];
+std::vector<float> g_dir_host, g_s_host;
+int cur_device() {
+    int d = 0;
+    if (cudaGetDevice(&d) != cudaSuccess || d < 0 || d >= kDevices) d = 0;
+    return d;
+}
+bool upload_here(std::string& err) {
+    DevTables& t = g_dev[cur_device()];
+    if (t.dir != nullptr) return true;
+    const int flag = g_on_host ? 1 : 0;
+    if (cudaMalloc(&t.dir, g_dir_host.size() * sizeof(float)) != cudaSuccess ||
+        cudaMalloc(&t.s, g_s_host.size() * sizeof(float)) != cudaSuccess || cudaMalloc(&t.on, sizeof(int)) != cudaSuccess ||
+        cudaMemcpy(t.dir, g_dir_host.data(), g_dir_host.size() * sizeof(float), cudaMemcpyHostToDevice) != cudaSuccess ||
+        cudaMemcpy(t.s, g_s_host.data(), g_s_host.size() * sizeof(float), cudaMemcpyHostToDevice) != cudaSuccess ||
+        cudaMemcpy(t.on, &flag, sizeof(int), cudaMemcpyHostToDevice) != cudaSuccess) {
+        err = "control vector: device allocation failed";
+        t = DevTables{};
+        return false;
+    }
+    return true;
+}
 
 // the fused hyper-connection read's gate (fused_gr.cu), so a write done here is bitwise the one it would have folded
 __device__ __forceinline__ float sigmoidf_(float x) { return 1.0f / (1.0f + __expf(-x)); }
@@ -78,21 +102,26 @@ bool cvec_upload(const std::vector<float>& dir, const std::vector<float>& s, int
                  int64_t n_embd, int64_t hc, std::string& err) {
     if (n_embd < 1 || n_embd > (int64_t) THREADS * MAXK) { err = "control vector: unsupported n_embd"; return false; }
     if (s.empty() || dir.size() != s.size() * (size_t) n_embd) { err = "control vector: bad table sizes"; return false; }
-    float* d_dir = nullptr;
-    float* d_s = nullptr;
-    int* d_on = nullptr;
-    const int one = 1;
-    if (cudaMalloc(&d_dir, dir.size() * sizeof(float)) != cudaSuccess ||
-        cudaMalloc(&d_s, s.size() * sizeof(float)) != cudaSuccess || cudaMalloc(&d_on, sizeof(int)) != cudaSuccess ||
-        cudaMemcpy(d_dir, dir.data(), dir.size() * sizeof(float), cudaMemcpyHostToDevice) != cudaSuccess ||
-        cudaMemcpy(d_s, s.data(), s.size() * sizeof(float), cudaMemcpyHostToDevice) != cudaSuccess ||
-        cudaMemcpy(d_on, &one, sizeof(int), cudaMemcpyHostToDevice) != cudaSuccess) {
-        err = "control vector: device allocation failed";
-        return false;
+    int prev = 0;   // a new vector replaces the old one on every device
+    cudaGetDevice(&prev);
+    for (int d = 0; d < kDevices; ++d) {
+        if (g_dev[d].dir == nullptr) continue;
+        cudaSetDevice(d);
+        cudaDeviceSynchronize();
+        cudaFree(g_dev[d].dir);
+        cudaFree(g_dev[d].s);
+        cudaFree(g_dev[d].on);
+        g_dev[d] = DevTables{};
     }
-    g_cvec.dir = d_dir;
-    g_cvec.s = d_s;
-    g_cvec.on = d_on;
+    cudaSetDevice(prev);
+    g_dir_host = dir;
+    g_s_host = s;
+    g_on_host = true;
+    if (!upload_here(err)) return false;
+    const DevTables& t = g_dev[cur_device()];
+    g_cvec.dir = t.dir;
+    g_cvec.s = t.s;
+    g_cvec.on = t.on;
     g_cvec.mode = mode;
     g_cvec.first = first;
     g_cvec.last = last;
@@ -100,26 +129,35 @@ bool cvec_upload(const std::vector<float>& dir, const std::vector<float>& s, int
     g_cvec.hc = hc;
     g_cvec.steered.assign(s.size(), false);
     for (size_t l = 0; l < s.size(); ++l) g_cvec.steered[l] = s[l] != 0.0f;
-    g_on = d_on;
-    g_on_host = true;
     return true;
 }
 
+bool cvec_replicate(std::string& err) { return !g_cvec.loaded() || upload_here(err); }
+
 void cvec_set_enabled(bool on) {
-    if (g_on == nullptr || on == g_on_host) return;
-    cudaDeviceSynchronize();   // nothing in flight may still read the flag
+    if (!g_cvec.loaded() || on == g_on_host) return;
+    int prev = 0;
+    cudaGetDevice(&prev);
     const int v = on ? 1 : 0;
-    cudaMemcpy(g_on, &v, sizeof(int), cudaMemcpyHostToDevice);
+    for (int d = 0; d < kDevices; ++d) {
+        if (g_dev[d].on == nullptr) continue;
+        cudaSetDevice(d);
+        cudaDeviceSynchronize();   // nothing in flight may still read the flag
+        cudaMemcpy(g_dev[d].on, &v, sizeof(int), cudaMemcpyHostToDevice);
+    }
+    cudaSetDevice(prev);
     g_on_host = on;
 }
 
-bool cvec_enabled() { return g_on != nullptr && g_on_host; }
+bool cvec_enabled() { return g_cvec.loaded() && g_on_host; }
 
 void cvec_apply(float* R, int64_t layer, int64_t T, int64_t r_ld, const float* bo, int64_t bo_ld, const float* inj,
                 int64_t inj_ld, bool write, void* stream) {
     if (!g_cvec.loaded() || T < 1) return;
+    const DevTables& t = g_dev[cur_device()];
+    if (t.dir == nullptr) throw std::runtime_error("cvec_apply: the control vector is not on this device (cvec_replicate)");
     const dim3 grid((unsigned) g_cvec.hc, (unsigned) T);
-    cvec_kernel<<<grid, THREADS, 0, (cudaStream_t) stream>>>(R, g_cvec.dir, g_cvec.s, g_cvec.on, g_cvec.mode, layer,
+    cvec_kernel<<<grid, THREADS, 0, (cudaStream_t) stream>>>(R, t.dir, t.s, t.on, g_cvec.mode, layer,
                                                             (int) g_cvec.n_embd, (int) g_cvec.hc, r_ld, bo, bo_ld,
                                                             inj, inj_ld, write ? 1 : 0);
     if (cudaPeekAtLastError() != cudaSuccess) throw std::runtime_error("cvec_apply: launch failed");

src/kernels/cuda/elementwise.cu

@@ -225,6 +225,37 @@ __global__ void copy_from_mapped_kernel(float4* __restrict__ dst, const volatile
     }
 }
 
+// the CPU rows of a verify window, skipping the rows the GPU plan computes itself (the
+// pool writes +0.0 into those, so this writes +0.0 too): block = row, the plan's hit rows `dst[0, *count)`.
+__global__ void copy_rows_from_mapped_kernel(float4* __restrict__ dst, const volatile float4* src, int64_t row4,
+                                             const int32_t* __restrict__ hit_rows, const int32_t* __restrict__ count) {
+    const int row = blockIdx.x;
+    __shared__ int hit;
+    if (threadIdx.x == 0) {
+        int h = 0;
+        const int c = *count;
+        for (int i = 0; i < c; ++i) h |= hit_rows[i] == row;
+        hit = h;
+    }
+    __syncthreads();
+    float4* d = dst + (int64_t) row * row4;
+    if (hit) {
+        for (int64_t i = threadIdx.x; i < row4; i += blockDim.x) d[i] = make_float4(0.0f, 0.0f, 0.0f, 0.0f);
+    } else {
+        const volatile float4* sr = src + (int64_t) row * row4;
+        for (int64_t i = threadIdx.x; i < row4; i += blockDim.x) d[i] = const_cast<const float4*>(sr)[i];
+    }
+}
+void copy_rows_from_mapped(float* dst, const float* src, int64_t rows, int64_t width, const int32_t* hit_rows,
+                           const int32_t* count, void* stream) {
+    if (rows <= 0) return;
+    if ((width & 3) != 0 || ((uintptr_t) dst & 15) != 0 || ((uintptr_t) src & 15) != 0) {
+        std::fprintf(stderr, "copy_rows_from_mapped: width must be a multiple of 4 and both pointers 16-byte aligned\n");
+        std::exit(1);
+    }
+    copy_rows_from_mapped_kernel<<<(unsigned) rows, 128, 0, (cudaStream_t) stream>>>((float4*) dst, (const volatile float4*) src,
+                                                                                  width / 4, hit_rows, count);
+}
 void copy_from_mapped(float* dst, const float* src, int64_t n, void* stream) {
     if (n <= 0) return;
     if ((n & 3) != 0 || ((uintptr_t) dst & 15) != 0 || ((uintptr_t) src & 15) != 0) {

src/kernels/cuda/fused_gr.cu

@@ -1,6 +1,7 @@
 // src/kernels/cuda/fused_gr.cu - see include/strata/kernels/fused_gr.hpp.
 #include "strata/kernels/fused_gr.hpp"
 #include "strata/kernels/bf16_bits.hpp"
+#include "strata/kernels/verify_kernels.hpp"
 
 #include <cuda_runtime.h>
 
@@ -295,7 +296,8 @@ __global__ void __launch_bounds__(THREADS) gr_up_multi_kernel(GrMulti m) {
 
 }  // namespace
 
-void fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, void* stream) {
+void fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, void* stream, unsigned long long* stamp_buf,
+                         int stamp_i0) {
     if (n_tok < 1 || n_tok > kFusedGrMaxT || xn_scratch == nullptr) {
         std::fprintf(stderr, "fused_gr_read_multi: invalid arguments\n");
         std::exit(1);
@@ -315,13 +317,24 @@ void fused_gr_read_multi(const FusedGrArgs* a, int n_tok, float* xn_scratch, voi
     m.T = n_tok;
     cudaStream_t st = (cudaStream_t) stream;
     gr_norm_multi_kernel<<<n_tok, THREADS, 0, st>>>(m);
-    static bool attr = false;
-    if (!attr) {
-        cudaFuncSetAttribute(gr_down_multi_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize,
-                             (int) (kFusedGrMaxT * TILE * sizeof(float)));
-        attr = true;
+    if (stamp_buf) gpu_stamp(stamp_buf, stamp_i0, stream);
+    // the shared-memory opt-in is a per-DEVICE setting: once per device, not once per process (a layer split
+    // runs this kernel on two cards)
+    static bool attr[64] = {};
+    int dev = 0;
+    cudaGetDevice(&dev);
+    if (dev < 0 || dev >= 64 || !attr[dev]) {
+        // at most what the card allows (Turing: 64 KB - enough for windows of up to 6 tokens)
+        int optin = 0;
+        cudaDeviceGetAttribute(&optin, cudaDevAttrMaxSharedMemoryPerBlockOptin, dev);
+        int want = (int) (kFusedGrMaxT * TILE * sizeof(float));
+        if (optin > 0 && want > optin) want = optin;
+        cudaFuncSetAttribute(gr_down_multi_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, want);
+        cudaGetLastError();
+        if (dev >= 0 && dev < 64) attr[dev] = true;
     }
     gr_down_multi_kernel<<<DOWN_BLOCKS + 1, THREADS, (size_t) n_tok * TILE * sizeof(float), st>>>(m);
+    if (stamp_buf) gpu_stamp(stamp_buf, stamp_i0 + 1, stream);
     gr_up_multi_kernel<<<UPM_BLOCKS, THREADS, 0, st>>>(m);
     const cudaError_t e = cudaGetLastError();
     if (e != cudaSuccess) {