Layer split spill walk: stage-weight trim for short cards, drafter drop and head on host per card, KV window and fewer-cards steps; launcher retry without a drafter and the engine's own reason as the last line

CMakeLists.txt

@@ -388,6 +388,11 @@ if(STRATA_BUILD_TESTS)
   target_include_directories(stage_reserve_test PRIVATE include)
   add_test(NAME stage_reserve_test COMMAND stage_reserve_test)
 
+  # the layer split's spill walk (header-only arithmetic: who is short, which step comes next)
+  add_executable(split_walk_test tests/program/split_walk_test.cpp)
+  target_include_directories(split_walk_test PRIVATE include)
+  add_test(NAME split_walk_test COMMAND split_walk_test)
+
   # the prompt planner's host-memory model (#2824): header-only, no GPU
   add_executable(prefill_plan_test tests/program/prefill_plan_test.cpp)
   target_include_directories(prefill_plan_test PRIVATE include)

apps/host/scripts/make-df2-locales.py

@@ -193,6 +193,15 @@ add("The all-cards split engine could not start, so the normal layer split runs.
     "O motor de divisão entre todas as placas não conseguiu iniciar, então a divisão normal por camadas está em uso.",
     "모든 카드를 쓰는 분할 엔진을 시작하지 못해 일반 레이어 분할로 실행합니다.",
     "Le moteur réparti sur toutes les cartes n'a pas pu démarrer, la répartition normale par couches est donc utilisée.")
+add("Drafting: off (the cards were short of memory)",
+    "预测草稿：关闭（显卡内存不足）",
+    "先読み：オフ（グラフィックスカードのメモリが不足していました）",
+    "Vorausraten: aus (den Karten fehlte Speicher)",
+    "Predicción: desactivada (a las tarjetas les faltaba memoria)",
+    "Предсказание: выключено (на картах не хватило памяти)",
+    "Previsão: desligada (faltou memória nas placas)",
+    "초안 예측: 꺼짐(카드 메모리가 부족했습니다)",
+    "Anticipation : désactivée (les cartes manquaient de mémoire)")
 here = Path(__file__).resolve().parent.parent / "src" / "locales"
 NBSP = " "
 

apps/host/src/locales/df2-de.json

@@ -17,5 +17,6 @@
  "df16": "Der MTP-Entwurfsbaustein konnte nicht heruntergeladen werden.",
  "df17": "Vorausraten: MTP (DFlash2 ist nicht gestartet)",
  "df18": "Automatisch nutzt die Alle-Karten-Split-Engine mit der MTP-Entwurfsschicht: etwa {0} GB, einmalig herunterzuladen.",
- "df19": "Die Alle-Karten-Split-Engine konnte nicht starten, deshalb läuft die normale Schichtaufteilung."
+ "df19": "Die Alle-Karten-Split-Engine konnte nicht starten, deshalb läuft die normale Schichtaufteilung.",
+ "df20": "Vorausraten: aus (den Karten fehlte Speicher)"
 }

apps/host/src/locales/df2-en.json

@@ -17,5 +17,6 @@
  "df16": "The MTP drafter could not be downloaded.",
  "df17": "Drafting: MTP (DFlash2 didn't start)",
  "df18": "Auto uses the all-cards split engine with the MTP draft layer: about {0} GB to download, once.",
- "df19": "The all-cards split engine could not start, so the normal layer split runs."
+ "df19": "The all-cards split engine could not start, so the normal layer split runs.",
+ "df20": "Drafting: off (the cards were short of memory)"
 }

apps/host/src/locales/df2-es.json

@@ -17,5 +17,6 @@
  "df16": "No se pudo descargar el predictor de MTP.",
  "df17": "Predicción: MTP (DFlash2 no arrancó)",
  "df18": "Automático usa el motor de reparto entre todas las tarjetas con la capa de borrador MTP: unos {0} GB, una sola vez.",
- "df19": "El motor de reparto entre todas las tarjetas no pudo arrancar, así que se usa el reparto normal por capas."
+ "df19": "El motor de reparto entre todas las tarjetas no pudo arrancar, así que se usa el reparto normal por capas.",
+ "df20": "Predicción: desactivada (a las tarjetas les faltaba memoria)"
 }

apps/host/src/locales/df2-fr.json

@@ -17,5 +17,6 @@
  "df16": "Le module d’anticipation MTP n’a pas pu être téléchargé.",
  "df17": "Anticipation : MTP (DFlash2 n’a pas démarré)",
  "df18": "Auto utilise le moteur réparti sur toutes les cartes avec la couche de brouillon MTP : environ {0} Go à télécharger, une seule fois.",
- "df19": "Le moteur réparti sur toutes les cartes n’a pas pu démarrer, la répartition normale par couches est donc utilisée."
+ "df19": "Le moteur réparti sur toutes les cartes n’a pas pu démarrer, la répartition normale par couches est donc utilisée.",
+ "df20": "Anticipation : désactivée (les cartes manquaient de mémoire)"
 }

apps/host/src/locales/df2-ja.json

@@ -17,5 +17,6 @@
  "df16": "MTP のドラフトモデルをダウンロードできませんでした。",
  "df17": "先読み：MTP（DFlash2 は起動しませんでした）",
  "df18": "自動では、全カードのスプリットエンジンと MTP ドラフト層を使います。約 {0} GB を一度だけダウンロードします。",
- "df19": "全カードのスプリットエンジンを起動できなかったため、通常のレイヤー分割で動作しています。"
+ "df19": "全カードのスプリットエンジンを起動できなかったため、通常のレイヤー分割で動作しています。",
+ "df20": "先読み：オフ（グラフィックスカードのメモリが不足していました）"
 }

apps/host/src/locales/df2-ko.json

@@ -17,5 +17,6 @@
  "df16": "MTP 초안 모델을 내려받지 못했습니다.",
  "df17": "초안 예측: MTP(DFlash2가 시작되지 않았습니다)",
  "df18": "자동은 모든 카드를 쓰는 분할 엔진과 MTP 초안 층을 사용합니다(약 {0}GB, 한 번만 내려받음).",
- "df19": "모든 카드를 쓰는 분할 엔진을 시작하지 못해 일반 레이어 분할로 실행합니다."
+ "df19": "모든 카드를 쓰는 분할 엔진을 시작하지 못해 일반 레이어 분할로 실행합니다.",
+ "df20": "초안 예측: 꺼짐(카드 메모리가 부족했습니다)"
 }

apps/host/src/locales/df2-pt.json

@@ -17,5 +17,6 @@
  "df16": "Não foi possível baixar o preditor do MTP.",
  "df17": "Previsão: MTP (o DFlash2 não iniciou)",
  "df18": "O automático usa o motor de divisão entre todas as placas com a camada de rascunho MTP: cerca de {0} GB, uma vez.",
- "df19": "O motor de divisão entre todas as placas não conseguiu iniciar, então a divisão normal por camadas está em uso."
+ "df19": "O motor de divisão entre todas as placas não conseguiu iniciar, então a divisão normal por camadas está em uso.",
+ "df20": "Previsão: desligada (faltou memória nas placas)"
 }

apps/host/src/locales/df2-ru.json

@@ -17,5 +17,6 @@
  "df16": "Не удалось скачать черновую модель MTP.",
  "df17": "Предсказание: MTP (DFlash2 не запустился)",
  "df18": "Авто использует движок с разделением по всем картам и черновой слой MTP: около {0} ГБ, скачивается один раз.",
- "df19": "Движок с разделением по всем картам не запустился, поэтому работает обычное послойное разделение."
+ "df19": "Движок с разделением по всем картам не запустился, поэтому работает обычное послойное разделение.",
+ "df20": "Предсказание: выключено (на картах не хватило памяти)"
 }

apps/host/src/locales/df2-zh.json

@@ -17,5 +17,6 @@
  "df16": "无法下载 MTP 草稿模型。",
  "df17": "预测草稿：MTP（DFlash2 未能启动）",
  "df18": "自动模式使用全部显卡的拆分引擎和 MTP 草稿层：约 {0} GB，只下载一次。",
- "df19": "全显卡拆分引擎未能启动，因此改用常规的分层拆分。"
+ "df19": "全显卡拆分引擎未能启动，因此改用常规的分层拆分。",
+ "df20": "预测草稿：关闭（显卡内存不足）"
 }

include/strata/program/split_walk.hpp

@@ -0,0 +1,92 @@
+// include/strata/program/split_walk.hpp - the spill walk of a layer split: what each card gives up, in which order, when it is short of VRAM.
+//
+// The 0.2.10 walk (draft layer dropped, a loan that keeps 16 slots, the prompt path's own small buffers, the output head in pinned host memory, a smaller KV window)
+// ran for ONE card.  A layer split went straight to "expert cache: no room" or "the GPU has too little free VRAM for the prompt path" (two 8 GB cards, 128K: 6 of 6
+// starts failed).  The arithmetic of the layer split's version is here, pure, so a host test pins it (tests/program/split_walk_test.cpp); src/program/generate.cpp
+// reads the cards and prints the lines.
+//
+// The steps, mildest first, each only when the one before did not leave the card enough:
+//   1. trim: every card loads only the dense weights of ITS layers (--trim-stage-weights).  Without it each card holds the whole 3.5 GiB set (IQ3_S).  Bit-exact in
+//      itself; the cache simply gets the room.  This is the step that matters on 8 GB cards.
+//   2. the KV window down (32768 -> 20480 -> 8192 -> 4096 cells a layer; the rest streams from RAM), when no placement can start the prompt path.
+//   3. the drafter dropped on the card that holds it (draft layer or DFlash2's pack and buffers): decode is slower, the start works.
+//   4. the loan keeps 16 slots, and the prompt path's own small buffers (256..1024-token chunks), instead of 128 and 1024..8192.
+//   5. the output head in pinned host memory, on the card that holds it.
+//   6. fewer cards: a card with too little free memory to hold a share is left out of the split; one card left is the single-card start (with its own walk).
+// The context and the number of requests are never touched.  A big card never reaches any of them: trim_needed() is false when the card has the room.
+#pragma once
+
+#include <algorithm>
+#include <cstdint>
+
+namespace strata::program::split_walk {
+
+constexpr int64_t kContextMib = 1550;          ///< CUDA context, streams, scratch and tables of a card that holds the whole dense set (8 GB hog, IQ3_S: 1559 MiB measured)
+constexpr int64_t kWorkingCacheSlots = 263;    ///< a cache the engine calls working (generate.cpp #496)
+constexpr int64_t kPromptHeadroomMib = 1100;   ///< a 512-token chunk's own buffers (570 MiB) + the serve check's 512 MiB headroom
+constexpr int64_t kMinCardMib = 4608;          ///< under this a card cannot hold even a trimmed share beside the context, a working cache and the prompt buffers
+constexpr int64_t kStageLateAllocMib = 256;    ///< what a stage allocates after its cache that nothing prices (the single card's kLateAllocMib, #4474)
+
+/// What one card needs, from the model and the start (all MiB).
+struct Model {
+    int64_t dense_mib = 0;      ///< the whole dense set a card loads without the trim (pack arena + native projections)
+    int64_t session_mib = 0;    ///< the session of one card's layers at the asked context and KV window
+    int64_t head_mib = 0;       ///< the output head (the last stage only)
+    int64_t drafter_mib = 0;    ///< the draft layer or DFlash2 (the stage that carries it only)
+    int64_t slot_mib = 3;       ///< the largest expert blob, rounded up
+    int64_t reserve_mib = 796;  ///< what the plan keeps free on a later stage (700 + the verify windows)
+};
+
+/// MiB a card needs beyond its free memory figure when it loads the whole dense set: the weights, the context, its session, (the last stage) the head and the drafter,
+/// the reserve, a working cache and the prompt path's headroom.
+inline int64_t untrimmed_need_mib(const Model& m, bool carries_head_and_drafter) {
+    return m.dense_mib + kContextMib + m.session_mib + (carries_head_and_drafter ? m.head_mib + m.drafter_mib : 0) + m.reserve_mib +
+           kWorkingCacheSlots * m.slot_mib + kPromptHeadroomMib;
+}
+
+/// Is this card too short to start without the trim?  (free_mib: its free memory before it loaded anything.)
+inline bool trim_needed(int64_t free_mib, const Model& m, bool carries_head_and_drafter) {
+    return free_mib > 0 && free_mib < untrimmed_need_mib(m, carries_head_and_drafter);
+}
+
+/// May this card take part in a layer split at all?
+inline bool card_usable(int64_t free_mib) { return free_mib >= kMinCardMib; }
+
+/// The single card's rule for dropping the draft layer, unchanged (generate.cpp, "SHORT OF VRAM: THE DRAFT LAYER IS THE LAST THING TO GO"): the layer, the head, a working
+/// cache beside the prompt buffers (214 slots, 543 MiB) and what is allocated after it.  Used for the card that holds the drafter in a layer split too.
+inline uint64_t draft_layer_need_bytes(uint64_t layer_bytes, uint64_t head_bytes) {
+    return layer_bytes + head_bytes + (220ull << 20) + (1550ull << 20);
+}
+inline bool draft_layer_short(uint64_t free_bytes, uint64_t layer_bytes, uint64_t head_bytes) {
+    return free_bytes > 0 && free_bytes < draft_layer_need_bytes(layer_bytes, head_bytes);
+}
+
+/// DFlash2 on a stage: its pack and its K/V ring and buffers (0.25 GiB), its draft head, and the same working cache and late allocations.
+inline uint64_t dflash2_need_bytes(uint64_t pack_bytes, uint64_t head_bytes) {
+    return pack_bytes + (250ull << 20) + head_bytes + (220ull << 20) + (1550ull << 20);
+}
+
+/// The next KV window down from `current` (0: the whole context in VRAM) for a context of `ctx` cells, or 0 when it is already the smallest (`min_cells`).
+inline int64_t next_kv_window(int64_t current, int64_t ctx, int64_t min_cells) {
+    constexpr int64_t steps[] = {20480, 8192, 4096};
+    const int64_t cur = current > 0 ? current : ctx;
+    for (const int64_t s : steps)
+        if (s < cur && s < ctx && s >= min_cells) return s;
+    return 0;
+}
+
+/// The layer share of each of `n` cards, proportional to what each has free beyond `fixed_mib`; at least 2 layers a card, summing to `layers`.  Used when the split is
+/// chosen before any card has loaded (trimmed on both), so the weights of CUDA0 can be trimmed too.
+inline void proportional_layers(const int64_t* free_mib, int n, int64_t layers, int64_t fixed_mib, int64_t* out) {
+    int64_t total = 0;
+    for (int i = 0; i < n; ++i) total += std::max<int64_t>(free_mib[i] - fixed_mib, 1);
+    int64_t given = 0;
+    for (int i = 0; i < n; ++i) {
+        const int64_t w = std::max<int64_t>(free_mib[i] - fixed_mib, 1);
+        out[i] = i + 1 < n ? std::max<int64_t>(2, (layers * w + total / 2) / total) : 0;
+        given += out[i];
+    }
+    out[n - 1] = layers - given;
+}
+
+}  // namespace strata::program::split_walk

serve/server.py

@@ -1087,8 +1087,10 @@ class StrataEngine:
 
     def __init__(self, exe: str, args: list[str], cwd: str | None = None, log: str | None = None,
                  env: dict | None = None, lazy: bool = False, cpus: list[int] | None = None, _attempt: int = 0,
-                 retry_args: list[str] | None = None):
+                 retry_args: list[str] | None = None, retry_vram_args: list[str] | None = None):
         # retry_args: the arguments to start again with, once, when this start ends before the engine is ready (an Auto start that picked DFlash2: mtp_retry_args)
+        # retry_vram_args: the same start with no drafter (nodraft_retry_args), used instead when the engine ended because a card ran out of memory: swapping DFlash2 for
+        # the MTP takes MORE memory from a card that is already short, so that direction is never taken then
         self.spawn = (exe, list(args), cwd, log, env, False, cpus)   # to start it again after it died (issue #27)
         # the all-cards split engine (sFNX, strata-split-N[-arch]): its GEN parser accepts fewer keys than Strata's engine (tests/split/test_split_gen_keys.py pins them)
         self.split_engine = os.path.basename(str(exe)).startswith("strata-split")
@@ -1214,9 +1216,22 @@ class StrataEngine:
                           f"ready yet: starting it again in {wait_s:.0f} s (try {_attempt + 2} of {START_RETRIES + 1})",
                           flush=True)
                 time.sleep(wait_s)
-                StrataEngine.__init__(self, exe, args, cwd, log, env, lazy, cpus, _attempt + 1, retry_args)
+                StrataEngine.__init__(self, exe, args, cwd, log, env, lazy, cpus, _attempt + 1, retry_args, retry_vram_args)
                 return
-            if not timed_out and retry_args and retry_args != list(args):
+            short_reason = None if timed_out else vram_fit.engine_stop_reason(start_log_text(log, log_start).splitlines())
+            if short_reason and retry_vram_args and retry_vram_args != list(args):
+                # a card ran out of memory (the engine's own reason is in its log): one more start with no drafter, in one quiet line
+                print(f"[strata] {drafting.line('off', drafting.SHORT_OF_VRAM)}: starting again without a drafter ({short_reason})", flush=True)
+                try:
+                    with open(log, "a", encoding="utf-8") as handle:
+                        handle.write(f"[strata] {time.strftime('%Y-%m-%d %H:%M:%S')} the engine ended before it was ready because a card was short of memory "
+                                     f"(exit code {self.proc.returncode}); starting it again without a drafter\n")
+                except (OSError, TypeError):
+                    pass
+                drafting.remember_note(drafting.SHORT_OF_VRAM)
+                StrataEngine.__init__(self, exe, retry_vram_args, cwd, log, env, lazy, cpus, 0, None, None)
+                return
+            if not timed_out and not short_reason and retry_args and retry_args != list(args):
                 # Auto picked DFlash2 and the engine ended before it was ready (an engine that does not take one of its flags, a drafter file it cannot read...):
                 # one more start with the MTP, said in one quiet line. The person named DFlash2 themselves: no retry, the start fails with its reason below.
                 print(f"[strata] {drafting.line('mtp', drafting.DIDNT_START)}: starting again with the MTP", flush=True)
@@ -1227,7 +1242,7 @@ class StrataEngine:
                 except (OSError, TypeError):
                     pass
                 drafting.remember_note(drafting.DIDNT_START)
-                StrataEngine.__init__(self, exe, retry_args, cwd, log, env, lazy, cpus, 0, None)
+                StrataEngine.__init__(self, exe, retry_args, cwd, log, env, lazy, cpus, 0, None, None)
                 return
             why = (f"the engine did not report READY within {ENGINE_READY_S:.0f} s" if timed_out else
                    "the engine exited before it was ready")
@@ -3489,6 +3504,22 @@ def mtp_retry_args(cfg: dict, exe: str | None, extra: list[str] | None = None) -
     return alt if alt != first + list(extra or []) else None
 
 
+def nodraft_retry_args(cfg: dict, exe: str | None, extra: list[str] | None = None) -> list[str] | None:
+    """The arguments to start again with when an Auto start ends before the engine is ready because a card ran out of memory: the same start with no drafter
+    (the draft layer and DFlash2's buffers are the biggest optional part on a card that is short), never the MTP in place of DFlash2.  None when the drafting
+    was not Auto (the person named a drafter: that start fails with the engine's reason), when the first start has no drafter anyway, or when nothing differs."""
+    if drafting.normalize(cfg.get("drafting")) != "auto" or "args" not in cfg:
+        return None
+    try:
+        first = engine_args(cfg, exe, announce=False)
+        if "--draft" not in first and "--mtp" not in first:
+            return None
+        alt = engine_args({**cfg, "drafting": "off"}, exe, announce=False) + list(extra or [])
+    except Exception:                                  # noqa: BLE001 - a retry that cannot be worked out is no retry
+        return None
+    return alt if alt != first + list(extra or []) else None
+
+
 def engine_args(cfg: dict, exe: str | None = None, *, announce: bool = True) -> list[str]:
     """The engine's arguments: the config's, and with several GPUs the layer split across them ("layer_split" in the
     config: "auto" by default, or the first layer of each later GPU's share, e.g. "18" or "16,32"; see
@@ -9112,7 +9143,7 @@ def start_engine_or_fallback(cfg: dict, exe: str, effort_end: list[str] | None,
     fb = cfg.get("split_fallback")
     try:
         return StrataEngine(exe, engine_args(cfg, exe) + (effort_end or []), cwd=cfg.get("cwd"), log=cfg.get("log"), env=env, lazy=lazy,
-                            retry_args=mtp_retry_args(cfg, exe, effort_end))
+                            retry_args=mtp_retry_args(cfg, exe, effort_end), retry_vram_args=nodraft_retry_args(cfg, exe, effort_end))
     except Exception as e:
         if not isinstance(fb, dict) or not fb.get("exe"):
             raise
@@ -9140,7 +9171,7 @@ def start_engine_or_fallback(cfg: dict, exe: str, effort_end: list[str] | None,
     configure_batch_telemetry_env(env2, cfg2, batch_telemetry)
     effort2 = effort_end_args(cfg2, exe2, tok)
     engine = StrataEngine(exe2, engine_args(cfg2, exe2) + (effort2 or []), cwd=cfg2.get("cwd"), log=cfg2.get("log"), env=env2, lazy=lazy,
-                          retry_args=mtp_retry_args(cfg2, exe2, effort2))
+                          retry_args=mtp_retry_args(cfg2, exe2, effort2), retry_vram_args=nodraft_retry_args(cfg2, exe2, effort2))
     engine.info["split_fallback"] = True
     return engine
 
@@ -9168,7 +9199,7 @@ def start_replicas(cfg: dict, groups: list[list[int]], exe: str, effort_end: lis
         try:
             engines[i] = StrataEngine(exe, engine_args(rc, exe) + effort_end, cwd=rc.get("cwd"), log=rc.get("log"),
                                       env=child_env(rc, offline=offline), lazy=lazy, cpus=cpus[i] if cpus else None,
-                                      retry_args=mtp_retry_args(rc, exe, effort_end))
+                                      retry_args=mtp_retry_args(rc, exe, effort_end), retry_vram_args=nodraft_retry_args(rc, exe, effort_end))
         except BaseException as e:        # noqa: BLE001
             errors.append((i, e))
     rc0 = rp.replica_cfg(cfg, 0, groups[0])
@@ -9235,7 +9266,7 @@ def start_split(cfg: dict, spec: dict, exe: str, effort_end: list[str], lazy: bo
             if role == "prefill":
                 warn_budget_over_ram(engine_args(rc, announce=False) if "args" in rc else [])
             engines[role] = StrataEngine(exe, engine_args(rc, exe) + effort_end, cwd=rc.get("cwd"), log=rc.get("log"),
-                                         env=child_env(rc, offline=offline), lazy=lazy, retry_args=mtp_retry_args(rc, exe, effort_end))
+                                         env=child_env(rc, offline=offline), lazy=lazy, retry_args=mtp_retry_args(rc, exe, effort_end), retry_vram_args=nodraft_retry_args(rc, exe, effort_end))
     except BaseException:
         for e in engines.values():
             try:

serve/test_drafting_retry.py

@@ -19,6 +19,11 @@ if '--draft' in args and '--die-on-draft' in args:
     sys.stderr.write('strata generate: --draft dflash2 drafts without the MTP layer and needs --draft-model PACK, --spec 2 or more and --mtp DIR (for DIR/draft_vocab.bin)\n')
     sys.stderr.flush()
     raise SystemExit(2)
+if '--die-short' in args and ('--draft' in args or '--mtp' in args):
+    sys.stderr.write('ggml_cuda_init: failed to initialize CUDA: no CUDA-capable device is detected'+chr(10))
+    sys.stderr.write('strata generate: layer split, CUDA1 expert cache: no room'+chr(10))
+    sys.stderr.flush()
+    raise SystemExit(1)
 if '--die-always' in args:
     sys.stderr.write('strata generate: something else is wrong\n')
     sys.stderr.flush()
@@ -93,6 +98,33 @@ class AutoRetriesWithTheMtp(unittest.TestCase):
         self.assertEqual(len(self.started()), 1)
         self.assertEqual(engine.info.get("drafting_note", ""), "")
 
+    def test_a_start_that_ran_out_of_memory_is_repeated_with_no_drafter_never_with_the_mtp(self):
+        """2 small cards on a layer split: the engine says a card has no room.  The old retry swapped DFlash2 for the MTP, which takes MORE memory from a card that is short."""
+        first = [a for a in self.first if a != "--die-on-draft"] + ["--die-short"]
+        mtp = [a for a in self.mtp if a != "--die-on-draft"] + ["--die-short"]
+        nodraft = ["--pack", "/d/packs/g", "--starts", str(self.starts), "--die-short"]
+        engine = StrataEngine(str(self.script), first, log=self.log, retry_args=mtp, retry_vram_args=nodraft)
+        self.addCleanup(engine.close)
+        self.assertTrue(engine.alive())
+        starts = self.started()
+        self.assertEqual(len(starts), 2, "one start with DFlash2, one without a drafter, never the MTP in between")
+        self.assertIn("--draft dflash2", starts[0])
+        self.assertNotIn("--mtp", starts[1])
+        self.assertNotIn("--draft", starts[1])
+        self.assertEqual(engine.spawn[1], nodraft)
+        self.assertEqual(engine.info["drafting"], "off")
+        self.assertEqual(engine.info["drafting_note"], "Drafting: off (the cards were short of memory)")
+        text = Path(self.log).read_text(encoding="utf-8")
+        self.assertIn("short of memory", text)
+        self.assertIn("expert cache: no room", text, "the engine's own reason stays in the log")
+
+    def test_a_memory_stop_without_a_no_drafter_retry_fails_with_the_engines_reason_and_does_not_try_the_mtp(self):
+        first = [a for a in self.first if a != "--die-on-draft"] + ["--die-short"]
+        with self.assertRaises(EngineDied) as caught:
+            StrataEngine(str(self.script), first, log=self.log, retry_args=[*self.mtp, "--die-short"])
+        self.assertIn("expert cache: no room", str(caught.exception))
+        self.assertEqual(len(self.started()), 1)
+
     def test_the_same_arguments_are_not_a_retry(self):
         with self.assertRaises(EngineDied):
             StrataEngine(str(self.script), self.first, log=self.log, retry_args=list(self.first))

src/program/generate.cpp

@@ -99,6 +99,7 @@
 #include "strata/program/conv_cache.hpp"
 #include "strata/program/split_rules.hpp"
 #include "strata/program/stage_reserve.hpp"
+#include "strata/program/split_walk.hpp"
 #include "strata/program/gpu_report.hpp"
 #include "strata/prefill/share_rules.hpp"
 #include "strata/platform/host_limits.hpp"
@@ -3962,6 +3963,27 @@ int main(int argc, char** argv) {
                                  "(--split-device 0); disable parking with --conversation-cache-mib 0\n");
             return 2;
         }
+        // THE LAYER SPLIT'S SPILL WALK, STEP 6: FEWER CARDS.  Under `auto`, a later card with too little free memory to hold even a trimmed share (a card another program fills) is left out
+        // of the split, said plainly; with none left the start is the single card's, with its own walk and the rest of the experts in RAM.  Only the cards the engine would have picked.
+        if (ok && split_auto && !split_devs.empty() && std::getenv("STRATA_SPILL_OFF") == nullptr && spill_discrete_gpu()) {
+            std::vector<int> kept;
+            for (const int d : split_devs) {
+                size_t fb_c = 0, tb_c = 0;
+                bool usable = true;
+                {
+                    const strata::core::OnDevice on_c(d);
+                    usable = cudaMemGetInfo(&fb_c, &tb_c) != cudaSuccess || strata::program::split_walk::card_usable((int64_t) (fb_c >> 20));
+                    (void) cudaGetLastError();
+                }
+                if (usable) { kept.push_back(d); continue; }
+                std::fprintf(stderr, "strata generate: short of GPU memory: CUDA%d has only %.2f GiB free, too little to hold a share of the model beside its context, a working cache "
+                                     "and the prompt buffers (%.1f GiB at least): the layer split runs without it\n",
+                             d, (double) fb_c / 1073741824.0, (double) strata::program::split_walk::kMinCardMib / 1024.0);
+            }
+            if (kept.empty())
+                std::fprintf(stderr, "strata generate: short of GPU memory: no other card has room, so the model runs on CUDA0 alone with the experts that do not fit in RAM\n");
+            split_devs = kept;
+        }
         if (ok && split_auto && split_devs.empty()) {
             std::fprintf(stderr, "strata generate: --layer-split auto: one GPU visible, so no split\n");
             o.layer_split.clear();
@@ -4859,7 +4881,10 @@ int main(int argc, char** argv) {
     // Only `--layer-split auto` WITH the trim asked loads the later stages' weights after the split search (they can
     // only be trimmed once the split is known).  Every other run keeps the 0.1.39 order: weights first, then the
     // search - so without the trim nothing about a split moves.
-    const bool late_weights = trim_asked && multi_gpu && split_auto;
+    bool late_weights = trim_asked && multi_gpu && split_auto;
+    // THE LAYER SPLIT'S SPILL WALK, STEP 1 (include/strata/program/split_walk.hpp): a card too short for the whole dense set loads only its own layers' weights.  Decided below,
+    // once the dense set's size is known and before any later stage loads; `walk_trim` is that decision, `later_trim` the stages' trim (asked for, or the walk's).
+    bool walk_trim = false;
     // keep_routers: CUDA0 with --mmap-experts keeps every layer's router (ffn_gate_inp, ~2.5 MiB a layer) - the file
     // tier's routing-aware prefetch (RouterLookahead, below) copies all of them from CUDA0's arena, and would
     // otherwise turn itself off
@@ -4968,6 +4993,40 @@ int main(int argc, char** argv) {
         std::fprintf(stderr, "strata generate: %zu native projection matrices, %.2f MiB of weights, in %.1f s\n",
                      native_dense.tensor_count(), (double) native_dense.weight_bytes() / (1024.0 * 1024.0), load_s());
     }
+    // ---- THE LAYER SPLIT'S SPILL WALK, STEP 1: THE TRIM.  CUDA0 holds the whole dense set now; every later card is about to load a copy of it (3.5 GiB for IQ3_S), which on a
+    // card that is short (two 8 GB cards: 6 of 6 starts failed in 0.2.10) is what leaves its expert cache and its prompt path no room.  A card that cannot afford the copy
+    // beside its session, its head, its drafter and a working cache loads only the dense weights of its own layers instead (--trim-stage-weights, bit-exact in itself:
+    // the cache just gets the room).  A card with room never reaches this; STRATA_STAGE_TRIM=0 or STRATA_SPILL_OFF=1 keeps the old copy.  CUDA0 stays whole under
+    // `--layer-split auto` (its weights load before the split is known); it runs the card's share of layers the search gives it.
+    if (multi_gpu && !trim_asked && !split_devs.empty() && std::getenv("STRATA_SPILL_OFF") == nullptr &&
+        !(std::getenv("STRATA_STAGE_TRIM") != nullptr && std::string(std::getenv("STRATA_STAGE_TRIM")) == "0") && spill_discrete_gpu()) {
+        strata::program::split_walk::Model wm;
+        wm.dense_mib = (int64_t) ((pool_bytes + (o.native_dense_gguf.empty() ? 0 : native_dense.weight_bytes())) >> 20);
+        const int ns_w = (int) split_devs.size() + 1;
+        wm.session_mib = (int64_t) (strata::core::session_bytes(g, o.max_context, K, 0, (g.n_layers + ns_w - 1) / ns_w) >> 20);
+        if (const strata::core::WeightRef* wo_w = wt.find("output.weight"); wo_w != nullptr && !o.native_head_gguf.empty())
+            wm.head_mib = (int64_t) (((uint64_t) wo_w->ne1 * (uint64_t) (g.n_embd / 256) * 176ull) >> 20);   // Q5_K: 176 bytes per 256 values
+        wm.drafter_mib = (!o.mtp.empty() || o.draft == "dflash2") ? strata::program::stage_reserve::kDrafterMib : 0;
+        wm.slot_mib = std::max<int64_t>(1, (int64_t) ((strata::kernels::cpu::expert_layout().max_blob + (1u << 20) - 1) >> 20));
+        wm.reserve_mib = (o.vram_reserve_later_mib >= 0 ? o.vram_reserve_later_mib : o.vram_reserve_mib) + strata::program::stage_reserve::kWindowMib;
+        for (size_t i = 0; i < split_devs.size(); ++i) {
+            const strata::core::OnDevice on_w(split_devs[i]);
+            size_t fb_w = 0, tb_w = 0;
+            if (cudaMemGetInfo(&fb_w, &tb_w) != cudaSuccess) { (void) cudaGetLastError(); continue; }
+            const bool last_w = i + 1 == split_devs.size();
+            if (strata::program::split_walk::trim_needed((int64_t) (fb_w >> 20), wm, last_w)) {
+                std::fprintf(stderr, "strata generate: short of GPU memory: CUDA%d has %.2f GiB free and a layer split card needs about %.2f GiB beside a full copy of the dense weights "
+                                     "(%.2f GiB): each later card loads only the dense weights of its own layers (STRATA_STAGE_TRIM=0 keeps the copy)\n",
+                             split_devs[i], (double) fb_w / 1073741824.0, (double) strata::program::split_walk::untrimmed_need_mib(wm, last_w) / 1024.0,
+                             (double) wm.dense_mib / 1024.0);
+                walk_trim = true;
+                break;
+            }
+        }
+        if (walk_trim && split_auto) late_weights = true;
+    }
+    // the stages' dense weights are trimmed to their own layers: asked for (an explicit split), or by the walk; under `auto` they load after the split search (late_weights)
+    const bool later_trim = stage_trim || (walk_trim && !split_auto && !split_at.empty());
 
     strata::kernels::gr_set_fp32_activations(o.gr_fp32_activations);
     strata::kernels::gr_set_native_mmvf(o.gr_native_mmvf);
@@ -5529,9 +5588,9 @@ int main(int argc, char** argv) {
         // Later stages' weights load here (0.1.39), except `--layer-split auto` with the trim asked: those load after
         // the split search (see late_weights), so `auto` can trim them too.
         if (!late_weights) {
-            const int64_t lb = stage_trim ? split_at[i] : 0;
-            const int64_t le = stage_trim ? (i + 1 < split_at.size() ? split_at[i + 1] : g.n_layers) : 0;
-            if (!load_stage_weights(st, lb, le, stage_trim)) return 1;
+            const int64_t lb = later_trim ? split_at[i] : 0;
+            const int64_t le = later_trim ? (i + 1 < split_at.size() ? split_at[i + 1] : g.n_layers) : 0;
+            if (!load_stage_weights(st, lb, le, later_trim)) return 1;
         }
         // THE SESSION AND THE HEAD WAIT FOR THE SPLIT SEARCH.  `session_bytes` prices a stage's session by its
         // LAYER RANGE (the carve - every stage used to hold all 48 layers' state whatever it ran), so the
@@ -5748,8 +5807,11 @@ int main(int argc, char** argv) {
             std::fprintf(stderr, "strata generate: layer split: CUDA%d free memory: %s\n", dev < 0 ? 0 : dev,
                          cudaGetErrorString(e));
         const int64_t pf = search && split_own_auto && !place_with_reserve ? 0 : split_pf_mib;
+        // what a later stage allocates after its cache and nothing prices (cuBLAS, the modules and the MMQ pool the prompt path warms up, ~250 MiB measured): the single card prices
+        // it (kLateAllocMib, #4474); a stage did not, and two 16 GB cards ended a start with 131 MiB free and ran out of memory in the first long prompt.  Not under STRATA_DEFAULTS=026.
+        const int64_t late_alloc = later && !strata::qwen_defaults_026() ? strata::program::split_walk::kStageLateAllocMib : 0;
         const int64_t base_reserve =
-            later && o.vram_reserve_later_mib >= 0 ? o.vram_reserve_later_mib : o.vram_reserve_mib;
+            (later && o.vram_reserve_later_mib >= 0 ? o.vram_reserve_later_mib : o.vram_reserve_mib) + late_alloc;
         strata::program::stage_reserve::Inputs rin;
         rin.base_reserve_mib = base_reserve;
         rin.prompt_buffers_mib = pf;
@@ -6261,6 +6323,27 @@ int main(int argc, char** argv) {
             std::fprintf(stderr, "strata generate: to make room: a smaller --max-context, --prefill 512, a smaller "
                                  "--vram-reserve-mib, an explicit --layer-split, or close other programs using the GPU\n");
         };
+        // THE LAYER SPLIT'S SPILL WALK, STEP 2: THE KV WINDOW DOWN.  No placement can start the prompt path with the window asked for: the sessions of the cards are smaller with a
+        // smaller one (the rest of the K/V streams from pinned RAM), so the search runs again at 20480, 8192, 4096 cells a layer (the engine's smallest) and keeps the first window
+        // that has a startable placement.  The context and the number of requests are not touched.  Never reached by a start that has a placement.
+        if (best.empty() && std::getenv("STRATA_SPILL_OFF") == nullptr && spill_discrete_gpu()) {
+            const int64_t kv_asked = o.kv_resident;
+            for (int64_t w; (w = strata::program::split_walk::next_kv_window(o.kv_resident, o.max_context, strata::core::qsa_kv_resident_min())) > 0;) {
+                o.kv_resident = w;
+                strata::core::qsa_set_kv_resident(w);
+                search();
+                if (!best.empty()) {
+                    std::fprintf(stderr, "strata generate: short of GPU memory: the KV window is %lld cells a layer in VRAM instead of %s, the rest is read from RAM: the context "
+                                         "(%lld tokens) is unchanged\n", (long long) w,
+                                 kv_asked > 0 ? std::to_string((long long) kv_asked).c_str() : "the whole context", (long long) o.max_context);
+                    break;
+                }
+            }
+            if (best.empty()) {
+                o.kv_resident = kv_asked;
+                strata::core::qsa_set_kv_resident(kv_asked);
+            }
+        }
         if (best.empty()) {
             // recommend, never force: with nothing startable on paper the search runs again without the gate, so the
             // run is as it was before the gate, and what is short is said
@@ -6752,20 +6835,54 @@ int main(int argc, char** argv) {
     // the sessions) the start used to stop at "no VRAM is left for the expert cache" or run out of memory in the head upload.  Without the layer
     // the model still answers (the verify window drafts from the prompt alone): slower decode, a start that works.  Not with DFlash2 (its own
     // block decides), a split, or STRATA_MTP_KEEP=1.
-    if (!o.mtp.empty() && !use_df2 && !multi_gpu && stages.empty() && std::getenv("STRATA_MTP_KEEP") == nullptr && spill_discrete_gpu()) {
-        std::error_code fec1, fec2;
-        const uint64_t layer_b = (uint64_t) std::filesystem::file_size(std::filesystem::path(o.mtp) / "dense.bin", fec1) +
-                                 (uint64_t) std::filesystem::file_size(std::filesystem::path(o.mtp) / "experts.bin", fec2);
-        const strata::core::WeightRef* wo_mtp = wt.find("output.weight");
-        const uint64_t head_b = (!o.native_head_gguf.empty() && wo_mtp != nullptr)
-                                    ? (uint64_t) wo_mtp->ne1 * (uint64_t) (g.n_embd / 256) * 176ull : 0;   // Q5_K: 176 bytes per 256 values
-        const uint64_t free_now = (uint64_t) strata::core::device_free_bytes();
-        const uint64_t need = layer_b + head_b + (220ull << 20) + (1550ull << 20);   // + the cache the loan needs (214 slots, 543 MiB) + what is allocated after it and the drafter beyond DFlash2 (measured: DFlash2 alone ends 391 MiB free, MTP 41 at the same cache)
-        if (!fec1 && !fec2 && free_now > 0 && free_now < need) {
-            std::fprintf(stderr, "strata generate: short of GPU memory: %.2f GiB is free, and the draft layer (%.2f GiB), the head (%.2f GiB) and a working cache "
-                                 "need about %.2f GiB: this start runs without the draft layer (decode is slower; STRATA_MTP_KEEP=1 keeps it)\n",
-                         (double) free_now / 1073741824.0, (double) layer_b / 1073741824.0, (double) head_b / 1073741824.0, (double) need / 1073741824.0);
-            o.mtp.clear();
+    // A LAYER SPLIT WALKS THE SAME STEP ON THE CARD THAT HOLDS THE DRAFTER (the last stage's, or CUDA0 with the drafter first): its free memory is read there, after its weights, session
+    // and head are in.  DFlash2 (its pack, its ring and buffers, its draft head) is dropped by the same rule on that card; the single card keeps it as before (its own block decides).
+    {
+        const bool split_walk_drafter = multi_gpu && !stages.empty() && std::getenv("STRATA_SPILL_OFF") == nullptr;
+        if (!o.mtp.empty() && !use_df2 && (split_walk_drafter || (!multi_gpu && stages.empty())) && std::getenv("STRATA_MTP_KEEP") == nullptr && spill_discrete_gpu()) {
+            std::error_code fec1, fec2;
+            const uint64_t layer_b = (uint64_t) std::filesystem::file_size(std::filesystem::path(o.mtp) / "dense.bin", fec1) +
+                                     (uint64_t) std::filesystem::file_size(std::filesystem::path(o.mtp) / "experts.bin", fec2);
+            const strata::core::WeightRef* wo_mtp = wt.find("output.weight");
+            const uint64_t head_b = (!o.native_head_gguf.empty() && wo_mtp != nullptr)
+                                        ? (uint64_t) wo_mtp->ne1 * (uint64_t) (g.n_embd / 256) * 176ull : 0;   // Q5_K: 176 bytes per 256 values
+            const int draft_dev = split_walk_drafter ? (last_st && !mtp_first ? last_st->dev : -1) : -1;
+            uint64_t free_now = 0;
+            {
+                const strata::core::OnDevice on_draft(draft_dev);
+                free_now = (uint64_t) strata::core::device_free_bytes();
+            }
+            const uint64_t need = strata::program::split_walk::draft_layer_need_bytes(layer_b, head_b);   // + the cache the loan needs (214 slots, 543 MiB) + what is allocated after it and the drafter beyond DFlash2 (measured: DFlash2 alone ends 391 MiB free, MTP 41 at the same cache)
+            if (!fec1 && !fec2 && strata::program::split_walk::draft_layer_short(free_now, layer_b, head_b)) {
+                const std::string card = split_walk_drafter ? "CUDA" + std::to_string(draft_dev < 0 ? 0 : draft_dev) + ": " : std::string();
+                std::fprintf(stderr, "strata generate: short of GPU memory: %s%.2f GiB is free, and the draft layer (%.2f GiB), the head (%.2f GiB) and a working cache "
+                                     "need about %.2f GiB: this start runs without the draft layer (decode is slower; STRATA_MTP_KEEP=1 keeps it)\n",
+                             card.c_str(), (double) free_now / 1073741824.0, (double) layer_b / 1073741824.0, (double) head_b / 1073741824.0, (double) need / 1073741824.0);
+                o.mtp.clear();
+            }
+        }
+        // DFlash2 alone (--draft-fallback none, what Auto passes) on a stage that cannot hold it: no drafter, said in one line, instead of a start that ends
+        if (split_walk_drafter && df2_solo && !use_df2 && std::getenv("STRATA_MTP_KEEP") == nullptr && spill_discrete_gpu()) {
+            std::error_code fec3;
+            const uint64_t pack_b = (uint64_t) std::filesystem::file_size(o.draft_model, fec3);
+            const strata::core::WeightRef* wo_df = wt.find("output.weight");
+            const uint64_t head_b = (!o.native_head_gguf.empty() && wo_df != nullptr)
+                                        ? (uint64_t) wo_df->ne1 * (uint64_t) (g.n_embd / 256) * 176ull : 0;
+            const int dev_df = last_st && !df2_first ? last_st->dev : -1;
+            uint64_t free_df = 0;
+            {
+                const strata::core::OnDevice on_df(dev_df);
+                free_df = (uint64_t) strata::core::device_free_bytes();
+            }
+            const uint64_t need_df = strata::program::split_walk::dflash2_need_bytes(pack_b, head_b / 2);
+            if (!fec3 && free_df > 0 && free_df < need_df) {
+                std::fprintf(stderr, "strata generate: short of GPU memory: CUDA%d: %.2f GiB is free, and DFlash2 (%.2f GiB with its buffers and draft head) and a working cache need about "
+                                     "%.2f GiB: this start runs without a drafter (decode is slower; STRATA_MTP_KEEP=1 keeps it)\n",
+                             dev_df < 0 ? 0 : dev_df, (double) free_df / 1073741824.0, (double) pack_b / 1073741824.0 + 0.45, (double) need_df / 1073741824.0);
+                df2_solo = false;
+                o.draft.clear();
+                o.draft_model.clear();
+            }
         }
     }
     if (!o.mtp.empty() || df2_solo) {   // (df2_solo: o.mtp is empty and no MTP layer is loaded; only the DFlash2 block below runs)
@@ -7630,7 +7747,7 @@ int main(int argc, char** argv) {
         // than that (8 GB NVIDIA, 128K: 112-212 slots) used to be squeezed by lowering the reserve to give the cache its 326 working slots, which left 100 MiB
         // free and stopped the first long prompt ("mmq: out of memory"), or gave the prompt path its own 600 MiB of buffers on top and stopped with "do not
         // fit".  The loan may leave 16 slots instead: the cache is smaller, prompts are read in smaller chunks and slower, the reserve stays, and the start works.
-        if (o.prefill_chunk > 0 && pf_borrow && o.prefill_auto && o.expert_cache < min_slots && !o.vram_reserve_given && !multi_gpu && stages.empty() &&
+        if (o.prefill_chunk > 0 && pf_borrow && o.prefill_auto && o.expert_cache < min_slots && !o.vram_reserve_given &&
             std::getenv("STRATA_SPILL_OFF") == nullptr && spill_discrete_gpu()) {
             constexpr int64_t kTightKeep = 16;
             const int64_t tight_min = ((int64_t) strata::prefill::Prefill::bytes_needed(g, ss, 256) + blob - 1) / blob + kTightKeep;
@@ -7677,7 +7794,7 @@ int main(int argc, char** argv) {
         // verify arena, or DFlash2's staging) stopped with "do not fit" (8 GB NVIDIA, 128K, 2 requests: a 74-slot cache and 35 MiB left).  Price them
         // here instead, at the largest chunk that still leaves a cache (1024, 512, 256 tokens), and give the cache the rest: slower (a smaller
         // cache, smaller prompt chunks), never a refused start.  Not with a split, a cache or chunk given on the command line, or STRATA_SPILL_OFF=1.
-        if (auto_cache && pf_borrow && o.prefill_auto && !o.vram_reserve_given && o.expert_cache < min_slots && !multi_gpu && stages.empty() && !profile.empty() &&
+        if (auto_cache && pf_borrow && o.prefill_auto && !o.vram_reserve_given && o.expert_cache < min_slots && !profile.empty() &&
             std::getenv("STRATA_SPILL_OFF") == nullptr && spill_discrete_gpu()) {
             constexpr int64_t kSpillFloorMib = 100;   // the verify arena, the head check and the cuBLAS workspace beside the late allocations
             int64_t pick_chunk = 0, pick_slots = 0, pick_own = 0, pick_late = 0;
@@ -8163,8 +8280,25 @@ int main(int argc, char** argv) {
         const auto& lay = strata::kernels::cpu::expert_layout();
         // the drafter and the head are already allocated by now (they load above, before this), so what is left
         // to hold back is the windows - and `free_b` has already lost the drafter.
-        const int64_t room = stage_room(st.dev, true, false);
+        int64_t room = stage_room(st.dev, true, false);
         const strata::core::OnDevice on(st.dev);
+        // THE LAYER SPLIT'S SPILL WALK, STEP 5: THE OUTPUT HEAD ON THE HOST, LAST.  The card that holds the head (the last stage) has no room for a working cache beside it: the head
+        // (0.41 GiB) moves to mapped pinned host memory and every logits row reads it over PCIe (decode is slower, the start works).  Only a card that would otherwise refuse.
+        if (&st == last_st && st.head.loaded() && !st.head.on_host() && !o.vram_reserve_given && std::getenv("STRATA_HEAD_HOST_OFF") == nullptr &&
+            std::getenv("STRATA_SPILL_OFF") == nullptr && spill_discrete_gpu() &&
+            room < strata::program::split_walk::kWorkingCacheSlots * (int64_t) lay.max_blob) {
+            std::string herr;
+            const uint64_t head_mib = st.head.weight_bytes() >> 20;
+            if (st.head.to_host(herr)) {
+                room = stage_room(st.dev, true, false);
+                std::fprintf(stderr, "strata generate: short of GPU memory: CUDA%d: the cache would have no room, so the output head (%llu MiB) moves to pinned host memory and is read over "
+                                     "PCIe for every logits row: decode is slower; the room for experts is now %.2f GiB\n",
+                             st.dev, (unsigned long long) head_mib, (double) room / 1073741824.0);
+            } else {
+                std::fprintf(stderr, "strata generate: short of GPU memory: CUDA%d: the output head stays in VRAM (%s)\n", st.dev,
+                             herr.empty() ? "type or build not movable" : herr.c_str());
+            }
+        }
         {
             size_t fb = 0, tb = 0;
             cudaMemGetInfo(&fb, &tb);
@@ -8261,6 +8395,23 @@ int main(int argc, char** argv) {
     }
     if (multi_gpu)
         std::fprintf(stderr, "strata generate: layer split: CUDA0 runs layers 0-%lld\n", (long long) (split_at[0] - 1));
+    // THE LAYER SPLIT'S SPILL WALK, STEP 4: THE LOAN KEEPS 16.  A stage whose cache is too small to lend the prompt path a 256-token chunk and keep 128 slots behind makes the path
+    // allocate its own buffers on top (more than the reserve): the same short card as the single card's, which keeps 16 slots instead (the cache is smaller, prompts are read in
+    // smaller chunks, the start works).  Never on a card with room: it only fires when a cache is below the usual loan.
+    if (multi_gpu && !stages.empty() && g_lend_keep == 128 && o.prefill_chunk > 0 && pf_borrow && o.prefill_auto && !o.vram_reserve_given &&
+        std::getenv("STRATA_SPILL_OFF") == nullptr && spill_discrete_gpu()) {
+        for (const auto& stp : stages) {
+            const int64_t blob_s = (int64_t) nibble_max_slot_bytes();
+            const int64_t need_s = ((int64_t) strata::prefill::Prefill::bytes_needed(g, stp->ss, 256) + blob_s - 1) / blob_s + g_lend_keep;
+            if ((int64_t) stp->cache.slots() < need_s) {
+                std::fprintf(stderr, "strata generate: short of GPU memory: CUDA%d: the cache (%lld slots) is too small to lend the prompt path 256-token buffers and keep %lld more: "
+                                     "the loan keeps 16, prompts are read in smaller chunks, expect them to be slower\n",
+                             stp->dev, (long long) stp->cache.slots(), (long long) g_lend_keep);
+                g_lend_keep = 16;
+                break;
+            }
+        }
+    }
     if (nib::any_on()) {
         // STRATA_EXPERT_NIBBLE: every expert of every layer must be in VRAM, in the cache of the card that runs the layer.
         // A converted layer's miss would be computed from the GGUF blob by the CPU or over PCIe, and a non-converted

strata_cli/launch/runner.py

@@ -22,6 +22,7 @@ import sys
 import tempfile
 import threading
 import time
+from strata_control import vram_fit
 from strata_control import atomic as _atomic_mod
 from dataclasses import dataclass, field
 from pathlib import Path
@@ -2685,6 +2686,8 @@ CAUSE_LINE = re.compile(r"error|exception|failed|cannot|can't|could not|no devic
 HSA_START = re.compile(r"hsa_exception|HSA_STATUS_ERROR", re.I)
 HSA_START_NEXT = ("The graphics card was not ready yet; wait a few seconds and run it again. If it keeps happening, restart the PC "
                   "and report the log.")
+VRAM_STOP_NEXT = ("A graphics card is too full for this start, even after Strata moved what it could to the processor and the RAM. Close other programs that use the "
+                  "cards, or start on fewer cards (--gpu), or turn pictures off (--vision off); the context and the number of requests were not changed.")
 KNOWN_CAUSES = (
     (re.compile(r"cannot run the expert kernel|needs AVX-?512", re.I),
      "This model folder holds the pack for CPUs with AVX-512, and this PC's CPU does not have it. Run `strata setup` again: on a CPU "
@@ -2756,7 +2759,10 @@ def explain_engine_exit(result: dict, config: Path | None, marks: dict | None =
         if own:
             lines, log = own, str(server_log)
     lines = lines or [x.strip() for x in str(result.get("log_tail") or "").split(" | ") if x.strip()]
-    cause = next((x for x in reversed(lines) if CAUSE_LINE.search(x) and not x.lstrip().startswith(("strata generate: ", "strata serve: expert"))), None)
+    # a card that ran out of memory: the engine's own line is the cause (the picture reader's "ggml_cuda_init: ... no CUDA-capable device", which it prints when the
+    # cards are full and it runs on the processor, matched "failed" and used to be the last line shown)
+    vram_cause = vram_fit.engine_stop_reason(lines)
+    cause = vram_cause or next((x for x in reversed(lines) if CAUSE_LINE.search(x) and not x.lstrip().startswith(("strata generate: ", "strata serve: expert"))), None)
     cause = (cause or (lines[-1] if lines else "")).strip()
     cause = re.sub(r"^(\[strata\]\s*)?(\d{4}-\d\d-\d\d[ T][\d:.]+\s*)?", "", cause)[:240]
     state, why = result.get("state"), result.get("error") or result.get("message")
@@ -2779,7 +2785,7 @@ def explain_engine_exit(result: dict, config: Path | None, marks: dict | None =
     text = stage + (f": {cause}" if cause else ".")
     if not text.endswith((".", "!")):
         text += "."
-    step = HSA_START_NEXT if hsa else next((words for pattern, words in KNOWN_CAUSES if pattern.search(" ".join([cause, str(why or "")]))), "Run it again; if it repeats, report the log.")
+    step = HSA_START_NEXT if hsa else (VRAM_STOP_NEXT if vram_cause else next((words for pattern, words in KNOWN_CAUSES if pattern.search(" ".join([cause, str(why or "")]))), "Run it again; if it repeats, report the log."))
     if shared_memory and not hsa and not any(p.search(" ".join([cause, str(why or "")])) for p, _ in KNOWN_CAUSES[:2]):
         # a Strix Halo: the settings fix the expert cache beside a RAM budget, and the engine ended with no error text of its own (STRIX-2Q, 11-01)
         try:

strata_control/drafting.py

@@ -422,7 +422,8 @@ NO_MTP = "the MTP drafter isn't downloaded yet"
 NONE_HERE = "no drafter is downloaded yet"
 DIDNT_START = "DFlash2 didn't start"          # Auto picked DFlash2, the engine ended before it was ready, and the start was repeated once with the MTP (serve/server.py)
 NOT_SERVED = NOT_HERE
-REASONS = (DIDNT_FIT, NOT_HERE, NOT_SUPPORTED, NOT_DOWNLOADED, NOT_THIS_CARD, NO_MTP, NONE_HERE, DIDNT_START)
+SHORT_OF_VRAM = "the cards were short of memory"   # the engine ended before it was ready for lack of graphics memory: the start was repeated once with no drafter (serve/server.py), never with the MTP
+REASONS = (DIDNT_FIT, NOT_HERE, NOT_SUPPORTED, NOT_DOWNLOADED, NOT_THIS_CARD, NO_MTP, NONE_HERE, DIDNT_START, SHORT_OF_VRAM)
 
 
 def line(drafter: str, reason: str = "") -> str:

strata_control/vram_fit.py

@@ -35,6 +35,7 @@ It is an estimate; the engine still says what is short if it is off.
 
 from __future__ import annotations
 
+import re
 from dataclasses import dataclass, field
 
 CHUNK_STEPS = (1024, 512, 256)                   # a smaller prompt chunk, tried after the default one (1024 is the value measured end to end)
@@ -329,3 +330,33 @@ def fit_start(cfg: dict, args: list, vram_mib: float | None, backend: str | None
 def fit_context(model: str, vram_gb: float, ctx: int, kv: str = "int8", reserve_mib: int = 300, vendor: str = "intel") -> int:
     """The context to recommend: always `ctx`.  A fit does not change the context any more (owner, 2026-11-02); kept for the callers that ask."""
     return int(ctx)
+
+
+# ----- a start that ended because a card ran out of memory --------------------------------------------------------------------------------------------------
+# The engine's own lines for it (src/program/generate.cpp, serve): the cache has no room, the prompt path or a weight arena does not fit, the card has too
+# little free VRAM.  They start with "strata generate:" / "strata serve:", which the launcher's generic cause search skips (those logs are chatty), and the words
+# are not the generic "error|failed" ones, so the last such line used to be the picture reader's harmless "ggml_cuda_init: failed to initialize CUDA: no
+# CUDA-capable device" (it runs on the processor when the cards are full).  One list, so the launcher's final line and the server's retry agree.
+ENGINE_STOP_PATTERNS = (
+    re.compile(r"expert cache: no room", re.I),
+    re.compile(r"no VRAM is left for the expert cache", re.I),
+    re.compile(r"too little free VRAM", re.I),
+    re.compile(r"cannot start the prompt path", re.I),
+    re.compile(r"device buffers .* do not fit", re.I),
+    re.compile(r"weight arena (does not fit|failed)|cudaMalloc\(\d+\) for the weight arena failed", re.I),
+    re.compile(r"(session state|head|native dense projections|weights).*(out of memory|does not fit|failed)", re.I),
+)
+ENGINE_LINE = re.compile(r"^(?:\[strata\]\s*)?strata (?:generate|serve|mtp|dflash2|prefill): ", re.I)
+# a line that only reports what the walk did, never why a start stopped
+_NOT_A_STOP = re.compile(r"short of GPU memory: |trying a smaller|trying \d+ of \d+ slots|a card this small|LOW: ", re.I)
+
+
+def engine_stop_reason(lines) -> str | None:
+    """The engine's own last line saying a card was out of memory, without its "strata generate: " prefix; None when the start stopped for another reason.
+    `lines` are the log lines of ONE start (the launcher passes only what this attempt wrote)."""
+    for raw in reversed([str(x).strip() for x in lines or []]):
+        if not ENGINE_LINE.match(raw) or _NOT_A_STOP.search(raw):
+            continue
+        if any(p.search(raw) for p in ENGINE_STOP_PATTERNS):
+            return ENGINE_LINE.sub("", raw, count=1).strip()
+    return None

tests/program/split_walk_test.cpp

@@ -0,0 +1,85 @@
+// tests/program/split_walk_test.cpp - the layer split's spill walk (include/strata/program/split_walk.hpp): who is short, who is not, and what is given up in which order.
+#include "strata/program/split_walk.hpp"
+
+#include <cstdint>
+#include <cstdio>
+#include <string>
+
+using namespace strata::program::split_walk;
+
+namespace {
+int g_fail = 0;
+void check(bool ok, const std::string& what) {
+    if (ok) return;
+    std::fprintf(stderr, "FAIL: %s\n", what.c_str());
+    ++g_fail;
+}
+// IQ3_S on Qwen3.8-Flash-Next, 128K, int8 KV, window 32768, MTP: the figures the 8 GB start log gave (3485 MiB dense set, 835 MiB draft layer, 410 MiB head)
+Model iq3s() {
+    Model m;
+    m.dense_mib = 1466 + 2019;
+    m.session_mib = 400;
+    m.head_mib = 410;
+    m.drafter_mib = 1000;
+    m.slot_mib = 3;
+    m.reserve_mib = 796;
+    return m;
+}
+}  // namespace
+
+int main() {
+    const Model m = iq3s();
+    // 1. the trim: 8 GB cards are short, 12 GB and bigger are not (a big card never reaches the walk)
+    check(trim_needed(7635, m, true), "an 8 GB card holding the head and the drafter is short");
+    check(trim_needed(7635, m, false), "an 8 GB card without them is short too");
+    check(!trim_needed(12288, m, true), "a 12 GB card is not short");
+    check(!trim_needed(16384, m, true), "a 16 GB card is not short");
+    check(!trim_needed(97000, m, true), "a 96 GB card is never short");
+    check(!trim_needed(0, m, true), "an unreadable card (0 free) decides nothing");
+    check(untrimmed_need_mib(m, true) > untrimmed_need_mib(m, false), "the last stage needs more");
+    // monotone in the card: the threshold is one number
+    int64_t edge = 0;
+    for (int64_t f = 1000; f < 40000; ++f)
+        if (!trim_needed(f, m, true)) { edge = f; break; }
+    check(edge == untrimmed_need_mib(m, true), "the threshold is the need");
+    for (int64_t f = edge; f < edge + 5000; f += 97) check(!trim_needed(f, m, true), "never short above the threshold");
+    // a model with a smaller dense set (a smaller quant) is short later
+    Model small = m;
+    small.dense_mib = 1500;
+    check(!trim_needed(7635, small, false), "a small dense set fits an 8 GB card untrimmed");
+    // 2. fewer cards
+    check(!card_usable(2048), "a card with 2 GB free leaves the split");
+    check(card_usable(7635), "an 8 GB card stays");
+    check(card_usable(kMinCardMib) && !card_usable(kMinCardMib - 1), "the edge");
+    // 3. the draft layer: the single card's arithmetic, unchanged
+    const uint64_t layer = 835ull << 20, head = 410ull << 20;
+    check(draft_layer_need_bytes(layer, head) == layer + head + (220ull << 20) + (1550ull << 20), "the single card's need");
+    check(draft_layer_short(1830ull << 20, layer, head), "1.8 GiB free: the layer goes");
+    check(!draft_layer_short(3640ull << 20, layer, head), "3.6 GiB free (trimmed 8 GB card): the layer stays");
+    check(!draft_layer_short(0, layer, head), "0 free is an unreadable figure");
+    check(dflash2_need_bytes(715ull << 20, 213ull << 20) < draft_layer_need_bytes(layer, head), "DFlash2 needs less than the draft layer");
+    // 4. the KV window walks 32768 -> 20480 -> 8192 -> 4096 -> stop, and never goes past the context or the engine's minimum
+    check(next_kv_window(32768, 131072, 4096) == 20480, "32768 -> 20480");
+    check(next_kv_window(20480, 131072, 4096) == 8192, "20480 -> 8192");
+    check(next_kv_window(8192, 131072, 4096) == 4096, "8192 -> 4096");
+    check(next_kv_window(4096, 131072, 4096) == 0, "4096 is the last");
+    check(next_kv_window(0, 131072, 4096) == 20480, "the whole context in VRAM goes to 20480");
+    check(next_kv_window(0, 16384, 4096) == 8192, "a context of 16K has no 20480 step");
+    check(next_kv_window(0, 4096, 4096) == 0, "a context at the minimum has none");
+    check(next_kv_window(8192, 131072, 8192) == 0, "a larger engine minimum stops the walk");
+    // 5. the shares of the layers
+    int64_t fm[2] = {7635, 7635}, out[2] = {0, 0};
+    proportional_layers(fm, 2, 48, 1550, out);
+    check(out[0] == 24 && out[1] == 24, "two equal cards share 48 layers 24/24");
+    int64_t fm2[2] = {11000, 7000};
+    proportional_layers(fm2, 2, 48, 1550, out);
+    check(out[0] + out[1] == 48 && out[0] > out[1], "the bigger card takes more layers");
+    int64_t fm3[3] = {8000, 8000, 8000}, out3[3];
+    proportional_layers(fm3, 3, 48, 1550, out3);
+    check(out3[0] == 16 && out3[1] == 16 && out3[2] == 16, "three equal cards 16/16/16");
+    int64_t fm4[2] = {100000, 1700}, out4[2];
+    proportional_layers(fm4, 2, 48, 1550, out4);
+    check(out4[0] + out4[1] == 48 && out4[1] >= 0, "a tiny card leaves the sum right");
+    if (g_fail == 0) std::printf("split_walk_test: ok\n");
+    return g_fail == 0 ? 0 : 1;
+}

tests/terminal/test_drafting.py

@@ -613,6 +613,27 @@ class TheInstalledEngineDecidesTheFlags(unittest.TestCase):
             mtp_retry_args({**base, "drafting": "auto"}, old)
         self.assertEqual(drafting.take_note(), "")
 
+    def test_the_no_drafter_retry_exists_for_an_auto_start_with_a_drafter_and_never_swaps_in_the_mtp(self):
+        """A start that ended for lack of graphics memory is repeated with no drafter (multi-card walk): DFlash2 -> MTP would take MORE memory from a card that is short."""
+        from serve.server import nodraft_retry_args
+        old = str(self.exe("old.exe", self.OLD))
+        base = {"model_name": GYRO, "args": list(self.args)}
+        with mock.patch("serve.server.drafting_vram_mib", return_value=None), mock.patch.object(drafting, "data_dir_of", return_value=self.data):
+            alt = nodraft_retry_args({**base, "drafting": "auto"}, old, ["--effort", "x"])
+            self.assertIsNotNone(alt)
+            for flag in ("--draft", "--draft-model", "--draft-fallback", "--mtp", "--mtp-draft-vocab"):
+                self.assertNotIn(flag, alt, flag)
+            self.assertEqual(alt[-2:], ["--effort", "x"], "what the start adds goes along")
+            self.assertIn("--native", alt, "everything else is the same start")
+            self.assertIsNotNone(nodraft_retry_args(dict(base), old), "no choice saved means Auto")
+            self.assertIsNone(nodraft_retry_args({**base, "drafting": "dflash2"}, old), "named by the person: it fails with its reason")
+            self.assertIsNone(nodraft_retry_args({**base, "drafting": "mtp"}, old), "named by the person: it fails with its reason")
+            self.assertIsNone(nodraft_retry_args({**base, "drafting": "off"}, old), "no drafter to drop")
+        drafting.take_note()
+        with mock.patch("serve.server.drafting_vram_mib", return_value=None), mock.patch.object(drafting, "data_dir_of", return_value=self.data):
+            nodraft_retry_args({**base, "drafting": "auto"}, old)
+        self.assertEqual(drafting.take_note(), "", "working the retry out leaves the note of the real start alone")
+
     def test_the_engine_sources_carry_the_words_the_program_is_read_for(self):
         src = (ROOT / "src" / "program" / "generate.cpp").read_text(encoding="utf-8")
         self.assertIn(drafting._MARK_FILE.decode(), src, "rc1's start message is what marks an engine that takes the vocabulary file alone")
@@ -657,9 +678,9 @@ class WordsMatchTheApp(unittest.TestCase):
         catalogs = {l: {**json.loads((folder / f"{l}.json").read_text(encoding="utf-8")), **json.loads((folder / f"df2-{l}.json").read_text(encoding="utf-8"))}
                     for l in ("en", "de", "es", "fr", "ja", "ko", "pt", "ru", "zh")}
         ids = {v: k for k, v in catalogs["en"].items()}
-        self.assertEqual(len(drafting.REASONS), 8)
+        self.assertEqual(len(drafting.REASONS), 9)
         lines = [drafting.mtp_line(r) for r in (drafting.DIDNT_FIT, drafting.NOT_HERE, drafting.NOT_SUPPORTED, drafting.NOT_DOWNLOADED, drafting.NOT_THIS_CARD, drafting.DIDNT_START)]
-        lines += [drafting.line("dflash2", drafting.NO_MTP), drafting.line("off", drafting.NONE_HERE)]
+        lines += [drafting.line("dflash2", drafting.NO_MTP), drafting.line("off", drafting.NONE_HERE), drafting.line("off", drafting.SHORT_OF_VRAM)]
         for line in lines:
             self.assertIn(line, ids, line)
             for lang, catalog in catalogs.items():

tests/terminal/test_multispill_launch.py

@@ -0,0 +1,78 @@
+"""Two small cards on a layer split (MULTISPILL, round 0212): what the launcher says when a card runs out of memory.
+
+The final line of a failed start is the engine's own reason, not the picture reader's harmless "ggml_cuda_init: ... no CUDA-capable device" that it prints when the
+cards are full and it runs on the processor; the next step does not tell the person to change the context or the number of requests.  Nothing here starts an engine."""
+from __future__ import annotations
+
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+from strata_cli.launch import runner as R
+from strata_control import vram_fit
+
+# the log of the 2 x 8 GB start that failed (HYBRID, srv118, 2 x RTX PRO 6000 held to 8 GB free by hogs, 128K): the engine's reason, then the picture reader's line
+SERVE_STOP = [
+    "strata serve: layer split, CUDA1 prompt path: prefill: device buffers for a chunk of 512 tokens do not fit (5 of 97247 MiB free at the failure; 555 MiB taken, the failing buffer wanted 2 MiB)",
+    "strata serve: the GPU has too little free VRAM for the prompt path: turn images off (setup: --vision no), close other programs using the GPU, use a shorter context, or read prompts in smaller chunks (--prefill 512)",
+    "ggml_cuda_init: failed to initialize CUDA: no CUDA-capable device is detected",
+]
+CACHE_STOP = [
+    "strata generate: layer split, CUDA1: 0.17 GiB free of 94.97, room for experts 0.00 GiB",
+    "strata generate: layer split, CUDA1 expert cache: no room",
+    "ggml_cuda_init: failed to initialize CUDA: no CUDA-capable device is detected",
+]
+
+
+class TheEnginesReason(unittest.TestCase):
+    def test_the_reason_is_the_engines_not_the_picture_readers(self):
+        self.assertEqual(vram_fit.engine_stop_reason(SERVE_STOP),
+                         "the GPU has too little free VRAM for the prompt path: turn images off (setup: --vision no), close other programs using the GPU, "
+                         "use a shorter context, or read prompts in smaller chunks (--prefill 512)")
+        self.assertEqual(vram_fit.engine_stop_reason(CACHE_STOP), "layer split, CUDA1 expert cache: no room")
+
+    def test_other_stops_and_the_walks_own_lines_are_not_a_memory_stop(self):
+        self.assertIsNone(vram_fit.engine_stop_reason(["strata generate: something else is wrong", "ggml_cuda_init: failed to initialize CUDA"]))
+        self.assertIsNone(vram_fit.engine_stop_reason([]))
+        self.assertIsNone(vram_fit.engine_stop_reason(None))
+        walk = ["strata generate: short of GPU memory: CUDA1 has 2.53 GiB free and a layer split card needs about 9.1 GiB: each later card loads only the dense weights of its own layers",
+                "strata serve: CUDA1: 293 MiB of VRAM free with everything loaded - LOW: its window graphs are captured on first use"]
+        self.assertIsNone(vram_fit.engine_stop_reason(walk), "what the walk did and a LOW warning say nothing about why a start stopped")
+
+    def test_the_last_memory_line_wins(self):
+        lines = ["strata generate: layer split, CUDA1 expert cache: no room", "strata serve: the GPU has too little free VRAM for the prompt path: x"]
+        self.assertTrue(vram_fit.engine_stop_reason(lines).startswith("the GPU has too little free VRAM"))
+
+
+class TheLastLine(unittest.TestCase):
+    def explain(self, lines, **result):
+        with tempfile.TemporaryDirectory() as tmp:
+            log = Path(tmp) / "engine.log"
+            log.write_text("\n".join(lines) + "\n", encoding="utf-8")
+            cfg = Path(tmp) / "run.json"
+            cfg.write_text(json.dumps({"exe": "e", "args": [], "log": str(log)}), encoding="utf-8")
+            return R.explain_engine_exit({"state": "failed", "error": "server_exited", "engine_exit_code": 1, **result}, cfg)
+
+    def test_a_prompt_path_stop_shows_the_engines_line_and_a_next_step_that_keeps_the_context(self):
+        text, nxt = self.explain(SERVE_STOP)
+        self.assertIn("too little free VRAM for the prompt path", text)
+        self.assertNotIn("ggml_cuda_init", text)
+        self.assertNotIn("no CUDA-capable device", text)
+        self.assertIn("were not changed", nxt, "the context and the requests stay what the person chose")
+        self.assertIn("fewer cards", nxt)
+        self.assertNotIn("--context 16k", nxt, "the old advice asked for a smaller context")
+
+    def test_a_stage_cache_stop_shows_the_engines_line(self):
+        text, _ = self.explain(CACHE_STOP)
+        self.assertIn("layer split, CUDA1 expert cache: no room", text)
+        self.assertNotIn("ggml_cuda_init", text)
+
+    def test_a_stop_for_another_reason_is_worded_as_before(self):
+        text, nxt = self.explain(["strata generate: cudaMalloc failed", "unsupported quant type 99"])
+        self.assertIn("unsupported quant type 99", text)
+        self.assertNotIn("were not changed", nxt)
+
+
+if __name__ == "__main__":
+    unittest.main()