fit_start: CUDA Flash-Next request-slot figures (8 GB mimic), fewer requests before one at a time, plain hint on a slot-session OOM

BASE_SLOTTED_MIB gains (cuda, flash-next) 4990 from the 12 GB and 8 GB CUDA cells; a CUDA start with request slots needs a working cache (263 slots) not the one-slot refusal limit; the one-request fallback keeps the largest context a single request runs (16K on 8 GB); plan() tries fewer requests (down to 2) before one at a time; the draft-layer and head loads name --batch/--max-context when they run out of memory.

src/program/generate.cpp

@@ -277,6 +277,16 @@ std::string vram_free_note() {
     return buf;
 }
 
+// The plain reason for an out-of-memory in the loads that follow the request slots' sessions (the draft layer, the native head): with --batch N
+// those N sessions came out of the card before the cache was sized, and a card that is short for them has nothing left for these.
+std::string vram_batch_hint(const std::string& err, int batch) {
+    if (batch < 2 || err.find("out of memory") == std::string::npos) return {};
+    char buf[320];
+    std::snprintf(buf, sizeof buf, "; this start keeps %d request slots (--batch %d), each a session of its own, so the card is too small for them at this context: "
+                  "use a smaller --max-context or fewer --batch slots", batch, batch);
+    return buf;
+}
+
 #if defined(__linux__)
 // STRATA_CRASH_BT=1 (research): a fatal signal prints the call stack to stderr (the addresses go through addr2line -e <binary>)
 namespace {
@@ -6709,7 +6719,7 @@ int main(int argc, char** argv) {
         // longer window up in parts)
         const int mtp_t = o.pipeline_windows >= 2 ? strata::kernels::kVerifyNarrowT : std::min(o.spec, (int) strata::kernels::kVerifyNarrowT);
         if (o.pipeline_windows >= 2) mtp.set_force_capture(true);
-        if (!o.mtp.empty() && !mtp.load(o.mtp, draft_geometry, last_st && !mtp_first ? last_st->ss : ss, mtp_t, err, o.mtp_window)) { std::fprintf(stderr, "strata generate: %s%s\n", err.c_str(), vram_free_note().c_str()); return 1; }
+        if (!o.mtp.empty() && !mtp.load(o.mtp, draft_geometry, last_st && !mtp_first ? last_st->ss : ss, mtp_t, err, o.mtp_window)) { std::fprintf(stderr, "strata generate: %s%s%s\n", err.c_str(), vram_free_note().c_str(), vram_batch_hint(err, o.batch).c_str()); return 1; }
         if (!o.mtp.empty()) mtp.set_ple_session(&ss);
 #if defined(STRATA_HAVE_DFLASH2)
         if (o.draft == "dflash2") {
@@ -6833,7 +6843,7 @@ int main(int argc, char** argv) {
     if (!o.native_head_gguf.empty() && !multi_gpu) {   // a layer split's head is on its last stage
         const auto head_t0 = std::chrono::steady_clock::now();
         if (!native_head.load(o.native_head_shards, g.n_embd, n_vocab, err)) {
-            std::fprintf(stderr, "strata generate: %s%s\n", err.c_str(), vram_free_note().c_str());
+            std::fprintf(stderr, "strata generate: %s%s%s\n", err.c_str(), vram_free_note().c_str(), vram_batch_hint(err, o.batch).c_str());
             return 1;
         }
         std::fprintf(stderr, "strata generate: experimental native Q5_K head, %llu bytes, in %.1f s\n",

strata_control/vram_fit.py

@@ -60,8 +60,24 @@ BASE_GENERIC_MIB = 2448
 # engine log: 7721 MiB usable (8 GB less the 471 MiB gap), 2519 MiB free after everything but the expert cache, which held the weights (1466 + 1781 MiB), the
 # MTP layer (802), the Q5_K head (417), the main session (234) and the two slot sessions (468):  base = 7721 - 2519 - 234 - 468 = 4500 MiB.  The cache's
 # largest expert blob is 2.33 MB (2.22 MiB).  Another quant or card holds somewhat different weights; MARGIN_MIB and the engine's own refusal cover the rest.
-BASE_SLOTTED_MIB = {("intel", "flash-next"): 4500}      # (vendor, model family): measured on the A750 only; no CUDA / HIP entry until one is measured
+# CUDA, RTX PRO 5000 (sm_120) held to a 12 GB and an 8 GB card with a memory hog, IQ3_S pack, the MTP layer, --kv int8 --kv-resident 32768 at 131072 tokens (DF2-MEM,
+# RESULTS-DF2MEM.md section 17), the engine log.  The plan's own terms (head, 700 MiB reserve, prompt buffers at the 2048 chunk) stay in the formula, so this base is the
+# one that makes slack_mib() equal the engine's auto sizing of the cache, "free, less 700 reserved, 218 for the draft head, 256 allocated after the cache":
+#   12 GB, --batch 2: 3.53 GiB free before the cache  -> 2441 MiB for it (961 slots);   12 GB, --batch 4: 2.07 GiB -> 946 MiB (371 slots)
+#   base = 12000 usable - 748 main session - N * 748 slot sessions - (110 + 700 + 1522) - cache = 4984 (N 2) and 4994 (N 4), 4990 taken.
+# 8 GB (7800 MiB left, the same engine): --batch 2 has 0.83 GiB free after the two slot sessions, the MTP layer takes 835 and the head upload dies "out of memory
+# (7 MiB free)"; --batch 4 fits three sessions and the MTP layer's dense weights die "out of memory (99 MiB free)".  The formula says -1765 MiB for the first (the engine's own
+# figures say -1757), so both are refused by the plan and given a context or fewer requests that fit.
+BASE_SLOTTED_MIB = {("intel", "flash-next"): 4500, ("cuda", "flash-next"): 4990}      # (vendor, model family): the A750 and the CUDA engine; HIP has no entry until one is measured
 SLOT_SLOTTED_MIB = 2.22
+# CUDA, 8 GB mimic (7942 MiB left), 2 requests: a start the plan counted 177 MiB over the one-slot limit loaded, sized a 336-slot cache with the reserve lowered to
+# 424 MiB ("a working cache needs 263"), had 59 MiB free, and aborted at its first request ("ggml-cuda (strata mmq): out of memory").  So for request slots on CUDA the
+# threshold is the working cache the engine itself asks for: 263 slots of the largest blob (2.54 MiB with the IQ3_S pack).
+WORKING_CACHE_MIB = {"cuda": 263 * 2.54}
+# One request at a time on CUDA (a start that cannot hold its request slots): the largest context for which a single request runs, measured on the same mimic with the
+# MTP layer, 7942 MiB left: 2K / 4K / 8K / 16K start and decode (239 / 235 / 235 / 125 MiB free), 32K and 128K do not start ("the draft head does not fit",
+# "the device arena does not fit").  The formula below (measured base, one session, the one-slot limit) gives 471 / 442 / 386 / 273 and 47 / -128 MiB.
+SINGLE_FALLBACK_VENDORS = ("cuda",)
 SLOT_RUN_MIN = 1                                       # the engine's minimum: generate.cpp refuses a start whose auto sizing leaves the cache no slot
 KV_RESIDENT_MIN = 20480                                # qsa_kv_resident_min(): a smaller --kv-resident is raised to this
 STREAM_EXTRA_BYTES = 1400                              # per cell of context a streamed session still keeps in VRAM (rope table, indexer rows, page table); measured: the CUDA engine's slot session at 128K with 32768 resident is 0.73 GiB (RESULTS-DF2MEM.md section 17), the 32K session 0.56 GiB
@@ -126,14 +142,27 @@ def slack_mib(model: str, vram_gb: float, ctx: int, kv: str = "int8", reserve_mi
     return (vram_gb * 1024 - gap - base - session_mib(ctx, kv, kv_resident if slots >= 2 else 0) - extra - HEAD_MIB - max(int(reserve_mib), 0) - prompt)
 
 
+def single_slack_mib(model: str, vram_gb: float, ctx: int, kv: str = "int8", reserve_mib: int = 300, prefill="auto", vendor: str = "intel",
+                     kv_resident: int = 0) -> float:
+    """The slack of ONE request on the measured base (no slot sessions): what the one-request-at-a-time fallback is judged by."""
+    gap = GAP_MIB.get(str(vendor or "").lower(), GAP_OTHER_MIB)
+    base = slotted_base_mib(model, vendor)
+    if base is None:
+        base = base_mib(model)
+    return (vram_gb * 1024 - gap - base - session_mib(ctx, kv, kv_resident) - HEAD_MIB - max(int(reserve_mib), 0)
+            - (PROMPT_BASE_MIB + PROMPT_PER_TOKEN_MIB * chunk_of(prefill)))
+
+
 def slotted(model: str, slots: int, vendor: str = "intel") -> bool:
     """Is this a start with request slots whose cost is known (a measured base for this vendor)?"""
     return slots >= 2 and slotted_base_mib(model, vendor) is not None
 
 
 def needed_mib(model: str = "", slots: int = 0, vendor: str = "intel") -> float:
-    if slotted(model, slots, vendor):      # the engine's own minimum: the cache keeps at least a slot (plus the estimate's margin)
-        return SLOT_RUN_MIN * SLOT_SLOTTED_MIB + MARGIN_MIB
+    if slotted(model, slots, vendor):
+        if vendor in WORKING_CACHE_MIB:    # CUDA: a cache the engine calls working, not its one-slot refusal limit (see WORKING_CACHE_MIB)
+            return WORKING_CACHE_MIB[vendor] + MARGIN_MIB
+        return SLOT_RUN_MIN * SLOT_SLOTTED_MIB + MARGIN_MIB     # the engine's own minimum: the cache keeps at least a slot (plus the estimate's margin)
     return MIN_SLOTS * SLOT_MIB + MARGIN_MIB
 
 
@@ -182,27 +211,50 @@ def plan(model: str, vram_gb: float, ctx: int, kv: str = "int8", reserve_mib: in
     ctxs = [ctx] if ctx_fixed else [ctx] + [c for c in steps if c < ctx]
     chunks = [prefill] if prefill_fixed else [prefill] + [c for c in CHUNK_STEPS if c < chunk_of(prefill)]
     reserves = [reserve_mib] if reserve_fixed or reserve_mib <= RESERVE_FLOOR_MIB else [reserve_mib, RESERVE_FLOOR_MIB]
-    best = None
-    for c in ctxs:
-        for p in chunks:
-            for r in reserves:
-                s = slack_mib(model, vram_gb, c, kv, r, p, vendor, slots, kv_resident)
-                best = (c, p, r, s)
-                if s >= need:
-                    out = _result(model, vram_gb, title, ctx, reserve_mib, prefill, c, p, r, s, True)
-                    out.slots = slots
-                    if slots and out.changed:
-                        out.line = out.line.replace(" so it fits", f" so {slots} requests at once fit")
-                    return out
-    c, p, r, s = best
+
+    def ladder(n):
+        best = None
+        for c in ctxs:
+            for p in chunks:
+                for r in reserves:
+                    s = slack_mib(model, vram_gb, c, kv, r, p, vendor, n, kv_resident)
+                    best = (c, p, r, s)
+                    if s >= need:
+                        return best, True
+        return best, False
+
+    # the count asked for first, then fewer requests at once (down to 2) before one at a time: the card that cannot hold 4 slot sessions often holds 2
+    for n in ([slots] + list(range(slots - 1, 1, -1))):
+        (c, p, r, s), ok = ladder(n)
+        if ok:
+            out = _result(model, vram_gb, title, ctx, reserve_mib, prefill, c, p, r, s, True)
+            out.slots = n
+            if n != slots:
+                out.changed = True
+                out.line = (f"{title or 'This model'} on this {vram_gb:.0f} GB card: {slots} requests at once need more memory than is free here, so this start "
+                            f"answers {n} at once" + (f" with the context at {ctx_label(c)}" if c != ctx else ""))
+            elif slots and out.changed:
+                out.line = out.line.replace(" so it fits", f" so {slots} requests at once fit")
+            return out
+        if n == slots:
+            first = (c, p, r, s)
+    c, p, r, s = first
     out = _result(model, vram_gb, title, ctx, reserve_mib, prefill, c, p, r, s, False)
     out.slots = slots
     if slots and not ctx_fixed:      # not even the smallest context leaves the expert cache room for the request slots: one at a time, said plainly
         name = title or "This model"
         out.slots = 1
         out.changed = True
-        out.line = (f"{name} on this {vram_gb:.0f} GB card: {slots} requests at once need more memory than is free here, even with the context at "
-                    f"{ctx_label(c)}, so this start answers one request at a time")
+        if vendor in SINGLE_FALLBACK_VENDORS:     # one request needs far less than its slot sessions did: the largest context it runs with, not the smallest
+            one = SLOT_RUN_MIN * SLOT_SLOTTED_MIB + MARGIN_MIB
+            c1 = next((x for x in ctxs if single_slack_mib(model, vram_gb, x, kv, r, p, vendor, kv_resident) >= one), ctxs[-1])
+            out.ctx = c1
+            out.slack_mib = single_slack_mib(model, vram_gb, c1, kv, r, p, vendor, kv_resident)
+            out.line = (f"{name} on this {vram_gb:.0f} GB card: {slots} requests at once need more memory than is free here, so this start answers one request "
+                        f"at a time" + (f" with the context at {ctx_label(c1)}" if c1 != ctx else ""))
+        else:
+            out.line = (f"{name} on this {vram_gb:.0f} GB card: {slots} requests at once need more memory than is free here, even with the context at "
+                        f"{ctx_label(c)}, so this start answers one request at a time")
     return out
 
 
@@ -247,7 +299,7 @@ def fit_start(cfg: dict, args: list, vram_mib: float | None, backend: str | None
     reserve = int(reserve_s) if reserve_s and str(reserve_s).isdigit() else 700      # the engine's default reserve
     user = [str(a) for a in (cfg.get("user_args") or [])]
     model = str(cfg.get("model_name") or flag("--native") or "")
-    fr = plan(model, float(vram_mib) / 1024.0, ctx, flag("--kv") or "int8", reserve, flag("--prefill") or "auto", "intel" if backend == "sycl" else "other",
+    fr = plan(model, float(vram_mib) / 1024.0, ctx, flag("--kv") or "int8", reserve, flag("--prefill") or "auto", "intel" if backend == "sycl" else ("cuda" if backend == "cuda" else "other"),
               title=str(cfg.get("model_name") or "This model"), ctx_fixed="--max-context" in user, reserve_fixed=True, prefill_fixed=True, slots=n,
               kv_resident=kv_resident)
     if not fr.changed:

tests/terminal/test_vram_fit.py

@@ -280,5 +280,62 @@ class FitStart(unittest.TestCase):
         self.assertIn("--max-context", args)
 
 
+class CudaSlotted(unittest.TestCase):
+    """The CUDA engine's measured base (DF2-MEM, RESULTS-DF2MEM.md sections 17 and 18): an 8 GB NVIDIA card with 2 or 4 requests died with "native head upload: out of
+    memory" at 128K; 12 GB and up start.  The figures are the engine's own auto sizing of the cache on the RTX PRO 5000 held to 12000 / 7800 MiB."""
+
+    M = "qwen3.8-flash-next-iq3_s"
+    A = ["--native", "/m/Qwen3.8-Flash-Next-IQ3_S-00001-of-00002.gguf", "--max-context", "131072", "--kv", "int8", "--kv-resident", "32768",
+         "--vram-reserve-mib", "700"]
+
+    def slack(self, free_mib, slots):
+        return V.slack_mib(self.M, (free_mib + V.GAP_OTHER_MIB) / 1024.0, 131072, "int8", 700, "auto", "cuda", slots, 32768)
+
+    def start(self, vram_mib, n, backend="cuda"):
+        return V.fit_start({"model_name": self.M, "parallel": n}, list(self.A), vram_mib, backend)
+
+    def test_the_base_reproduces_the_engines_cache_room(self):
+        self.assertAlmostEqual(self.slack(12000, 2), 3615 - 700 - 218 - 256, delta=60)     # "expert cache auto: 3.53 GiB free ... -> 961 slots"
+        self.assertAlmostEqual(self.slack(12000, 4), 2120 - 700 - 218 - 256, delta=60)     # "2.07 GiB free ... -> 371 slots"
+        self.assertAlmostEqual(self.slack(7800, 2), 850 - 835 - 598 - 700 - 218 - 256, delta=60)     # 0.83 GiB after the sessions, then the MTP layer and the head: the OOM
+
+    def test_an_8gb_card_with_2_or_4_requests_runs_one_at_a_time_at_the_context_one_request_runs(self):
+        for n in (2, 4):
+            args, par, note = self.start(8192, n)
+            self.assertEqual(par, 1, n)
+            self.assertEqual(args[args.index("--max-context") + 1], "16384", n)       # measured: 16K starts and decodes on the 8 GB mimic, 32K and 128K do not
+            self.assertIn("one request at a time with the context at 16K", note)
+
+    def test_the_8gb_mimic_figures_behind_it(self):
+        gb = (7942 + V.GAP_OTHER_MIB) / 1024.0
+        one = V.SLOT_RUN_MIN * V.SLOT_SLOTTED_MIB + V.MARGIN_MIB
+        for c in (2048, 4096, 8192, 16384):
+            self.assertGreaterEqual(V.single_slack_mib(self.M, gb, c, "int8", 700, "auto", "cuda", 32768), one, c)     # started and decoded
+        for c in (32768, 131072):
+            self.assertLess(V.single_slack_mib(self.M, gb, c, "int8", 700, "auto", "cuda", 32768), one, c)             # did not start
+        two_at_4k = V.slack_mib(self.M, gb, 4096, "int8", 700, "auto", "cuda", 2, 32768)
+        self.assertLess(two_at_4k, V.needed_mib(self.M, 2, "cuda"))                                                     # 59 MiB free, aborted at its first request
+
+    def test_a_card_in_between_loses_requests_before_it_loses_all_of_them(self):
+        args, par, note = self.start(9216, 8)
+        self.assertEqual(par, 5)
+        self.assertIn("answers 5 at once", note)
+
+    def test_12_16_and_24gb_keep_128k_with_2_and_4_requests(self):
+        for mib in (12288, 12282, 16384, 24576):
+            for n in (2, 3, 4):
+                self.assertEqual(self.start(mib, n), (self.A, None, ""), (mib, n))
+
+    def test_only_the_cuda_engine_is_claimed(self):
+        self.assertEqual(self.start(8192, 2, "hip"), (self.A, None, ""))          # no measured HIP figure: the engine decides
+        self.assertEqual(self.start(8192, 2, "metal"), (self.A, None, ""))
+
+    def test_the_persons_own_context_is_never_lowered_for_requests(self):
+        f = V.plan(self.M, 8.0, 131072, "int8", 700, "auto", "cuda", slots=8, ctx_fixed=True, reserve_fixed=True, prefill_fixed=True, kv_resident=32768)
+        self.assertEqual(f.ctx, 131072)
+        self.assertEqual(f.slots, 8)                                              # the person's own context is kept and nothing is lowered under it: the engine's refusal answers
+        self.assertFalse(f.changed)
+
+
 if __name__ == "__main__":
     unittest.main()