Fit checks and slot sizing count the draft layer's ~1 GB of graphics memory while MTP is on

apps/host/server/native-parallel.mjs

@@ -41,6 +41,12 @@ export const CACHE_LOSS_SHARE = 0.2;
  * about 6.9 GiB goes to everything else; a second card in a layer split about 2 GiB).
  */
 export const FIRST_CARD_OTHER_GB = 6.9;
+/**
+ * The draft layer (MTP) is part of that 6.9: about 1 GB of graphics memory (786 MiB of files and the draft head's ~218 MiB,
+ * phaseA-tests/notes-021-settings.md). With "Draft layer (MTP)" off in Expert settings (`"mtp": false`) the engine does not load it, so
+ * the card has that much more cache and the slots may use it.
+ */
+export const MTP_VRAM_GB = 1.0;
 export const NEXT_CARD_OTHER_GB = 2.0;
 /**
  * `--batch-mtp` lets each slot verify draft tokens. Where it pays and where it does not (all measured):
@@ -77,7 +83,7 @@ export function engineFacts(config = {}) {
   const gpus = Array.isArray(config.gpu) ? config.gpu : config.gpu === undefined || config.gpu === null ? [0] : [config.gpu];
   return {
     args, context, kv, family, gpus,
-    mtp: args.includes('--mtp'),
+    mtp: args.includes('--mtp') && config.mtp !== false,
     batchMtp: args.includes('--batch-mtp'),
     explicitBatch: args.includes('--batch') || args.includes('--slots'),
   };
@@ -107,7 +113,7 @@ export function chooseParallel({ config = {}, values = {}, hardware = null, requ
   if (!cards.length) return { slots: PARALLEL_POLICY === 'fixed' ? requested : 1, batchMtp: false, explicit: false, requested, note: null, slotGb: slot };
   const total = cards.reduce((a, b) => a + b, 0);
   const oneCard = facts.gpus.length === 1 && cards.length === 1;
-  const soloCache = Math.max(0, total - FIRST_CARD_OTHER_GB - NEXT_CARD_OTHER_GB * (cards.length - 1));
+  const soloCache = Math.max(0, total - (FIRST_CARD_OTHER_GB - (facts.mtp ? 0 : MTP_VRAM_GB)) - NEXT_CARD_OTHER_GB * (cards.length - 1));
   let slots = requested;
   if (PARALLEL_POLICY === 'fit') while (slots > 1 && slots * slot > CACHE_LOSS_SHARE * soloCache) slots -= 1;
   const note = slots >= requested ? null

apps/host/tests/parallel-default.test.mjs

@@ -168,3 +168,12 @@ test('a load that fails even with one slot reports the failure, and a chosen num
     assert.equal(chosen.saved.length, 1);
   } finally { await chosen.native.close(); chosen.cleanup(); }
 });
+
+test('the draft layer (MTP) is part of what the card is sized for; switched off, its 1 GB goes to the slots', () => {
+  if (PARALLEL_POLICY !== 'fit') return;
+  const slots = (gb, extra = {}, args) => chooseParallel({ config: { ...cfg(32768, extra), ...(args ? { args } : {}) }, hardware: hw(gb) }).slots;
+  assert.equal(slots(26.5), 3, 'MTP on (the default): the measured sizing stands');
+  assert.equal(slots(26.5, { mtp: false }), 4, 'a card with the draft layer off has 1 GB more cache');
+  assert.equal(slots(26.5, {}, ['--max-context', '32768', '--spec', '4']), 4, 'no draft layer in the arguments');
+  assert.equal(slots(12, { mtp: false }), 1, 'a 12 GB card still gets one request at a time');
+});

setup.py

@@ -4961,32 +4961,37 @@ def low_ram_needed(model, ram) -> bool:
     return ram < MODELS[model]["arena_gb"] + LOW_RAM_HEADROOM_GB
 
 
-def low_ram_gpu_gb(model, vram_gb, ctx=32768, kv="int8") -> float:
+#: The draft layer (MTP) takes about 1 GB of graphics memory: its 786 MiB of files plus the draft head's ~218 MiB
+#: (phaseA-tests/notes-021-settings.md).  It comes out of the expert cache, so every fit check counts it while MTP is on.
+MTP_VRAM_GB = 1.0
+
+
+def low_ram_gpu_gb(model, vram_gb, ctx=32768, kv="int8", mtp=True) -> float:
     """About how many GB of the model's experts the GPU's cache holds: its VRAM minus ~5 GB for the dense weights,
     buffers and a 32K context's KV cache, minus the KV cache of a longer context (in VRAM in the low-RAM mode: its RAM
-    has no room for KV streaming)."""
+    has no room for KV streaming), minus ~1 GB for the draft layer (MTP) when it is on."""
     kv_tok = 13 * KV_CELL_BYTES.get(kv, 1056)           # bytes per context token: 12 QSA layers + the draft layer
     longer = max(0, ctx - 32768) * kv_tok / 1e9
-    return max(0.0, min(MODELS[model]["arena_gb"], vram_gb - 5 - longer))
+    return max(0.0, min(MODELS[model]["arena_gb"], vram_gb - 5 - (MTP_VRAM_GB if mtp else 0.0) - longer))
 
 
-def low_ram_gpu_share(model, vram_gb, ctx=32768, kv="int8") -> float:
+def low_ram_gpu_share(model, vram_gb, ctx=32768, kv="int8", mtp=True) -> float:
     """About how much of the model's experts the GPU holds."""
-    return low_ram_gpu_gb(model, vram_gb, ctx, kv) / MODELS[model]["arena_gb"]
+    return low_ram_gpu_gb(model, vram_gb, ctx, kv, mtp) / MODELS[model]["arena_gb"]
 
 
-def low_ram_resident(model, ram, vram_gb, ctx=32768, kv="int8") -> bool:
+def low_ram_resident(model, ram, vram_gb, ctx=32768, kv="int8", mtp=True) -> bool:
     """In the low-RAM mode: the experts the GPU does not hold fit the RAM with the usual room beside them, so they are
     copied into RAM once (the resident variant, `--resident-experts`) instead of being read through the OS file cache
     (plain `--mmap-experts`, which a PC this short of RAM keeps re-reading from the SSD)."""
-    rest = MODELS[model]["arena_gb"] - low_ram_gpu_gb(model, vram_gb, ctx, kv)
+    rest = MODELS[model]["arena_gb"] - low_ram_gpu_gb(model, vram_gb, ctx, kv, mtp)
     return ram >= rest + LOW_RAM_HEADROOM_GB
 
 
-def low_ram_fits(model, ram, vram_gb) -> bool:
+def low_ram_fits(model, ram, vram_gb, mtp=True) -> bool:
     """In the low-RAM mode: the experts the GPU does not hold fit the RAM left beside the rest (as file cache)."""
     arena = MODELS[model]["arena_gb"]
-    return ram - 6 + max(0.0, vram_gb - 5) >= arena
+    return ram - 6 + max(0.0, vram_gb - 5 - (MTP_VRAM_GB if mtp else 0.0)) >= arena
 
 
 def low_ram_wanted(model, ram, choice="auto") -> bool:
@@ -7711,12 +7716,13 @@ def main() -> int:
     if low_ram and not fam.get("ready"):               # NVFP4's packs: the RAM-budget mode instead (step 7)
         arena = MODELS[model]["arena_gb"]
         vram = low_ram_vram(gpu) - (VISION[vision]["reserve_mib"] / 1024 if vision != "none" else 0)
-        share = low_ram_gpu_share(model, vram, ctx, kv)
-        rest = arena - low_ram_gpu_gb(model, vram, ctx, kv)
-        resident = a.low_ram == "resident" or (a.low_ram != "mmap" and low_ram_resident(model, ram, vram, ctx, kv))
+        with_mtp = a.mtp != "off"                       # the draft layer's ~1 GB comes out of the expert cache
+        share = low_ram_gpu_share(model, vram, ctx, kv, with_mtp)
+        rest = arena - low_ram_gpu_gb(model, vram, ctx, kv, with_mtp)
+        resident = a.low_ram == "resident" or (a.low_ram != "mmap" and low_ram_resident(model, ram, vram, ctx, kv, with_mtp))
         if multi:      # every chosen card's share (the image encoder on the main one); #364 #384: the mapped variant
-            held = min(arena, low_ram_gpu_gb(model, vram, ctx, kv) +
-                       sum(low_ram_gpu_gb(model, low_ram_vram(x), ctx, kv) for x in chosen[1:]))
+            held = min(arena, low_ram_gpu_gb(model, vram, ctx, kv, with_mtp) +
+                       sum(low_ram_gpu_gb(model, low_ram_vram(x), ctx, kv, with_mtp) for x in chosen[1:]))
             share, rest = held / arena, arena - held
             # #642: the resident variant on a split, by the same rule as on one card - what no card holds fits the RAM
             resident = resident_split() and (a.low_ram == "resident" or

tools/strata_mcp.py

@@ -1013,7 +1013,7 @@ class Strata:
 
         def low_fits(m):
             try:
-                return S.low_ram_fits(m, ram, vram) if S else ram - 6 + max(0.0, vram - 5) >= models[m]["arena_gb"]
+                return S.low_ram_fits(m, ram, vram) if S else ram - 6 + max(0.0, vram - 5 - 1.0) >= models[m]["arena_gb"]
             except Exception:                           # noqa: BLE001
                 return False
         notes = []

tools/test_setup_lowram.py

@@ -80,5 +80,23 @@ class PagingQuestion(unittest.TestCase):
                 self.assertIn("[y]" if choice == "off" else "[n]", asked[0])
 
 
+class DraftLayerCountsInTheFit(unittest.TestCase):
+    """The draft layer (MTP) takes about 1 GB of graphics memory out of the expert cache: every fit check counts it while it is on."""
+
+    def test_the_card_holds_one_gb_less_with_mtp_on(self):
+        on = setup.low_ram_gpu_gb("IQ3_XXS", 12.0)
+        off = setup.low_ram_gpu_gb("IQ3_XXS", 12.0, mtp=False)
+        self.assertAlmostEqual(off - on, setup.MTP_VRAM_GB)
+        self.assertEqual(setup.MTP_VRAM_GB, 1.0)
+        self.assertAlmostEqual(setup.low_ram_gpu_share("IQ3_XXS", 12.0, mtp=False) - setup.low_ram_gpu_share("IQ3_XXS", 12.0),
+                               1.0 / setup.MODELS["IQ3_XXS"]["arena_gb"])
+
+    def test_the_edge_of_what_fits_moves_by_the_draft_layer(self):
+        arena = setup.MODELS["IQ1_M"]["arena_gb"]
+        ram = arena + 6 - 7.0 + 0.5            # a card of 12 GB holds 7 GB beside the dense part; the draft layer needs one of them
+        self.assertFalse(setup.low_ram_fits("IQ1_M", ram, 12.0))
+        self.assertTrue(setup.low_ram_fits("IQ1_M", ram, 12.0, mtp=False))
+
+
 if __name__ == "__main__":
     unittest.main()