batch-mtp on automatically for one big card (24 GB+, several slots, MTP on); off on small cards and layer splits; Expert mode MTP per request overrides

apps/host/server/native-parallel.mjs

@@ -43,12 +43,21 @@ export const CACHE_LOSS_SHARE = 0.2;
 export const FIRST_CARD_OTHER_GB = 6.9;
 export const NEXT_CARD_OTHER_GB = 2.0;
 /**
- * `--batch-mtp` lets each slot verify draft tokens. Measured on a 12 GB RTX 3060 (docs/BATCHING.md says it pays on larger
- * cards): a lone chat 35.6 tok/s with it against 37.8 without, 2 chats 37.2 against 33.8 together, 4 chats 42.5 against 38.4 -
- * a small gain for several chats, a small loss for one, and 400 MiB less expert cache. Off until a card shows it pays; Expert
- * mode adds it by hand (`--batch-mtp` in the engine arguments).
+ * `--batch-mtp` lets each slot verify draft tokens. Where it pays and where it does not (all measured):
+ *   - RTX PRO 5000 72 GB, 32K (FP4-Local): 1 stream 140 tok/s either way; 2 streams 129 -> 160 (+24%) with it; 4 streams 177 either way;
+ *     the text is exactly the batch text.
+ *   - 12 GB RTX 3060: a lone chat 35.6 tok/s with it against 37.8 without, 2 chats 37.2 against 33.8, 4 chats 42.5 against 38.4, and
+ *     400 MiB less expert cache: not worth it on a small card.
+ *   - 2 cards in a layer split with all experts in VRAM: -22% (docs/BATCHING.md): never on a layer split.
+ * 'auto' turns it on for one card of at least BATCH_MTP_MIN_VRAM_GB with a draft layer, 2 or more slots, and room for the slot drafters
+ * with a margin. 'off' never adds it. Expert mode decides either way: `--batch-mtp` in the engine arguments, or "batchMtp": true / false
+ * in the model's settings (or the load's values).
  */
-export const BATCH_MTP_POLICY = 'off';
+export const BATCH_MTP_POLICY = 'auto';
+export const BATCH_MTP_MIN_VRAM_GB = 24;
+/** Memory the slot drafters need per slot (their K/V ring and buffers; measured about 0.1 GiB, rounded up) and the cache that must remain after them. */
+export const BATCH_MTP_DRAFTER_GB = 0.25;
+export const BATCH_MTP_MARGIN_GB = 2;
 
 const argValue = (args, flag) => { const at = args.indexOf(flag); return at >= 0 ? args[at + 1] : undefined; };
 
@@ -90,19 +99,25 @@ export function chooseParallel({ config = {}, values = {}, hardware = null, requ
   // A number someone chose (including 1) is the person's decision, and so is an explicit --batch in the engine arguments.
   if (facts.explicitBatch || (chosen !== null && Number.isFinite(chosen))) {
     const slots = facts.explicitBatch ? null : Math.max(1, Math.min(1024, Math.trunc(chosen)));
-    return { slots: slots ?? 0, batchMtp: false, explicit: true, requested, note: null, slotGb: slot };
+    const wantMtp = (values.batchMtp ?? config.batchMtp) === true && !facts.batchMtp && facts.mtp && facts.gpus.length === 1 && (slots ?? 0) > 1;
+    return { slots: slots ?? 0, batchMtp: wantMtp, explicit: true, requested, note: null, slotGb: slot };
   }
   const cards = (hardware?.gpus ?? []).filter(g => facts.gpus.includes(g.index)).map(g => Number(g.vram_gb)).filter(n => Number.isFinite(n) && n > 0);
   // Without a hardware read nothing can be sized: one at a time is the safe default (a number can still be chosen).
   if (!cards.length) return { slots: PARALLEL_POLICY === 'fixed' ? requested : 1, batchMtp: false, explicit: false, requested, note: null, slotGb: slot };
   const total = cards.reduce((a, b) => a + b, 0);
+  const oneCard = facts.gpus.length === 1 && cards.length === 1;
   const soloCache = Math.max(0, total - FIRST_CARD_OTHER_GB - NEXT_CARD_OTHER_GB * (cards.length - 1));
   let slots = requested;
   if (PARALLEL_POLICY === 'fit') while (slots > 1 && slots * slot > CACHE_LOSS_SHARE * soloCache) slots -= 1;
   const note = slots >= requested ? null
     : slots > 1 ? `Your graphics card has room for ${slots} requests at once with this context size, so Strata set that. A smaller context size leaves room for more.`
       : 'Strata set this model to one request at a time: your graphics card is best used that way, because more at once would make every answer slower. Helper agents and a second chat take their turn instead. A smaller context size leaves room for more at once.';
-  return { slots, batchMtp: BATCH_MTP_POLICY === 'on' && slots > 1 && facts.mtp && !facts.batchMtp && facts.gpus.length === 1, explicit: false, requested, note, slotGb: slot };
+  const wish = values.batchMtp ?? config.batchMtp;   // Expert mode: true adds it by hand, false keeps it off
+  const roomLeft = soloCache - slots * (slot + BATCH_MTP_DRAFTER_GB);
+  const auto = BATCH_MTP_POLICY === 'auto' && oneCard && cards[0] >= BATCH_MTP_MIN_VRAM_GB && roomLeft >= BATCH_MTP_MARGIN_GB;
+  const batchMtp = slots > 1 && facts.mtp && !facts.batchMtp && wish !== false && (wish === true ? oneCard : auto);
+  return { slots, batchMtp, explicit: false, requested, note, slotGb: slot };
 }
 
 /**
@@ -110,8 +125,11 @@ export function chooseParallel({ config = {}, values = {}, hardware = null, requ
  * Returns the configuration and the plain-words note (or null).
  */
 export function applyParallel(cfg, choice) {
-  if (choice.explicit) return { cfg, note: null };
-  const next = { ...cfg };
+  // `batchMtp` is the app's own setting (Expert mode); the engine's config does not know it, the argument is what it reads.
+  const { batchMtp: _wish, ...clean } = cfg;
+  if (choice.explicit && !choice.batchMtp) return { cfg: 'batchMtp' in cfg ? clean : cfg, note: null };
+  if (choice.explicit) return { cfg: { ...clean, args: [...(clean.args ?? []), '--batch-mtp'] }, note: null };
+  const next = { ...clean };
   if (choice.slots > 1) next.parallel = choice.slots; else delete next.parallel;
   if (choice.batchMtp) next.args = [...(next.args ?? []), '--batch-mtp'];
   return { cfg: next, note: choice.note };

apps/host/server/native-settings.mjs

@@ -7,6 +7,7 @@ export const NATIVE_FIELDS=[
  {id:'aliases',label:'API model aliases',group:'Serving',type:'array',hint:'Comma-separated names clients may use for this model.'},
  {id:'idle_unload_s',label:'Unload after idle seconds',group:'Serving',type:'number',min:0,max:86400,hint:'Zero keeps the model loaded.'},
  {id:'lazy_load',label:'Load on first request',group:'Serving',type:'boolean'},
+ {id:'batchMtp',label:'MTP per request',group:'Serving',type:'boolean',hint:'Each concurrent request uses the draft layer. Left alone, Strata turns it on for one graphics card with 24 GB or more and several requests at once, and off otherwise; choose on or off here to decide yourself.'},
  {id:'parallel',label:'Concurrent requests',group:'Serving',type:'number',min:1,max:8,hint:'More slots need more memory and can slow each response.'},
  {id:'engine_silence_s',label:'Engine stall timeout (seconds)',group:'Recovery',type:'number',min:10,max:3600},
  {id:'api_monitor',label:'Keep API request metadata',group:'Recovery',type:'boolean'},

apps/host/tests/parallel-default.test.mjs

@@ -38,17 +38,29 @@ test('sizing: the slots may take at most a fifth of the expert cache the card wo
   assert.doesNotMatch(small.note + some.note, /--|batch|VRAM|GiB|tok|cache/i, 'no engine words');
 });
 
-test('batch-mtp is off by default (measured: it does not pay on a 12 GB card); where the policy is on it applies to one card with a draft layer only', () => {
-  const mtp = c => chooseParallel({ config: c, hardware: hw(32) });
-  assert.equal(BATCH_MTP_POLICY, 'off');
-  assert.equal(mtp(cfg(32768)).batchMtp, false);
-  const applied = applyParallel(cfg(32768), mtp(cfg(32768)));
-  assert.equal(applied.cfg.parallel, 4); assert.ok(!applied.cfg.args.includes('--batch-mtp'));
-  // the rule itself, for the day the policy is turned on
-  if (BATCH_MTP_POLICY === 'on') {
-    assert.equal(mtp(cfg(32768)).batchMtp, true);
-    assert.equal(mtp(cfg(32768, { gpu: [0, 1] })).batchMtp, false);
-  }
+test('batch-mtp is automatic on big single cards only: 24 GB and up with room for the drafters, never on small cards or a layer split; Expert mode overrides either way', () => {
+  assert.equal(BATCH_MTP_POLICY, 'auto');
+  const mtp = (gb, extra = {}, cards = 1, values = {}, context = 32768) => chooseParallel({ config: cfg(context, extra), hardware: hw(gb, cards), values });
+  assert.deepEqual([12, 16, 20].map(gb => mtp(gb).batchMtp), [false, false, false], 'small cards: it did not pay (3060: -6% for one chat, 400 MiB less cache)');
+  assert.deepEqual([24, 32, 48, 72].map(gb => mtp(gb).batchMtp), [true, true, true, true]);
+  assert.equal(mtp(72).slots, 4);
+  assert.equal(mtp(24, {}, 1, {}, 131072).batchMtp, false, 'one slot only at this context: nothing to batch');
+  assert.equal(mtp(72, { gpu: [0, 1] }, 2).batchMtp, false, 'a layer split: -22% with all experts in VRAM');
+  assert.equal(chooseParallel({ config: { args: ['--max-context', '32768'], gpu: 0 }, hardware: hw(72) }).batchMtp, false, 'no draft layer, nothing to batch');
+  assert.equal(chooseParallel({ config: cfg(32768, { args: ['--max-context', '32768', '--mtp', '/m', '--batch-mtp'] }), hardware: hw(72) }).batchMtp, false, 'already there: not added twice');
+  const applied = applyParallel(cfg(32768), mtp(72));
+  assert.ok(applied.cfg.args.includes('--batch-mtp')); assert.equal(applied.cfg.parallel, 4);
+  // Expert mode: off wins on a big card; on wins on a small one-card setup; an explicit slot count can still ask for it
+  assert.equal(mtp(72, {}, 1, { batchMtp: false }).batchMtp, false);
+  assert.equal(mtp(72, { batchMtp: false }).batchMtp, false);
+  assert.equal(mtp(20, {}, 1, { batchMtp: true }).batchMtp, true, '20 GB: 2 slots, asked for by hand');
+  assert.equal(mtp(72, { gpu: [0, 1] }, 2, { batchMtp: true }).batchMtp, false, 'never forced on a layer split');
+  const manual = chooseParallel({ config: cfg(32768), values: { parallel: 2, batchMtp: true }, hardware: hw(12) });
+  assert.deepEqual([manual.slots, manual.explicit, manual.batchMtp], [2, true, true]);
+  assert.ok(applyParallel(cfg(32768), manual).cfg.args.includes('--batch-mtp'));
+  assert.ok(!applyParallel(cfg(32768, { parallel: 2 }), chooseParallel({ config: cfg(32768, { parallel: 2 }), hardware: hw(72) })).cfg.args.includes('--batch-mtp'), 'a number the person chose alone does not add it');
+  // the retry with one request at a time takes it away again
+  assert.ok(!withoutParallel(applied.cfg).args.includes('--batch-mtp'));
 });
 
 test("a number the person chose is kept exactly: Expert mode's value, the saved settings, an explicit --batch", () => {
@@ -106,7 +118,7 @@ test('the installed app loads with 4 request slots by default', async () => {
     assert.equal(job.state, 'done', job.error);
     assert.equal(f.saved.length, 1);
     assert.equal(f.saved[0].parallel, 4);
-    assert.ok(!f.saved[0].args.includes('--batch-mtp'));
+    assert.ok(f.saved[0].args.includes('--batch-mtp'), 'a 32 GB card: MTP per request is on automatically');
     assert.equal(f.native.state().instances[0].parallel, 4, 'the agent reads the slot count from here');
   } finally { await f.native.close(); f.cleanup(); }
 });