mirror of
https://github.com/Niko1221/Strata.git
synced 2026-11-07 06:15:58 -10:00
split: Auto drafting on a setup the split accepts runs the split with its MTP; DFlash2 or off chosen by the person keeps the layer split
split_config writes "drafting": "mtp" (what the split engine runs), so serve/server.py's drafter step, which resolves Auto to DFlash2 where the engine has it, cannot add DFlash2 flags to the split engine's arguments; the fallback keeps the person's own drafting word (or none: Auto), so the layer split that starts instead gets DFlash2 as usual. The rule refuses (quietly in auto) for an explicit dflash2 or off, and for Auto on a card where the split has not been measured to decode faster than the layer split with DFlash2. Tests: all four cases, the card rule, the server's drafter step on a split config.
This commit is contained in:
1 parent
ac1965edea
commit
15fb6ffe3f
3 files changed
+73
-1
No files matched your search
@@ -97,6 +97,29 @@ class Fallback(unittest.TestCase):
|
||||
self.assertTrue(os.path.exists(log + ".split-engine")) # the normal engine will overwrite the log; the refusal stays readable
|
||||
self.assertEqual([s[0] for s in FakeEngine.started], ["/e/strata-split-2", NORMAL])
|
||||
|
||||
def test_a_split_config_keeps_its_arguments_through_the_drafter_step(self):
|
||||
"""strata_control/split_engine.split_config writes "drafting": "mtp" (what the split engine runs), so the drafter step, which resolves Auto to
|
||||
DFlash2 where the engine has it, adds no DFlash2 flag to the split engine's arguments (it would refuse them and every start would fall back)."""
|
||||
from strata_control import split_engine as SE
|
||||
|
||||
base = {"exe": NORMAL, "args": ["--pack", "/p", "--native", "/m/a.gguf", "--mtp", "/d/mtp/rt", "--layer-split", "auto"], "cwd": "/x",
|
||||
"tokenizer": "/t", "port": 8080}
|
||||
decision = SE.Decision(True, [], 2)
|
||||
cfg = SE.split_config(base, decision, exe="/e/strata-split-2", native="/m/a.gguf", mtp_dir="/d/mtp/rt", draft_vocab="/d/v.bin", context=131072)
|
||||
args = S.engine_args(cfg)
|
||||
self.assertEqual(args[:2], ["--native", "/m/a.gguf"])
|
||||
self.assertNotIn("--draft", args)
|
||||
self.assertNotIn("--draft-model", args)
|
||||
self.assertEqual(args[args.index("--mtp") + 1], "/d/mtp/rt")
|
||||
self.assertEqual(args[args.index("--spec") + 1], "6")
|
||||
self.assertNotIn("--layer-split", args)
|
||||
# the layer split that starts instead keeps the person's drafting word, or none (Auto)
|
||||
self.assertNotIn("drafting", cfg["split_fallback"])
|
||||
cfg = SE.split_config({**base, "drafting": "auto"}, decision, exe="/e/strata-split-2", native="/m/a.gguf", mtp_dir="/d/mtp/rt",
|
||||
draft_vocab="/d/v.bin", context=131072)
|
||||
self.assertEqual(cfg["split_fallback"]["drafting"], "auto")
|
||||
self.assertEqual(cfg["drafting"], "mtp")
|
||||
|
||||
def test_without_a_fallback_the_failure_stays_a_failure(self):
|
||||
cfg = split_cfg()
|
||||
del cfg["split_fallback"]
|
||||
|
||||
@@ -37,6 +37,9 @@ TESTED_CARD_COUNTS = (2,) # run with the host exchange: 2 cards (boston
|
||||
TESTED_ARCHS = (86, 120) # compute capability: sm_86 (the 61-task suite, pictures, hashes) and sm_120 (decode, prompts, parallel, hashes,
|
||||
# pictures; no suite there)
|
||||
MODES = ("auto", "on", "off")
|
||||
#: cards (compute capability) on which Auto drafting resolves to split + MTP rather than layer split + DFlash2: the split decodes faster there
|
||||
#: (phaseA-tests/sfnx/RESULTS.md section 17). Another card is not auto-split at all (TESTED_ARCHS), so it keeps Auto = layer split + DFlash2.
|
||||
AUTO_SPLIT_BEATS_DFLASH2 = (86, 120)
|
||||
TESTED_CONTEXT = 131072
|
||||
MAX_CARDS = 4
|
||||
LAYERS, EXPERTS = 48, 512 # Qwen3.8-Flash-Next
|
||||
@@ -223,6 +226,11 @@ def split_config(base: dict, decision: Decision, *, exe: str, native: str, mtp_d
|
||||
# every expert in the cards' memory, no expert cache, no arena: the engine reads these two at start
|
||||
cfg["env"] = {**(base.get("env") or {}), "FNX_ARENA": "0", "FNX_EXPERT_RAM_GIB": "off"}
|
||||
cfg["split_fallback"] = {k: base[k] for k in ("exe", "args", "env", "layer_split", "replicas", "user_args", "drafting", "cwd") if k in base}
|
||||
# The split engine runs the model's own draft layer (the MTP) and nothing else. Saying so in the config ("drafting": "mtp") keeps serve/server.py's
|
||||
# drafter step, which resolves Auto to DFlash2 where the engine has it, from adding DFlash2 flags to the split engine's arguments (it would refuse them
|
||||
# and every start would fall back to the layer split). The fallback keeps the person's own choice (or none: Auto), so the layer split that starts
|
||||
# instead gets DFlash2 as usual.
|
||||
cfg["drafting"] = "mtp"
|
||||
cfg["split_engine_note"] = decision.line
|
||||
return cfg
|
||||
|
||||
@@ -316,8 +324,14 @@ def _apply(cfg: dict, *, exe_for_cards, system, allow_untested, probe, ram, scan
|
||||
if not native:
|
||||
return Decision(False, ["this model is not a single GGUF file set"])
|
||||
# choices the split engine does not honour: the normal layer split keeps them
|
||||
if str(cfg.get("drafting") or "").lower() == "dflash2":
|
||||
# Drafting: Auto (nothing written, or "auto") on a setup the split accepts runs the split with its MTP: measured faster to decode than the layer
|
||||
# split with DFlash2 on both tested cards (AUTO_SPLIT_BEATS_DFLASH2, numbers in docs/SPLIT_ENGINE.md). The person's own "dflash2" (or "off", which the
|
||||
# split cannot do: it needs the draft layer) keeps the layer split; "mtp" is what the split runs.
|
||||
chosen = str(cfg.get("drafting") or "auto").lower()
|
||||
if chosen == "dflash2":
|
||||
return Decision(False, ["DFlash2 drafting was chosen, which the all-cards split engine does not run"])
|
||||
if chosen == "off":
|
||||
return Decision(False, ["drafting is off, and the all-cards split engine needs the draft layer"])
|
||||
if cfg.get("user_args") or cfg.get("replicas"):
|
||||
return Decision(False, ["this run config has engine arguments of its own or replicas, which the all-cards split engine does not take"])
|
||||
kv = _flag(args, "--kv")
|
||||
@@ -336,6 +350,8 @@ def _apply(cfg: dict, *, exe_for_cards, system, allow_untested, probe, ram, scan
|
||||
allow_untested=allow_untested)
|
||||
if not d.use_split:
|
||||
return d
|
||||
if chosen == "auto" and not all(c.cc in AUTO_SPLIT_BEATS_DFLASH2 for c in cards):
|
||||
return Decision(False, ["on this card the layer split with DFlash2 has not been measured slower than the all-cards split engine"], d.cards)
|
||||
exe = exe_for_cards(d.cards, cards[0].cc)
|
||||
if not exe:
|
||||
return Decision(False, [f"this engine package has no strata-split-{d.cards}"], d.cards)
|
||||
|
||||
@@ -299,6 +299,39 @@ class Apply(unittest.TestCase):
|
||||
cfg["args"].extend(["--kv", "int8"])
|
||||
self.assertTrue(self.run_apply(cfg).use_split)
|
||||
|
||||
def test_drafting_auto_resolves_to_the_split_with_its_mtp_and_the_fallback_keeps_the_layer_split_choice(self):
|
||||
import tempfile
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
for chosen, split in ((None, True), ("auto", True), ("mtp", True), ("dflash2", False), ("off", False)):
|
||||
cfg = self.cfg(tmp)
|
||||
del cfg["split_engine"]
|
||||
if chosen:
|
||||
cfg["drafting"] = chosen
|
||||
before_fallback = {k: v for k, v in cfg.items() if k == "drafting"}
|
||||
d = self.run_apply(cfg)
|
||||
self.assertEqual(d.use_split, split, chosen)
|
||||
if split:
|
||||
# Auto/MTP: the split engine runs its MTP; the config says so, so serve/server.py's drafter step (Auto = DFlash2) leaves its arguments alone
|
||||
self.assertEqual(cfg["drafting"], "mtp", chosen)
|
||||
self.assertEqual(cfg["split_fallback"].get("drafting"), before_fallback.get("drafting"), chosen) # the layer split keeps the person's word, or Auto
|
||||
self.assertNotIn("--draft", cfg["args"])
|
||||
else:
|
||||
self.assertEqual(cfg.get("drafting"), chosen) # untouched: the layer split gets DFlash2 as chosen
|
||||
self.assertNotIn("split_fallback", cfg)
|
||||
|
||||
def test_auto_needs_a_card_where_the_split_decodes_faster_than_the_layer_split_with_dflash2(self):
|
||||
import tempfile
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
cfg = self.cfg(tmp)
|
||||
del cfg["split_engine"]
|
||||
d = self.run_apply(cfg, cards_=cards(2, cc=89), allow_untested=True) # a tester allowed it, but nobody measured it there
|
||||
self.assertFalse(d.use_split)
|
||||
self.assertIn("DFlash2", d.line)
|
||||
cfg["drafting"] = "mtp" # the person asked for the MTP: no comparison to win
|
||||
self.assertTrue(self.run_apply(cfg, cards_=cards(2, cc=89), allow_untested=True).use_split)
|
||||
|
||||
def test_the_engine_is_picked_by_card_count_and_architecture(self):
|
||||
import tempfile
|
||||
|
||||
|
||||
Reference in new issue
Block a user