DFlash2 defaults: conf 0.55 on the large-VRAM Blackwell class for IQ3_S only (the measured set); window 8 stays for every set there

Per the coordinator: SetDefault gets big_blackwell_conf (set on ("cuda", "iq3_s") = 0.55), applied
with the tuned pack on cc 12.0 with >= 48 GB. UD-IQ4_XS-v31 and UD-Q4_K_XL on that class keep conf
0.35 (they were not measured there); the window of 8 is set-independent in the engine and stays.
Snapshot goldens: nv_resident ud_iq4_xs_v31 and ud_q4_k_xl conf back to 0.35 (spec stays 8);
tests updated.
This commit is contained in:
Niko1221 committed 2026-11-08 20:13:27 +01:00
1 parent 6ec4d68ab2
commit a04860fadf
6 files changed
+59 -40

No files matched your search

+16 -8
View File
@@ -276,10 +276,10 @@ MAX_SLOTS = 24576 # every (layer, expert) pair of Qwen3.8-Flash-Next
# DFlash2 --spec 6 against MTP --spec 2, 3 interleaved rounds x 2 reps, six request kinds (phaseA-tests/round0215/RESULTS-MAC-OUT.md): conf 0.35 / 0.5 / 0.6 = +10.0 / +14.3 / +12.4% over all
# six kinds (0.5 faster than MTP on every kind in 3 of 3 rounds; the slot rule would have written 0.8 for the Mac's --expert-cache 64).
BACKEND_CONF = {"sycl": 0.6, "metal": 0.5}
# Large-VRAM Blackwell (cc 12.0 = sm_120, at least 48 GB: l620's RTX PRO 5000 72 GB; NOT the DGX Spark, sm_121, and not a consumer card under 48 GB, which were not measured): conf 0.55 and a window
# of 8 (the engine cap is 8 there too). 11-08 sweep (phaseA-tests/df2r2/RESULTS-CONF.md), IQ3_S, 36 requests, 300 greedy tokens, 5 interleaved pairs against conf 0.35 / window 6: +2.4%, 5 of 5 pairs
# faster, output identical (conf 0.55 alone +1.1%, window 8 alone +0.5%). Only where every expert is resident (the first CONF_STEPS step), the case that was measured.
BIG_BLACKWELL_CONF = 0.55
# Large-VRAM Blackwell (cc 12.0 = sm_120, at least 48 GB: l620's RTX PRO 5000 72 GB; NOT the DGX Spark, sm_121, and not a consumer card under 48 GB, which were not measured): a window of 8 for every
# set (the engine cap is 8 there too, set-independent: window 8 alone at conf 0.35 was +0.5% on IQ3_S) and, for IQ3_S only (SetDefault.big_blackwell_conf in SET_DEFAULTS, owner/coordinator 11-08),
# conf 0.55. 11-08 sweep (phaseA-tests/df2r2/RESULTS-CONF.md), IQ3_S, 36 requests, 300 greedy tokens, 5 interleaved pairs against conf 0.35 / window 6: +2.4%, 5 of 5 pairs faster, output identical
# (conf 0.55 alone +1.1%). Only where every expert is resident (the first CONF_STEPS step), the case that was measured; the other sets on that card were not measured and keep 0.35.
BIG_CARD_BYTES = 48_000_000_000 # "48 GB" the way cards are sold (a 48 GB card reports about 47.5 GiB); the engine's kDflash2BigCardBytes is the same number
@@ -312,6 +312,7 @@ class SetDefault:
tuned_conf: float | None = None
where: str = ""
pack: str = "selq4"
big_blackwell_conf: float | None = None # the ``--draft-conf`` on the large-VRAM Blackwell class (big_blackwell: cc 12.0, >= 48 GB) with the tuned pack, for the sets measured there; None = the backend's own conf
#: Per (engine family, set) defaults, measured. THE ONE TABLE to switch a set either way. The set is the model_name without its "qwen3.8-flash-next-" prefix.
@@ -331,7 +332,7 @@ SET_DEFAULTS = {
("metal", "unsloth-ud-q4_k_xl"): SetDefault("dflash2", True, 0.7),
("metal", "unsloth-ud-iq4_xs-v31"): SetDefault("dflash2", True, 0.7),
("metal", "iq3_s"): SetDefault("mtp"),
("cuda", "iq3_s"): SetDefault("dflash2", True, None, "resident"),
("cuda", "iq3_s"): SetDefault("dflash2", True, None, "resident", big_blackwell_conf=0.55),
("cuda", "unsloth-ud-iq4_xs-v31"): SetDefault("dflash2", True, None, "resident"),
("hip", "iq3_s"): SetDefault("dflash2", True, None, "gfx1151"),
("hip", "unsloth-ud-q4_k_xl"): SetDefault("dflash2", True, None, "gfx1151"),
@@ -756,13 +757,19 @@ def engine_flags(pack: str | Path, vocab: str | Path | None = None, form: str |
def big_blackwell(backend: str | None, archs: Any = None, vram_mib: float | None = None, slots: int | float | None = None) -> bool:
"""Is this start on the large-VRAM Blackwell class (BIG_BLACKWELL_CONF): CUDA, the first card's architecture sm_120 (cc 12.0; the Spark is sm_121), the one card's memory ``vram_mib``
"""Is this start on the large-VRAM Blackwell class (the window of 8, and the sets' big_blackwell_conf): CUDA, the first card's architecture sm_120 (cc 12.0; the Spark is sm_121), the one card's memory ``vram_mib``
at least BIG_CARD_BYTES (None = several cards or unknown: no), and ``slots`` in the all-resident step (None counts as unknown: no)? Mirrors the engine's dflash2_big_blackwell."""
if str(backend or "").strip().lower() != "cuda" or first_arch(archs) != "sm_120" or not vram_mib or slots is None:
return False
return vram_mib * 1048576 >= BIG_CARD_BYTES and slots >= CONF_STEPS[0][0]
def big_blackwell_conf_for(backend: str | None, model_name: str | None, archs: Any = None, vram_mib: float | None = None, slots: int | float | None = None) -> float | None:
"""The ``--draft-conf`` this set has on the large-VRAM Blackwell class (SetDefault.big_blackwell_conf), or None: not that class, or a set nobody measured there (it keeps the slot rule's 0.35)."""
entry = tuned_entry(backend, model_name)
return entry.big_blackwell_conf if entry is not None and big_blackwell(backend, archs, vram_mib, slots) else None
def window_for(backend: str | None, archs: Any = None, vram_mib: float | None = None, slots: int | float | None = None) -> int:
"""The ``--spec`` of a DFlash2 start: WIDE_WINDOW (8) on Strix Halo (HIP gfx1151, engine cap 8) and on the large-VRAM Blackwell class (big_blackwell), SPEC_WINDOW (6) everywhere else, so no other
card's effective window changes (the engine caps 6 on the Spark and the smaller Blackwell cards, and does not cap the rest)."""
@@ -1146,8 +1153,9 @@ def apply(cfg: dict[str, Any], *, klass: str = "other", data_dir: str | Path | N
kept_conf = _flag_value(mine, "--draft-conf")
tuned_conf = tuned_conf_for(backend, cfg.get("model_name")) if Path(pack) == tuned_path(data_dir, tuned_variant(backend, cfg.get("model_name"))) else None # Metal: the conf the tuned pack was measured with; CUDA / HIP: None, the backend's own conf stays
held = slots_of(base, vram_mib)
wide = big_blackwell(backend, archs, vram_mib, held) # large-VRAM Blackwell: its own measured conf (BIG_BLACKWELL_CONF); Strix keeps 0.35 and widens only the window
flags[flags.index("--draft-conf") + 1] = kept_conf if kept_conf is not None else f"{tuned_conf if tuned_conf is not None else BIG_BLACKWELL_CONF if wide else conf_for(backend, held):g}"
tuned_here = Path(pack) == tuned_path(data_dir, tuned_variant(backend, cfg.get("model_name")))
card_conf = big_blackwell_conf_for(backend, cfg.get("model_name"), archs, vram_mib, held) if tuned_here else None # large-VRAM Blackwell, IQ3_S, tuned pack: the measured 0.55; Strix keeps 0.35 and only widens the window
flags[flags.index("--draft-conf") + 1] = kept_conf if kept_conf is not None else f"{tuned_conf if tuned_conf is not None else card_conf if card_conf is not None else conf_for(backend, held):g}"
flags[flags.index("--spec") + 1] = str(window_for(backend, archs, vram_mib, held))
if _flag_value(mine, "--spec") is None:
base = _without_flag(base, "--spec") # the DFlash2 window; the person's own --spec stays
@@ -212,11 +212,11 @@
"--spec",
"8",
"--draft-conf",
"0.55"
"0.35"
],
"env_set": {},
"notes": [
"[strata] drafting: DFlash2 alone (--draft dflash2, --draft-conf 0.55)"
"[strata] drafting: DFlash2 alone (--draft dflash2, --draft-conf 0.35)"
]
}
},
@@ -349,11 +349,11 @@
"--spec",
"8",
"--draft-conf",
"0.55"
"0.35"
],
"env_set": {},
"notes": [
"[strata] drafting: DFlash2 alone (--draft dflash2, --draft-conf 0.55)"
"[strata] drafting: DFlash2 alone (--draft dflash2, --draft-conf 0.35)"
]
}
},
@@ -471,11 +471,11 @@
"--spec",
"8",
"--draft-conf",
"0.55"
"0.35"
],
"env_set": {},
"notes": [
"[strata] drafting: DFlash2 alone (--draft dflash2, --draft-conf 0.55)"
"[strata] drafting: DFlash2 alone (--draft dflash2, --draft-conf 0.35)"
]
}
}
@@ -210,11 +210,11 @@
"--spec",
"8",
"--draft-conf",
"0.55"
"0.35"
],
"env_set": {},
"notes": [
"[strata] drafting: DFlash2 alone (--draft dflash2, --draft-conf 0.55)"
"[strata] drafting: DFlash2 alone (--draft dflash2, --draft-conf 0.35)"
]
}
},
@@ -345,11 +345,11 @@
"--spec",
"8",
"--draft-conf",
"0.55"
"0.35"
],
"env_set": {},
"notes": [
"[strata] drafting: DFlash2 alone (--draft dflash2, --draft-conf 0.55)"
"[strata] drafting: DFlash2 alone (--draft dflash2, --draft-conf 0.35)"
]
}
},
@@ -465,11 +465,11 @@
"--spec",
"8",
"--draft-conf",
"0.55"
"0.35"
],
"env_set": {},
"notes": [
"[strata] drafting: DFlash2 alone (--draft dflash2, --draft-conf 0.55)"
"[strata] drafting: DFlash2 alone (--draft dflash2, --draft-conf 0.35)"
]
}
}
+26 -13
View File
@@ -5,6 +5,7 @@ Every other card keeps conf and --spec 6, so its EFFECTIVE window (--spec lowere
from __future__ import annotations
import hashlib
import tempfile
import unittest
from pathlib import Path
@@ -17,6 +18,7 @@ Q4 = "qwen3.8-flash-next-unsloth-ud-q4_k_xl"
V31 = "qwen3.8-flash-next-unsloth-ud-iq4_xs-v31"
GIB = 1 << 30
TUNED = b"tuned-pack-" * 200_000
def old_cap(uma, major): # include/strata/core/draft_cap.hpp before 11-08
@@ -45,11 +47,17 @@ class CardWindow(unittest.TestCase):
(self.data / "dflash2" / drafting.VOCAB_NAME).write_bytes(b"v" * 100)
self.exe = self.data / "strata"
self.exe.write_bytes(b"--mtp-draft-vocab FILE (or --mtp DIR ... --draft-fallback ... selector codebooks: Q4_0 rows")
patcher = mock.patch.object(drafting, "ENGINE_SERVES_DFLASH2", True)
patcher.start()
self.addCleanup(patcher.stop)
for patcher in (mock.patch.object(drafting, "ENGINE_SERVES_DFLASH2", True),
mock.patch.multiple(drafting, TUNED_BYTES=len(TUNED), TUNED_SHA256=hashlib.sha256(TUNED).hexdigest())):
patcher.start()
self.addCleanup(patcher.stop)
def flags(self, name, backend, archs, vram, extra=(), user_args=None):
def flags(self, name, backend, archs, vram, extra=(), user_args=None, tuned=False):
tuned_file = self.data / "dflash2" / drafting.TUNED_NAME
if tuned:
tuned_file.write_bytes(TUNED)
else:
tuned_file.unlink(missing_ok=True)
cfg = {"model_name": name, "args": ["--native", "m.gguf", *extra], "drafting": "dflash2"}
if user_args:
cfg["user_args"] = list(user_args)
@@ -60,7 +68,7 @@ class CardWindow(unittest.TestCase):
return a[a.index("--spec") + 1], a[a.index("--draft-conf") + 1]
def test_the_constants(self):
self.assertEqual((drafting.SPEC_WINDOW, drafting.WIDE_WINDOW, drafting.DRAFT_CONF, drafting.BIG_BLACKWELL_CONF), (6, 8, 0.35, 0.55))
self.assertEqual((drafting.SPEC_WINDOW, drafting.WIDE_WINDOW, drafting.DRAFT_CONF), (6, 8, 0.35))
self.assertEqual(drafting.BIG_CARD_BYTES, 48_000_000_000)
def test_strix_halo_window_8_conf_unchanged(self):
@@ -68,22 +76,27 @@ class CardWindow(unittest.TestCase):
self.assertEqual(self.flags(name, "hip", ["gfx1151"], None), ("8", "0.35"), name)
def test_blackwell_cc_12_0_with_48_gb_or_more(self):
"""Window 8 for every set; conf 0.55 for IQ3_S only (the set measured there) and only with the tuned pack."""
for mib in (73728, 73415, 49152, 49140, 98304, 45777): # 72 GB, the l620 reading, 48 GiB, an RTX A6000-class 48 GB, 96 GB, the 48e9 byte line
for name in (IQ3S, V31):
self.assertEqual(self.flags(name, "cuda", ["sm_120"], mib), ("8", "0.55"), (name, mib))
self.assertEqual(self.flags(IQ3S, "cuda", ["sm_120"], mib, tuned=True), ("8", "0.55"), mib)
self.assertEqual(self.flags(IQ3S, "cuda", ["sm_120"], mib), ("8", "0.35"), "the stock pack was not measured at 0.55")
for name in (V31, Q4):
self.assertEqual(self.flags(name, "cuda", ["sm_120"], mib, tuned=True), ("8", "0.35"), (name, mib))
self.assertEqual(self.flags(name, "cuda", ["sm_120"], mib), ("8", "0.35"), (name, mib))
def test_blackwell_cc_12_0_under_48_gb_keeps_today(self):
for mib in (32768, 24576, 16384, 12288, 45776, None): # RTX 5090 32 GB .. 5070 12 GB, just under the line, memory unknown (several cards)
self.assertEqual(self.flags(IQ3S, "cuda", ["sm_120"], mib), ("6", f"{drafting.conf_for('cuda', drafting.slots_of([], mib)):g}"), mib) # the slot rule, as before
self.assertEqual(self.flags(IQ3S, "cuda", ["sm_120"], mib, tuned=True), ("6", f"{drafting.conf_for('cuda', drafting.slots_of([], mib)):g}"), mib) # the slot rule, as before
self.assertEqual(self.flags(IQ3S, "cuda", ["sm_120"], 32768)[1], "0.35")
def test_the_spark_is_unchanged(self):
self.assertEqual(self.flags(IQ3S, "cuda", ["sm_121"], None), ("6", "0.35"))
self.assertEqual(self.flags(V31, "cuda", ["sm_121"], None), ("6", "0.35"))
self.assertEqual(self.flags(IQ3S, "cuda", ["sm_121"], 131072), ("6", "0.35"), "even when a reading of its pool is passed")
for tuned in (False, True):
self.assertEqual(self.flags(IQ3S, "cuda", ["sm_121"], None, tuned=tuned), ("6", "0.35"))
self.assertEqual(self.flags(V31, "cuda", ["sm_121"], None, tuned=tuned), ("6", "0.35"))
self.assertEqual(self.flags(IQ3S, "cuda", ["sm_121"], 131072, tuned=tuned), ("6", "0.35"), "even when a reading of its pool is passed")
def test_the_big_card_with_a_small_expert_cache_keeps_the_slot_rule(self):
self.assertEqual(self.flags(IQ3S, "cuda", ["sm_120"], 73728, extra=["--expert-cache", "1000"]), ("6", "0.8"))
self.assertEqual(self.flags(IQ3S, "cuda", ["sm_120"], 73728, extra=["--expert-cache", "1000"], tuned=True), ("6", "0.8"))
def test_other_cards_are_unchanged(self):
for backend, archs, mib in (("cuda", ["sm_86"], 12288), ("cuda", ["sm_89"], 24576), ("cuda", ["sm_90"], 81920), ("cuda", ["sm_86"], 49152), ("hip", ["gfx1201"], 32768),
@@ -93,7 +106,7 @@ class CardWindow(unittest.TestCase):
self.assertEqual(conf, f"{drafting.conf_for(backend, drafting.slots_of([], mib)):g}", (backend, archs)) # the stock pack here: the backend's own rule
def test_the_persons_own_flags_win(self):
self.assertEqual(self.flags(IQ3S, "cuda", ["sm_120"], 73728, user_args=["--draft-conf", "0.4", "--spec", "5"], extra=["--draft-conf", "0.4", "--spec", "5"])[1], "0.4")
self.assertEqual(self.flags(IQ3S, "cuda", ["sm_120"], 73728, user_args=["--draft-conf", "0.4", "--spec", "5"], extra=["--draft-conf", "0.4", "--spec", "5"], tuned=True)[1], "0.4")
cfg = {"model_name": IQ3S, "args": ["--native", "m.gguf", "--spec", "5"], "user_args": ["--spec", "5"], "drafting": "dflash2"}
with mock.patch.object(drafting, "detect_hardware", return_value=("hip", ["gfx1151"])):
drafting.apply(cfg, data_dir=self.data, backend="hip", archs=["gfx1151"], engine=self.exe)
+1 -3
View File
@@ -181,11 +181,9 @@ class ConfidenceFollowsTheSlots(unittest.TestCase):
def test_the_card_decides_when_the_cache_is_auto(self):
args = ["--expert-cache", "auto", "--max-context", "131072", "--kv", "int8"]
for vram, want in ((8192, "0.8"), (12288, "0.8"), (16384, "0.7"), (20480, "0.7"), (24564, "0.5"), (32768, "0.35"), (45000, "0.35"), (49152, "0.55"), (73415, "0.55")):
for vram, want in ((8192, "0.8"), (12288, "0.8"), (16384, "0.7"), (20480, "0.7"), (24564, "0.5"), (32768, "0.35"), (49152, "0.35"), (73415, "0.35")):
self.assertEqual(self.conf(args, vram_mib=vram), want, vram)
self.assertEqual(self.conf(args, vram_mib=32768, backend="hip", archs=("gfx1201",)), "0.35", "the R9700 (12,570 slots measured) keeps 0.35")
for archs in (("sm_121",), ("sm_89",), ("sm_86",)):
self.assertEqual(self.conf(args, vram_mib=73415, archs=archs), "0.35", f"{archs}: only cc 12.0 with 48 GB or more has 0.55")
def test_the_estimate_matches_the_two_measured_starts(self):
n = drafting.expected_slots(32768, ["--max-context", "131072", "--kv", "int8"])
+4 -4
View File
@@ -475,10 +475,10 @@ class CudaAndHip(Fetch):
d2, args2 = self.apply_on(hardware, name)
self.assertEqual(flag(args2, "--draft-model"), str(drafting.pack_path(self.data, self.exe)), where)
self.assertNotEqual(flag(args2, "--draft-model"), flag(args, "--draft-model"), where)
self.assertEqual(flag(args, "--draft-conf"), flag(args2, "--draft-conf"), f"only the pack changes, not the conf {where}")
self.assertEqual(flag(args, "--draft-conf"), "0.55" if hardware is L620 else "0.35", where) # l620 0.55, 11-08
self.assertEqual(flag(args2, "--draft-conf"), "0.35", where)
self.assertEqual(flag(args, "--draft-conf"), "0.55" if (hardware is L620 and name == IQ3S) else "0.35", where) # IQ3_S on l620's class with the tuned pack: 0.55 (11-08); the stock pack keeps 0.35
self.assertEqual(flag(args, "--spec"), flag(args2, "--spec"), where)
strip = lambda a: [x for i, x in enumerate(a) if not (i and a[i - 1] == "--draft-model")] # noqa: E731
strip = lambda a: [x for i, x in enumerate(a) if not (i and a[i - 1] in ("--draft-model", "--draft-conf"))] # noqa: E731 (the conf is compared above)
self.assertEqual(strip(args), strip(args2), where)
def test_the_cells_nobody_measured_are_not_enabled(self):
@@ -537,7 +537,7 @@ class CudaAndHip(Fetch):
self.put_tuned()
after = flag(self.apply_on(hardware, name)[1], "--draft-conf")
self.drop_tuned()
self.assertEqual(before, after, hardware[0])
self.assertEqual(before, "0.35" if hardware is L620 else after, hardware[0]) # (l620's IQ3_S: 0.55 only with the tuned pack)
self.assertEqual(after, want[hardware[0]], hardware[0])
for hardware in (L620, SPARK, STRIX):
self.assertEqual(drafting.conf_for(hardware[1], drafting.slots_of(self.args, hardware[3])), 0.35, hardware[0])