mirror of
https://github.com/Niko1221/Strata.git
synced 2026-11-07 14:21:06 -10:00
DeepSeek plan: experts on the cards from the engine's own accounting; own files need no free disk
The plan said 2,947 experts for one R9700 and 13,180 for four; the engine held 2,366-2,566 and 12,147-12,192. It took 4.5 GB of dense weights and 1.15 GB of reserve per card and forgot the driver context, the prompt buffers and the dense weights every card repeats. Constants now come from the engine's startup lines (f-042-maya-rc d93547aed): 2.25 bpw one R9700 2,543, four 12,260; boston 2 x A6000 within 1%, A4000 and RTX 5070 within 5%.
strata launch --model-path on files that stay where they are printed 'tight: needs more disk space' with nothing to download: the menu row counted the whole download against Strata's folder. With own files the verdict now comes from the memory words alone.
This commit is contained in:
1 parent
d93547aed5
commit
67eb5f1f57
5 files changed
+147
-10
No files matched your search
@@ -370,6 +370,11 @@ class Catalog:
|
||||
recommended=(rec.get("family") == fam["family"] and rec.get("model") == size["model"])))
|
||||
return out
|
||||
|
||||
def own_files_verdict(self, hw: dict | None, model: str) -> tuple[str, str]:
|
||||
"""DeepSeek-V4.1-Flash with the person's own files used where they are (--model-path): nothing is downloaded or copied, so the disk
|
||||
part of the menu row (which counts the whole download of a set that is not in Strata's folder) does not apply: the memory words alone."""
|
||||
return verdict(self.strata._dsv41_memory_words(hw or self.hardware(), model))
|
||||
|
||||
def resolve(self, model: str, family: str | None = None) -> tuple[str, str]:
|
||||
"""`iq3_xxs` or `IQ3_XXS` (and an optional family) -> (family, model); raises ValueError with the closest match."""
|
||||
models, families, _ctx, _src = self.tables()
|
||||
|
||||
@@ -337,6 +337,12 @@ def apply_defaults(s: S.LaunchSettings, catalog) -> tuple[S.LaunchSettings, list
|
||||
rows = [r for r in catalog.rows(hw) if r.family == s.family and r.model == s.model]
|
||||
if rows:
|
||||
r = rows[0]
|
||||
if s.model_path and r.words == "tight: needs more disk space":
|
||||
# (the person's own files stay where they are: no download, so the free disk space of Strata's folder is not the question)
|
||||
try:
|
||||
r.verdict, r.words = catalog.own_files_verdict(hw, s.model)
|
||||
except Exception: # noqa: BLE001 - keep the menu row's words when the memory words cannot be read
|
||||
pass
|
||||
if r.verdict == "no":
|
||||
notices.append(S.Notice("warn", f"{r.title} does not fit this computer.", "It can still start; expect it to be slow or to run out of memory."))
|
||||
elif r.verdict == "tight":
|
||||
|
||||
+38
-10
@@ -86,7 +86,24 @@ ISTA_SETS["UD-IQ1_M"] = {"title": "Unsloth UD-IQ1_M", "set_bytes": 139_809_056_1
|
||||
"ppl": "?", "ram_ok_gb": 40.0, "disk_total": 258.0,
|
||||
"quality": "Unsloth's next dynamic version, the size of IST-DAS 2.00 bpw; not measured here yet"}
|
||||
ISTA_DENSE_GB = 6.16 # the dense weights in the shards (attention, router, shared experts, the head; the embedding stays on the CPU)
|
||||
ISTA_DENSE_ON_GPU_GB = 4.5 # what a card holds of them with a 12 GB card (measured on the 5070 / 3060: the engine places the rest itself)
|
||||
ISTA_DENSE_ON_GPU_GB = 4.5 # (the old plan's figure, kept for the Unsloth Q8 plan above; the official sets use the measured ones below)
|
||||
# What the engine really puts on a card, from its own startup line ("CUDA0 layers [0, 39] + head: weights 5.86 GB, experts 25.29 GB (2566 of 15360 in
|
||||
# VRAM), caches 0.02 GB, 576 MiB free") on f-042-maya-rc d93547aed, 11-01: the plan said 2,947 experts for one R9700 and the engine held 2,366-2,566;
|
||||
# 13,180 for four and it held 12,147-12,192. The old plan took 4.5 GB of dense weights and 1.15 GB of reserve and forgot the rest of a card's
|
||||
# memory: the driver's context, the prompt path's buffers (they grow with the prompt chunk) and the dense weights each card repeats.
|
||||
# dense weights on the cards = base + CARD_DENSE_REPEAT_GB for each card (2.25 bpw: 5.86 GB on one card, 6.57 on four -> 5.62 + 0.24 per card;
|
||||
# boston's UD-IQ1_M: 7.61 on one and on two cards; p620's UD-IQ1_S: 6.58)
|
||||
# the engine keeps ENGINE_FREE_GB free on each card (--reserve-mib 600 and the small caches)
|
||||
# what is left over per card (the "other" of the accounting, measured against the card's nominal memory): AMD (HIP) 1.60 GB when the cards hold
|
||||
# half of the experts or more (the prompt path works in chunks of 1024: 4 x R9700), 2.58 GB when they hold less (one R9700: the experts the
|
||||
# card cannot hold are read in place and the prompt chunk is 32768 with the borrowed slots; the 8192 chunk of the older engine took 4.5 GB and
|
||||
# the card held 2,366); NVIDIA and others 3.6 GB (2 x A6000 3.7, an A4000 3.5; not tied to the chunk)
|
||||
# and the experts that stay on a card are a little bigger than the set's average (the hot ones: 9.86-10.0 MB against 9.70 for 2.25 bpw): x1.025
|
||||
ISTA_DENSE_GPU_BASE_GB = {"IST-DAS-2.00bpw": 5.62, "IST-DAS-2.25bpw": 5.62, "IST-DAS-1.75bpw": 5.62, "UD-IQ1_S": 6.34, "UD-IQ1_M": 7.37}
|
||||
CARD_DENSE_REPEAT_GB = 0.24
|
||||
ENGINE_FREE_GB = 0.62
|
||||
CARD_OTHER_GB = {"amd": 1.60, "amd_low_share": 2.58, "other": 3.6}
|
||||
RESIDENT_EXPERT_FACTOR = 1.025
|
||||
ISTA_MEASURED = ("an RTX 5070 (12 GB) with 64 GB of RAM and a 3.6 GB/s NVMe drive: answers about 4.7 tokens a second, a 4,000-token prompt "
|
||||
"read at 121 tokens a second; an RTX 3060 (12 GB) with 46 GB of RAM: about 7.2 tokens a second; four Radeon AI PRO R9700 cards: "
|
||||
"about 31 tokens a second")
|
||||
@@ -284,20 +301,31 @@ def _plan_ista(model: str, cards: list[dict] | None, ram_gb: float | None, disk_
|
||||
warns.append("No graphics card Strata can use was found: this model needs one or more cards (8 GB or more; 12 GB is what it was "
|
||||
"measured with).")
|
||||
else:
|
||||
per_dense = ISTA_DENSE_ON_GPU_GB / n
|
||||
cap = [max(0.0, _cap_gb(float(c.get("vram_gb") or 0)) - per_dense - RESERVE_GB) for c in keep]
|
||||
vram_exp = min(v["experts_gb"], sum(cap))
|
||||
amd = all(str(c.get("vendor") or vendor or "").lower() == "amd" for c in keep)
|
||||
per_dense = (ISTA_DENSE_GPU_BASE_GB.get(model, ISTA_DENSE_GPU_BASE_GB["IST-DAS-2.25bpw"]) + CARD_DENSE_REPEAT_GB * n) / n
|
||||
res_gb = expert_gb * RESIDENT_EXPERT_FACTOR # what an expert that stays on a card takes
|
||||
ranges = split_layers([float(c.get("vram_gb") or 0) for c in keep])
|
||||
|
||||
def place(other: float):
|
||||
cap = [max(0.0, _cap_gb(float(c.get("vram_gb") or 0)) - per_dense - ENGINE_FREE_GB - other) for c in keep]
|
||||
vram_exp = min(v["experts_gb"] * RESIDENT_EXPERT_FACTOR, sum(cap))
|
||||
counts = []
|
||||
for room, (a, b) in zip(cap, ranges):
|
||||
gb = min(room, vram_exp * (room / sum(cap))) if sum(cap) else 0.0
|
||||
counts.append(min(int(gb / res_gb), (b - a + 1) * EXPERTS_PER_LAYER))
|
||||
return cap, counts
|
||||
|
||||
cap, counts = place(CARD_OTHER_GB["amd"] if amd else CARD_OTHER_GB["other"])
|
||||
if amd and sum(counts) < EXPERTS / 2: # cards that hold under half of the experts: the engine's larger prompt buffers
|
||||
cap, counts = place(CARD_OTHER_GB["amd_low_share"])
|
||||
in_vram = 0
|
||||
for c, room, (a, b) in zip(keep, cap, ranges):
|
||||
gb = min(room, vram_exp * (room / sum(cap))) if sum(cap) else 0.0
|
||||
ne = min(int(gb / expert_gb), (b - a + 1) * EXPERTS_PER_LAYER)
|
||||
for c, ne, (a, b) in zip(keep, counts, ranges):
|
||||
in_vram += ne
|
||||
out["cards"].append({"index": c.get("index"), "name": c.get("name"), "vram_gb": float(c.get("vram_gb") or 0), "layers": [a, b],
|
||||
"dense_gb": _round1(per_dense), "experts_gb": _round1(ne * expert_gb), "experts": ne})
|
||||
"dense_gb": _round1(per_dense), "experts_gb": _round1(ne * res_gb), "experts": ne})
|
||||
out["experts_in_vram"] = in_vram
|
||||
out["vram_experts_gb"] = _round1(in_vram * expert_gb)
|
||||
out["cpu_experts_gb"] = _round1(v["experts_gb"] - in_vram * expert_gb)
|
||||
out["vram_experts_gb"] = _round1(in_vram * res_gb)
|
||||
out["cpu_experts_gb"] = _round1(max(0.0, v["experts_gb"] - in_vram * res_gb))
|
||||
out["tp"] = tp_for(n, keep)
|
||||
out["args"] = _tp_args(n, out["tp"])
|
||||
names = " + ".join(f"{c['name'] or 'card ' + str(c['index'])} ({c['vram_gb']:.0f} GB)" for c in out["cards"])
|
||||
|
||||
@@ -77,3 +77,46 @@ class UnifiedMemoryPlan(unittest.TestCase):
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
class MeasuredCardAccounting(unittest.TestCase):
|
||||
"""The plan's "experts on the cards" against what the engine's own startup line reported (f-042-maya-rc d93547aed, 2026-11-01): the
|
||||
old plan said 2,947 for one R9700 (the engine held 2,366-2,566) and 13,180 for four (it held 12,147-12,192). Within 3% of the
|
||||
engine's counts on the R9700s; the other cards (boston 2 x A6000, an A4000, an RTX 5070) within 8%."""
|
||||
|
||||
@staticmethod
|
||||
def cards(n, gb, vendor):
|
||||
return [{"index": i, "name": "card", "vram_gb": gb, "vendor": vendor, "arch": "gfx1201"} for i in range(n)]
|
||||
|
||||
def plan(self, n, gb, vendor, model="IST-DAS-2.25bpw", ram=251.0):
|
||||
orig = F._on_windows
|
||||
F._on_windows = lambda: False
|
||||
try:
|
||||
return F.plan(self.cards(n, gb, vendor), ram, 500.0, model=model, vendor=vendor)
|
||||
finally:
|
||||
F._on_windows = orig
|
||||
|
||||
def test_one_r9700_holds_what_the_engine_holds(self):
|
||||
got = self.plan(1, 32.0, "amd", ram=126.0)["experts_in_vram"]
|
||||
self.assertAlmostEqual(got, 2566, delta=2566 * 0.03) # rc, the borrowed prompt slots; the 0.1.55 engine held 2,366
|
||||
self.assertLess(got, 2947)
|
||||
|
||||
def test_four_r9700_hold_what_the_engine_holds(self):
|
||||
p = self.plan(4, 32.0, "amd")
|
||||
self.assertAlmostEqual(p["experts_in_vram"], 12192, delta=12192 * 0.03)
|
||||
self.assertLess(p["experts_in_vram"], 13180)
|
||||
self.assertEqual(len(p["cards"]), 4)
|
||||
|
||||
def test_other_cards(self):
|
||||
self.assertAlmostEqual(self.plan(2, 48.0, "nvidia", ram=125.0)["experts_in_vram"], 8804, delta=8804 * 0.08)
|
||||
self.assertAlmostEqual(self.plan(1, 16.0, "nvidia", ram=64.0)["experts_in_vram"], 750, delta=750 * 0.08)
|
||||
self.assertAlmostEqual(self.plan(1, 12.0, "nvidia", ram=64.0)["experts_in_vram"], 287, delta=287 * 0.08)
|
||||
|
||||
def test_the_cpu_share_follows_the_cards_share(self):
|
||||
p = self.plan(1, 32.0, "amd", ram=126.0)
|
||||
self.assertAlmostEqual(p["vram_experts_gb"] + p["cpu_experts_gb"], 149.06, delta=0.2)
|
||||
|
||||
def test_the_bigger_dense_weights_of_an_unsloth_set_leave_less_room_for_experts(self):
|
||||
a = self.plan(1, 32.0, "amd", model="IST-DAS-2.25bpw", ram=126.0)["vram_experts_gb"]
|
||||
b = self.plan(1, 32.0, "amd", model="UD-IQ1_M", ram=126.0)["vram_experts_gb"] # 7.6 GB of dense weights against 5.9
|
||||
self.assertLess(b, a)
|
||||
@@ -303,3 +303,58 @@ class DiskCountsOnlyTheRest(unittest.TestCase):
|
||||
self.assertNotIn("free disk space", self.s.dsv41_words(hw, "IST-DAS-2.00bpw"))
|
||||
hw = {**self.hw, "disk_free_gb": 44.0}
|
||||
self.assertIn("free disk space", self.s.dsv41_words(hw, "IST-DAS-2.00bpw"))
|
||||
|
||||
|
||||
class OwnFilesNeedNoDisk(unittest.TestCase):
|
||||
"""`strata launch --model-path` on files that stay where they are: nothing to download or copy, so a small free disk in Strata's folder
|
||||
must not print "tight: needs more disk space" (p620, 11-01: 51 GB free, 274 GB set already on disk elsewhere)."""
|
||||
|
||||
def setUp(self):
|
||||
import strata_mcp as M
|
||||
self.s = M.Strata(ROOT)
|
||||
|
||||
def test_the_menu_row_is_tight_for_the_download_but_the_own_files_verdict_is_not(self):
|
||||
from strata_cli.launch.catalog import Catalog
|
||||
hw = {"ram_gb": 125.0, "gpus": [CARD], "engine_vendor": "nvidia", "disk_free_gb": 51.0}
|
||||
row = verdict(self.s.dsv41_words(hw, "IST-DAS-2.25bpw"))
|
||||
self.assertEqual(row, (TIGHT, "tight: needs more disk space")) # a download would not fit
|
||||
cat = Catalog.__new__(Catalog)
|
||||
cat.strata = self.s
|
||||
own = cat.own_files_verdict(hw, "IST-DAS-2.25bpw")
|
||||
self.assertNotIn("disk", own[1])
|
||||
|
||||
def test_apply_defaults_with_own_files_has_no_disk_notice(self):
|
||||
from strata_cli.launch import cmd, settings as S
|
||||
from strata_cli.launch.catalog import ModelRow
|
||||
row = ModelRow(family="deepseek41", family_title="DeepSeek", model="IST-DAS-2.25bpw", id="x", about="", download_gb=274.0, need_vram_gb=0.0,
|
||||
need_ram_gb=0.0, spill_gb=0.0, verdict=TIGHT, words="tight: needs more disk space", experimental=False, images=False, installed=False)
|
||||
|
||||
class Cat:
|
||||
def computer(self):
|
||||
return mock.MagicMock(raw={"ram_gb": 125.0})
|
||||
|
||||
def rows(self, hw):
|
||||
return [row]
|
||||
|
||||
def own_files_verdict(self, hw, model):
|
||||
return ("fits", "fits")
|
||||
|
||||
def max_context(self, *a):
|
||||
return 8192
|
||||
|
||||
def picture_default(self, images):
|
||||
return "off"
|
||||
|
||||
def reader_state(self):
|
||||
return "ok"
|
||||
|
||||
def pack_limit(self, family, model):
|
||||
return None
|
||||
|
||||
strata = mock.MagicMock(multi_family=lambda f: True)
|
||||
|
||||
s, notes = cmd.apply_defaults(S.LaunchSettings(family="deepseek41", model="IST-DAS-2.25bpw", model_path="/data/ds-ista/2.25bpw"), Cat())
|
||||
self.assertFalse([n for n in notes if "tight" in n.text], notes)
|
||||
row.verdict, row.words = TIGHT, "tight: needs more disk space"
|
||||
s, notes = cmd.apply_defaults(S.LaunchSettings(family="deepseek41", model="IST-DAS-2.25bpw"), Cat())
|
||||
self.assertTrue([n for n in notes if "tight" in n.text], notes) # a download: the disk still counts
|
||||
Reference in new issue
Block a user