Merge f-0243-dsimport (the app's model-folder preview recognises official DeepSeek sets) into f-0243-rel

strata_control/dsv41_sets.py

@@ -0,0 +1,71 @@
+"""The official DeepSeek-V4.1-Flash GGUF sets (IST-DAS 1.75 / 2.00 / 2.25 bpw), recognised by their pinned file names and sizes.
+
+The Terminal's setup downloads them (setup.py MULTI_FILE_FAMILIES, DSV41_ISTA_FILES) and accepts the files the first release was downloaded as
+(DSV41_LEGACY_FILES: model-2.00bpw-0000N-of-00004.gguf, engram-IQ4_NL.gguf). The app's "I already have the model" preview used to know only the
+Qwen fingerprints, so a folder with such a set ended in "the folder holds more than one model" (the Engram is a second file) or, for one shard,
+"this model type is not supported by the Strata engine". This module is how Control says what a folder of these files is, from the names and
+sizes alone: no tensor is read and no 40 GB file is hashed in a preview (setup checks the SHA-256 once, when the set is adopted).
+
+The tables are setup.py's own (read from it, never copied), so a new size added there is recognised here.
+"""
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+_TABLES: dict | None = None
+FAMILY = "deepseek41"
+
+
+def tables() -> dict | None:
+    """{model: {"label", "shards": {name: bytes}, "legacy": {name: bytes}}} for the official sets and {"engram": {name: bytes}}; None when
+    setup.py cannot be read (the preview then behaves as before)."""
+    global _TABLES
+    if _TABLES is not None:
+        return _TABLES or None
+    try:
+        spec = importlib.util.spec_from_file_location("strata_setup_tables", ROOT / "setup.py")
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)            # type: ignore[union-attr]
+        family = module.MULTI_FILE_FAMILIES[FAMILY]
+        ista, legacy, renamed = module.DSV41_ISTA_FILES, module.DSV41_LEGACY_FILES, module.DSV41_LEGACY_NAMES
+        result: dict = {"sets": {}, "engram": {}}
+        for model, entry in family["models"].items():
+            if entry.get("recipe") != "ista":
+                continue                                # Unsloth's 508 GB set is not an "already have it" case of the app
+            shards = {name: ista[name][0] for name in entry["shards"]}
+            old = {renamed[name]: legacy[renamed[name]][0] for name in entry["shards"] if name in renamed and renamed[name] in legacy}
+            result["sets"][model] = {"label": entry.get("label", model), "shards": shards, "legacy": old}
+        for name in (module.DSV41_ISTA_ENGRAM, renamed.get(module.DSV41_ISTA_ENGRAM)):
+            if name and (name in ista or name in legacy):
+                result["engram"][name] = (ista.get(name) or legacy[name])[0]
+        _TABLES = result
+    except Exception:                                   # noqa: BLE001 - a preview must never fail because of this
+        _TABLES = {}
+    return _TABLES or None
+
+
+def match(files: dict[str, int]) -> list[dict]:
+    """The official sets the files {name: bytes} of one folder belong to, best first: complete sets before partial ones.
+    Each: {"model", "label", "complete", "present": [names], "missing": [names], "wrong_size": [names], "legacy": bool, "engram": name|None}.
+    A set counts when at least one of its shards is there; names and sizes are the pinned ones."""
+    known = tables()
+    if not known:
+        return []
+    engram = next((name for name in known["engram"] if files.get(name) == known["engram"][name]), None)
+    found = []
+    for model, entry in known["sets"].items():
+        for legacy, pinned in ((False, entry["shards"]), (True, entry["legacy"])):
+            if not pinned:
+                continue
+            present = [n for n in pinned if n in files]
+            if not present:
+                continue
+            wrong = [n for n in present if files[n] != pinned[n]]
+            missing = [n for n in pinned if n not in files]
+            found.append({"model": model, "label": entry["label"], "legacy": legacy, "present": sorted(present), "missing": missing,
+                          "wrong_size": wrong, "complete": not missing and not wrong, "engram": engram,
+                          "bytes": sum(pinned[n] for n in present if n not in wrong) + (known["engram"][engram] if engram else 0)})
+    found.sort(key=lambda row: (not row["complete"], len(row["missing"]) + len(row["wrong_size"]), row["model"]))
+    return found

strata_control/models.py

@@ -604,6 +604,98 @@ def own_path_problem(picked: str) -> tuple[str, str] | None:
     return None
 
 
+def _dsv41_files(folder: Path) -> dict[str, int]:
+    """{name: bytes} of the .gguf files in one folder (a link to a file is followed)."""
+    found: dict[str, int] = {}
+    try:
+        with os.scandir(folder) as entries:
+            for entry in entries:
+                if not entry.name.lower().endswith(".gguf"):
+                    continue
+                try:
+                    found[entry.name] = entry.stat().st_size
+                except OSError:
+                    continue
+    except OSError:
+        pass
+    return found
+
+
+def _dsv41_plan(path: Path, details) -> dict | None:
+    """The preview of a folder (or one file) of an official DeepSeek-V4.1-Flash set: the IST-DAS 1.75 / 2.00 / 2.25 shards, under their
+    official names or the first release's, with or without the n-gram table (Engram). Recognised from the pinned names and sizes
+    (strata_control/dsv41_sets.py); None when nothing here is a DeepSeek file, so every other preview is exactly what it was."""
+    from strata_control import dsv41_sets
+    picked = path if stat.S_ISREG(details.st_mode) else None
+    if picked is not None and not picked.name.lower().endswith(".gguf"):
+        return None
+    if dsv41_sets.tables() is None:
+        return None
+    places = [path.parent] if picked is not None else [path]
+    found: list[tuple[Path, dict]] = []
+    for place in places:
+        for row in dsv41_sets.match(_dsv41_files(place)):
+            if picked is None or picked.name in row["present"] or picked.name == row["engram"]:
+                found.append((place, row))
+    if not found and picked is None:
+        try:
+            for place in _folders_with_models(path)[:8]:
+                for row in dsv41_sets.match(_dsv41_files(place)):
+                    found.append((place, row))
+        except OSError:
+            pass
+    if not found:
+        return None
+    complete = [(place, row) for place, row in found if row["complete"]]
+    issues: list[str] = []
+    messages: list[str] = []
+    folder_holds: list[dict] = []
+    missing_shards: list[str] = []
+    candidate = None
+    place, row = (complete[0] if complete else found[0])
+    if len(complete) > 1:
+        issues.append("multiple_model_groups")
+        folder_holds = [{"file": r["present"][0], "files": len(r["present"]), "folder": str(p), "family": dsv41_sets.FAMILY, "model": r["model"]}
+                        for p, r in complete]
+        names = [f"DeepSeek-V4.1-Flash {r['label']} ({r['present'][0]})" for _p, r in complete]
+        messages.append(f"The folder holds {len(complete)} DeepSeek sets: " + "; ".join(names) +
+                        ". Choose the .gguf file of the one you want.")
+    elif not complete:
+        issues.append("incomplete_or_invalid_shard_set")
+        missing_shards = list(row["missing"]) + list(row["wrong_size"])
+    else:
+        candidate = {"family": dsv41_sets.FAMILY, "model": row["model"], "basis": "known_names_and_sizes", "expected_expert_types": [],
+                     "expected_tensor_count": None,
+                     "setup_args_hint": ["--family", dsv41_sets.FAMILY, "--model", row["model"], "--gguf-dir", str(place)]}
+        messages.append(f"This is DeepSeek-V4.1-Flash, {row['label']}: all {len(row['present'])} model files are here"
+                        + (", and the n-gram table." if row["engram"] else "."))
+        if not row["engram"]:
+            messages.append("The n-gram table (111 GB) is not in this folder: setup downloads it, or put engram-IQ4_NL.gguf beside the model files.")
+        if row["legacy"]:
+            messages.append("These are the file names of the first release; Strata uses them as they are.")
+        messages.append("Strata checks the files once when it sets them up (about 5 minutes on a fast drive); nothing is copied or downloaded for them.")
+    sizes = _dsv41_files(place)
+    wanted = list(row["present"]) + ([row["engram"]] if row["engram"] else [])
+    files = [{"name": n, "path": str(place / n), "bytes": sizes.get(n, 0), "gguf_version": None} for n in sorted(wanted)]
+    shards = [place / n for n in sorted(row["present"])]
+    already = None
+    if candidate is not None and shards:
+        try:
+            already = _already_set_up(shards, candidate)
+        except Exception:                               # noqa: BLE001 - only a convenience for the screen
+            already = None
+    return {"source": str(path), "model_dir": str(place), "scope": "local_gguf_candidate", "files": files, "file_count": len(files),
+            "total_bytes": sum(item["bytes"] for item in files), "metadata": None, "candidate_selection": candidate,
+            "plain_issues": list(dict.fromkeys((missing_words(missing_shards) if (i == "incomplete_or_invalid_shard_set" and missing_shards)
+                                                else plain_issue(i)) for i in issues)),
+            "observed_expert_types": [], "observed_tensor_count": 0, "ignored_auxiliary_files": [], "projectors": [], "messages": messages,
+            "already_set_up": already, "reads_pictures": False, "folder_holds": folder_holds, "missing_shards": missing_shards,
+            "file_set_status": "plausible" if not issues else "invalid_or_incomplete", "issues": issues,
+            "adoption_status": "requires_name_size_sha256_and_engine_checks", "execution_compatibility": "unverified", "would_write": False,
+            "notes": ["The names and sizes are the pinned ones of the official sets; setup checks every SHA-256 once before it adopts them",
+                      "No model is adopted or copied by this preview"]}
+
+
 def plan_import(raw_path: str) -> dict:
     """Inspect a local candidate without copying, parsing tensors or adopting it.
 
@@ -618,6 +710,9 @@ def plan_import(raw_path: str) -> dict:
         details = path.stat()
     except OSError:
         raise ProfileError("model import path does not exist (or is a link to something that is gone)", 5) from None
+    official_deepseek = _dsv41_plan(path, details)         # IST-DAS sets under their official or first-release names
+    if official_deepseek is not None:
+        return official_deepseek
     notes: list[str] = []
     folder_holds: list[dict] = []
     issues: list[str] = []

tests/terminal/test_dsv41_import_preview.py

@@ -0,0 +1,94 @@
+"""The app's "I already have the model" preview recognises an official DeepSeek-V4.1-Flash set (IST-DAS 1.75 / 2.00 / 2.25 bpw), under the
+official file names or the first release's, from the pinned names and sizes. Qwen previews are unchanged (tests/terminal/test_models*.py)."""
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+from strata_control import dsv41_sets, models
+
+T = dsv41_sets.tables()
+
+
+def files_of(model, legacy=False, engram="official", drop=(), wrong=()):
+    entry = T["sets"][model]
+    pinned = dict(entry["legacy"] if legacy else entry["shards"])
+    out = {n: (s + 1 if n in wrong else s) for n, s in pinned.items() if n not in drop}
+    if engram:
+        name = next(n for n in T["engram"] if (n.startswith("engram") == (engram == "legacy")))
+        out[name] = T["engram"][name]
+    return out
+
+
+class DeepSeekPreview(unittest.TestCase):
+    def preview(self, listing, picked=None):
+        with tempfile.TemporaryDirectory() as tmp:
+            folder = Path(tmp)
+            target = folder
+            if picked:
+                (folder / picked).write_bytes(b"GGUF")
+                target = folder / picked
+            with mock.patch.object(models, "_dsv41_files", return_value=listing):
+                return models.plan_import(str(target)), folder
+
+    def test_the_tables_come_from_setup_and_cover_the_three_official_sizes(self):
+        self.assertEqual(sorted(T["sets"]), ["IST-DAS-1.75bpw", "IST-DAS-2.00bpw", "IST-DAS-2.25bpw"])
+        self.assertEqual(len(T["engram"]), 2)
+
+    def test_each_official_set_is_recognised_with_its_size_name(self):
+        for model in T["sets"]:
+            for legacy in (False, True):
+                if legacy and not T["sets"][model]["legacy"]:
+                    continue                                    # 1.75 has no first-release names
+                plan, folder = self.preview(files_of(model, legacy=legacy, engram="legacy" if legacy else "official"))
+                self.assertEqual(plan["file_set_status"], "plausible", (model, legacy, plan["issues"]))
+                self.assertEqual((plan["candidate_selection"]["family"], plan["candidate_selection"]["model"]), ("deepseek41", model))
+                self.assertEqual(plan["file_count"], 5)
+                self.assertEqual(plan["candidate_selection"]["setup_args_hint"][-2:], ["--gguf-dir", str(folder)])
+                self.assertEqual(plan["issues"], [])
+                self.assertTrue(plan["messages"])
+                self.assertEqual(bool(plan["already_set_up"] is None or "config" in plan["already_set_up"]), True)
+
+    def test_without_the_engram_it_is_still_the_set_and_says_what_setup_fetches(self):
+        plan, _ = self.preview(files_of("IST-DAS-2.00bpw", engram=None))
+        self.assertEqual(plan["file_set_status"], "plausible")
+        self.assertEqual(plan["file_count"], 4)
+        self.assertTrue(any("n-gram table" in m and "downloads it" in m for m in plan["messages"]))
+
+    def test_a_missing_or_short_shard_names_the_file(self):
+        gone = sorted(T["sets"]["IST-DAS-2.00bpw"]["shards"])[2]
+        plan, _ = self.preview(files_of("IST-DAS-2.00bpw", drop=(gone,)))
+        self.assertEqual(plan["file_set_status"], "invalid_or_incomplete")
+        self.assertIn("incomplete_or_invalid_shard_set", plan["issues"])
+        self.assertEqual(plan["missing_shards"], [gone])
+        self.assertIsNone(plan["candidate_selection"])
+        short = sorted(T["sets"]["IST-DAS-2.00bpw"]["shards"])[1]
+        plan, _ = self.preview(files_of("IST-DAS-2.00bpw", wrong=(short,)))
+        self.assertEqual(plan["missing_shards"], [short])
+
+    def test_two_sets_in_one_folder_ask_for_a_file_and_a_picked_file_decides(self):
+        listing = {**files_of("IST-DAS-2.00bpw"), **files_of("IST-DAS-2.25bpw")}
+        plan, _ = self.preview(listing)
+        self.assertIn("multiple_model_groups", plan["issues"])
+        self.assertEqual(sorted(h["model"] for h in plan["folder_holds"]), ["IST-DAS-2.00bpw", "IST-DAS-2.25bpw"])
+        self.assertIsNone(plan["candidate_selection"])
+        one = sorted(T["sets"]["IST-DAS-2.25bpw"]["shards"])[0]
+        plan, _ = self.preview(listing, picked=one)
+        self.assertEqual(plan["candidate_selection"]["model"], "IST-DAS-2.25bpw")
+        self.assertEqual(plan["file_set_status"], "plausible")
+
+    def test_one_shard_picked_is_the_whole_set_not_an_unsupported_model(self):
+        first = sorted(T["sets"]["IST-DAS-2.00bpw"]["shards"])[0]
+        plan, _ = self.preview(files_of("IST-DAS-2.00bpw"), picked=first)
+        self.assertEqual(plan["candidate_selection"]["model"], "IST-DAS-2.00bpw")
+        self.assertNotIn("unsupported_or_missing_architecture", plan["issues"])
+        self.assertFalse(any("not supported" in m for m in plan["messages"]))
+
+    def test_other_files_are_not_claimed(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            with mock.patch.object(models, "_dsv41_files", return_value={"Qwen3.8-Flash-Next-GSQ-RCO-IQ1_M-00001-of-00002.gguf": 5, "x.gguf": 3}):
+                self.assertIsNone(models._dsv41_plan(Path(tmp), Path(tmp).stat()))
+
+
+if __name__ == "__main__":
+    unittest.main()