setup: the low-RAM mode's resident variant on several GPUs (#642)

With an engine that keeps --resident-experts on a layer split, setup no longer
steers a low-RAM PC with two cards to one of them:

- at install, the low-RAM mode on several GPUs takes the resident variant by
  the same rule as on one card (what no card holds fits the RAM with
  LOW_RAM_HEADROOM_GB beside it), without the one-card question;
- at a start, a resident config on several GPUs keeps --resident-experts
  (split_mmap leaves it alone);
- a resident config started on a PC with two cards is offered both, "y" by
  default, as any other config.

It is gated on MIN_ENGINE >= RESIDENT_SPLIT_ENGINE (0.1.40 here, a guess at
the release that would carry the engine change), so until then setup behaves
as before and the golden configs are unchanged. Two tests run the new paths
with MIN_ENGINE patched; tools/test_setup_*.py pass (260 tests).

docs/MULTI_GPU.md

@@ -105,7 +105,9 @@ without it.
 **The resident RAM mode works on a split** (`--resident-experts`, the low-RAM mode). The RAM copy of the experts
 leaves out the ones every card's cache holds, not only the first card's, and when the rest does not fit whole it keeps
 the hottest by the expert profile over all the layers. An adaptive swap copies the evicted expert back into RAM from
-the card that owns its layer. Before, `--resident-experts` with a split ran as `--mmap-experts`. Swift 1.5 IQ3_XXS at
+the card that owns its layer. Before, `--resident-experts` with a split ran as `--mmap-experts`, and setup recommended
+one card in the low-RAM mode; with an engine that has it (`RESIDENT_SPLIT_ENGINE` in setup.py), setup keeps the cards
+together and the experts no card holds in RAM. Swift 1.5 IQ3_XXS at
 160K (q4_0 KV, `--prefill 4096`, `--spec 4` with the stock draft layer), RTX 4060 Ti (layers 0-19) + RTX 5080 (20-47),
 i9-14900KF, 32 GB of RAM, Windows 11, four greedy prompts at a time, decode tok/s:
 

setup.py

@@ -802,9 +802,10 @@ def split_mmap(cfg: dict) -> bool:
     """#364 #384: the low-RAM mode's resident variant (--resident-experts) has no layer split yet.  A config with it
     that runs on several GPUs reads the experts the GPUs do not hold through the OS file cache instead
     (--mmap-experts: the placement those reports measured 1.3-1.6x faster than one GPU), said plainly - the engine
-    used to refuse the pair.  True when the config changed."""
+    used to refuse the pair.  #642: the engines from RESIDENT_SPLIT_ENGINE run it on a split, so it is kept.  True
+    when the config changed."""
     a = cfg.get("args", [])
-    if "--resident-experts" not in a:
+    if "--resident-experts" not in a or resident_split():
         return False
     a[a.index("--resident-experts")] = "--mmap-experts"
     warn("the low-RAM mode's resident variant (--resident-experts) has no layer split yet: on several GPUs the experts "
@@ -897,8 +898,9 @@ def offer_together(cfg_path: Path, cfg: dict, yes: bool) -> dict:
         return cfg
     pair = can[:2]
     cfg["gpus_asked"] = True
-    # #364 #384: the resident low-RAM variant stays on one card unless the user says otherwise (its RAM use is steady)
-    resident = "--resident-experts" in cfg.get("args", [])
+    # #364 #384: the resident low-RAM variant stays on one card unless the user says otherwise (its RAM use is steady);
+    # #642: the engines from RESIDENT_SPLIT_ENGINE keep it on both cards, so they are offered as for any config
+    resident = "--resident-experts" in cfg.get("args", []) and not resident_split()
     say()
     say("  This PC has " + " and ".join(gpu_name(g) for g in pair) + ": Strata can share the model across both.")
     say("  Together they hold about twice the model's experts and read prompts about 20% faster (docs/MULTI_GPU.md).")
@@ -2342,6 +2344,14 @@ DATA_ITEMS = ("models", "packs", "mtp")
 
 LOW_RAM_HEADROOM_GB = 10   # RAM beside the experts: the OS, the engine's other buffers, the server
 RESIDENT_ENGINE = (0, 1, 30)   # the first engine with --resident-experts (the low-RAM mode's resident variant)
+RESIDENT_SPLIT_ENGINE = (0, 1, 40)   # #642: the first engine that runs it on a layer split
+
+
+def resident_split() -> bool:
+    """#642: the engine setup requires runs the low-RAM mode's resident variant on a layer split - each card's cache
+    holds the most-used experts of its own layers, and the RAM copy leaves out what any card holds.  Before it, a
+    resident config on several GPUs read those experts through the OS file cache (#364 #384)."""
+    return MIN_ENGINE >= RESIDENT_SPLIT_ENGINE
 
 
 def low_ram_needed(model, ram) -> bool:
@@ -3927,9 +3937,11 @@ def main() -> int:
              "the low-RAM mode")
     low_ram = budget is None and (a.low_ram in ("on", "resident", "mmap") or
                                   (a.low_ram == "auto" and low_ram_needed(model, ram)))
-    if low_ram and multi and not low_ram_together(a, model, ram, gpu, chosen):
+    # #642: the engines from RESIDENT_SPLIT_ENGINE run the low-RAM mode on the chosen cards as on one (decided below)
+    if low_ram and multi and not resident_split() and not low_ram_together(a, model, ram, gpu, chosen):
         multi, sel, chosen = [], [gpu["index"]], [gpu]
-    # (the low-RAM mode's variant is decided once the context is known, below; on several GPUs it is the mapped one)
+    # (the low-RAM mode's variant is decided once the context is known, below; on several GPUs it is the mapped one
+    # before RESIDENT_SPLIT_ENGINE, #642)
     if not low_ram and budget is None and ram < MODELS[model]["ram_gb"] - 4:
         confirm_paging(model, ram, a.low_ram, a.yes, bool(a.model))
     ok(f"size: {model}")
@@ -4019,16 +4031,24 @@ def main() -> int:
         share = low_ram_gpu_share(model, vram, ctx, kv)
         rest = arena - low_ram_gpu_gb(model, vram, ctx, kv)
         resident = a.low_ram == "resident" or (a.low_ram != "mmap" and low_ram_resident(model, ram, vram, ctx, kv))
-        if multi:      # #364 #384: every chosen card's share (the image encoder on the main one), the mapped variant
+        if multi:      # every chosen card's share (the image encoder on the main one); #364 #384: the mapped variant
             held = min(arena, low_ram_gpu_gb(model, vram, ctx, kv) +
                        sum(low_ram_gpu_gb(model, x["vram_gb"], ctx, kv) for x in chosen[1:]))
-            share, resident = held / arena, False
-            ok(f"low-RAM mode on {len(chosen)} GPUs: {model}'s experts ({arena:.0f} GB) are read from the model folder "
-               f"through the OS file cache instead of a copy in RAM ({ram:.0f} GB); the GPUs hold ~{100 * share:.0f}% "
-               "of them")
-            if share < 0.6:
-                warn("most of the experts are read from the SSD while it answers: expect it to be much slower than "
-                     "with enough RAM (a faster SSD and a smaller size help)")
+            share, rest = held / arena, arena - held
+            # #642: the resident variant on a split, by the same rule as on one card - what no card holds fits the RAM
+            resident = resident_split() and (a.low_ram == "resident" or
+                                             (a.low_ram != "mmap" and ram >= rest + LOW_RAM_HEADROOM_GB))
+            if resident:
+                ok(f"low-RAM mode on {len(chosen)} GPUs: they hold ~{100 * share:.0f}% of {model}'s experts "
+                   f"({arena:.0f} GB) and the other ~{rest:.0f} GB stay in RAM ({ram:.0f} GB), read once from a copy "
+                   "in the model folder")
+            else:
+                ok(f"low-RAM mode on {len(chosen)} GPUs: {model}'s experts ({arena:.0f} GB) are read from the model "
+                   f"folder through the OS file cache instead of a copy in RAM ({ram:.0f} GB); the GPUs hold "
+                   f"~{100 * share:.0f}% of them")
+                if share < 0.6:
+                    warn("most of the experts are read from the SSD while it answers: expect it to be much slower "
+                         "than with enough RAM (a faster SSD and a smaller size help)")
         elif resident:
             ok(f"low-RAM mode: the GPU holds ~{100 * share:.0f}% of {model}'s experts ({arena:.0f} GB) and the other "
                f"~{rest:.0f} GB stay in RAM ({ram:.0f} GB), read once from a copy in the model folder")

tools/test_setup_risk.py

@@ -244,6 +244,24 @@ class LowRamGpus(unittest.TestCase):
         self.assertEqual(cfg["gpu"], 0)
         self.assertFalse(any("Low-RAM mode" in q for q in asked), asked)
 
+    def test_low_ram_on_gpus_with_a_resident_split_engine(self):
+        """#642: an engine that runs the resident variant on a layer split keeps the cards together, resident."""
+        ram, found = PROFILES["32GB-2x24GB"]
+        with mock.patch.object(setup, "MIN_ENGINE", setup.RESIDENT_SPLIT_ENGINE):
+            code, out, cfg, asked = install(ram, found, ["--family", "qwen", "--model", "Q2_0", "--no-start"],
+                                            answers="")
+            self.assertEqual(code, 0, out)
+            self.assertEqual(cfg["gpu"], [0, 1])
+            self.assertIn("--resident-experts", cfg["args"])
+            self.assertFalse(any("Low-RAM mode" in q for q in asked), asked)
+            self.assertIn("stay in RAM", out)
+            code, out, cfg, _ = install(ram, found, ["--family", "qwen", "--model", "Q2_0", "--no-start",
+                                                     "--gpus", "0,1", "--low-ram", "resident"])
+            self.assertEqual(code, 0, out)
+            self.assertEqual(cfg["gpu"], [0, 1])
+            self.assertIn("--resident-experts", cfg["args"])
+            self.assertNotIn("no layer split yet", out)
+
 
 class StartOnSeveralGpus(unittest.TestCase):
     """A resident low-RAM config started on several GPUs reads the experts through the file cache (#364 #384)."""
@@ -280,6 +298,18 @@ class StartOnSeveralGpus(unittest.TestCase):
         code, out, asked, cfg = self.offer(["--mmap-experts"], None)           # other configs: as before
         self.assertEqual(cfg["gpu"], [0, 1])
 
+    def test_resident_split_engine_keeps_a_resident_config(self):
+        """#642: from RESIDENT_SPLIT_ENGINE a resident config is offered both cards as any other, and stays resident."""
+        with mock.patch.object(setup, "MIN_ENGINE", setup.RESIDENT_SPLIT_ENGINE):
+            cfg = {"args": ["--pack", "p", "--resident-experts", "--kv", "int8"]}
+            self.assertFalse(setup.split_mmap(cfg))
+            self.assertEqual(cfg["args"], ["--pack", "p", "--resident-experts", "--kv", "int8"])
+            code, out, asked, cfg = self.offer(["--resident-experts"], None)   # --yes: the recommendation
+            self.assertIsNone(code, out)
+            self.assertEqual(cfg["gpu"], [0, 1])
+            self.assertEqual(cfg["args"], ["--resident-experts", "--remote-expert-opt"])
+            self.assertNotIn("OS file cache", out)
+
     def test_start_with_gpus(self):
         found = PROFILES["32GB-2x24GB"][1]
         with tempfile.TemporaryDirectory() as d: