Setup streams the KV cache from 64K up on a PC with room for the experts plus 10 GB, not only for the model's all-in-RAM need (KV-128)

The rule asked ram_gb + KV + 1 (64.8 GB for IQ3_S, a 64 GB PC reads 63.1), which left a small card with the KV in VRAM and 720 expert slots at 128K. It is now the smaller of that and arena_gb + LOW_RAM_HEADROOM_GB + KV (62.1 GB for IQ3_S; the KV's RAM comes out of the engine's RAM copy or file cache, not on top). It never asks more than before; --kv-streaming on|off and WSL are unchanged; the size stays 32768.

Measured (phaseA-tests/gh-sweep-1031/KV-128.md): IQ3_S, 12 GB mimic on l620 under a 45 GiB cap, 756 -> 1,267 slots, 32K prompt 21-34 s -> 12-17 s, 118K prompt 76-122 s -> 45-63 s, decode flat to +10%, engine RAM unchanged; on the owner's RTX 5070 (single runs) 924 -> 1,585 slots, 32K prompt 130 -> 79 s, RAM copy 30.8 -> 28.9 GiB, no paging. Decode there is within noise.

Tests: need numbers, never-more-than-before over every model/context/KV, the rule at the edges and for WSL, setup writing --kv-resident on a 63.1 GB PC with a 12 GB card and not at 61.9 GB or at 32K. The golden baseline gets --kv-resident 32768 for the 64 GB / 32 GB card IQ3_S and recommended entries. Setup and tuning tests on boston: same 24 failures as f-0291-int3 (not touched), 7 more pass.

setup.py

@@ -6156,14 +6156,30 @@ def kv_streaming_ram_gb(ctx, kv) -> float:
     return ctx * (13 * KV_CELL_BYTES.get(kv, 1056)) / 1e9
 
 
+def kv_streaming_need_gb(model, ctx, kv) -> float:
+    """The RAM a PC needs before setup streams this model's KV cache on its own (`--kv-streaming on` skips the test):
+    the smaller of two numbers.  (1) The model's all-in-RAM need (`ram_gb`) + the KV cache + 1 GB: the experts all fit
+    and the cache comes on top.  (2) The experts (`arena_gb`) + LOW_RAM_HEADROOM_GB beside them (the line
+    `low_ram_needed` draws) + the KV cache: a small card holds so few experts that the engine's RAM copy is the
+    rest, and the KV's RAM then comes out of that copy (or out of the OS file cache when the arena does not fit),
+    not on top of it.  KV-128 (phaseA-tests/gh-sweep-1031/KV-128.md), IQ3_S on a 12 GB card at 128K, 45 GiB of RAM
+    for the engine (Linux mimic): streaming gave 756 -> 1,267 expert slots and a 32K prompt in 12-17 s instead of
+    21-34 s, with the RAM the engine used unchanged (RAM budget 31 -> 29 GiB); on the owner's RTX 5070 (R5070's
+    run) 924 -> 1,585 slots, a 32K prompt 130 -> 79 s, the RAM copy 30.8 -> 28.9 GiB, no paging.  Rule (1) alone
+    asked 64.8 GB of a PC that reads 63.1 and left the KV in VRAM."""
+    kv_gb = kv_streaming_ram_gb(ctx, kv)
+    m = MODELS[model]
+    return min(m["ram_gb"] + kv_gb + 1, m["arena_gb"] + LOW_RAM_HEADROOM_GB + kv_gb)
+
+
 def kv_streaming_wanted(model, ctx, kv, ram, choice="auto") -> bool:
     """Will setup stream this model's KV cache on this PC: from 64K up, when the RAM holds the cache beside the
-    model's experts (+1 GB); `--kv-streaming on|off` overrides the RAM test (the owner's rule), and WSL never
-    streams.  Step 7 writes `--kv-resident` on exactly this answer, and the launcher's preset diff
-    asks it too: a config's `--kv-resident` and a preset's "auto" are the same setting when this says yes."""
+    model's experts (`kv_streaming_need_gb`); `--kv-streaming on|off` overrides the RAM test (the owner's rule), and
+    WSL never streams.  Step 7 writes `--kv-resident` on exactly this answer, and the launcher's preset diff asks it
+    too: a config's `--kv-resident` and a preset's "auto" are the same setting when this says yes."""
     if is_wsl() or choice == "off":
         return False
-    return ctx >= 65536 and (ram >= MODELS[model]["ram_gb"] + kv_streaming_ram_gb(ctx, kv) + 1 or choice == "on")
+    return ctx >= 65536 and (ram >= kv_streaming_need_gb(model, ctx, kv) or choice == "on")
 
 
 def low_ram_one_gpu_why(model, ram, choice, sel=None) -> list[str]:
@@ -10320,7 +10336,8 @@ def main() -> int:
     # when it fits.
     kv_ram_gb = kv_streaming_ram_gb(ctx, kv)      # the branches below are kv_streaming_wanted, with its messages
     # --kv-streaming on|off overrides the RAM test (the owner's rule); WSL stays off - it cannot stream.
-    stream_fits = ram >= MODELS[model]["ram_gb"] + kv_ram_gb + 1
+    stream_need = kv_streaming_need_gb(model, ctx, kv)
+    stream_fits = ram >= stream_need
     if is_wsl() and ctx >= 65536:
         ok("WSL: KV streaming off (the driver pins only about 1 GB of RAM); the KV cache stays in VRAM")
         if a.kv_streaming == "on":
@@ -10332,8 +10349,8 @@ def main() -> int:
         args += ["--kv-resident", "32768"]
         ok(f"KV streaming on: the context's KV cache lives in RAM ({kv_ram_gb:.1f} GB), more experts fit in VRAM")
         if not stream_fits:
-            warn(f"KV streaming needs ~{kv_ram_gb:.1f} GB of RAM beside the ~{MODELS[model]['ram_gb']} GB {model} "
-                 f"uses; this PC has {ram:.0f}. Kept as you chose (--kv-streaming on): it may page or run out of RAM "
+            warn(f"KV streaming needs ~{stream_need:.0f} GB of RAM for {model} (its experts, the room beside them and "
+                 f"the {kv_ram_gb:.1f} GB cache); this PC has {ram:.0f}. Kept as you chose (--kv-streaming on): it may page or run out of RAM "
                  "under load")
         if q4_split:                                   # #498: no budget to take it out of
             pass
@@ -10344,8 +10361,8 @@ def main() -> int:
             warn(f"the KV cache's {kv_ram_gb:.1f} GB of RAM come on top of your {budget:g} GiB RAM budget (setup "
                  f"would take them out of it: {resident_budget_gib(model, ram, kv_ram_gb)} GiB); kept as you chose")
     elif ctx >= 65536:   # #620: say why, so a regenerated config that lost --kv-resident is not a surprise
-        ok(f"KV streaming off: it needs ~{kv_ram_gb:.1f} GB of RAM beside the ~{MODELS[model]['ram_gb']} GB {model} "
-           f"uses, and this PC has {ram:.0f}; the KV cache stays in VRAM (fewer cached experts). --kv-streaming on "
+        ok(f"KV streaming off: it needs ~{stream_need:.0f} GB of RAM for {model} (its experts, the room beside them "
+           f"and the {kv_ram_gb:.1f} GB cache), and this PC has {ram:.0f}; the KV cache stays in VRAM (fewer cached experts). --kv-streaming on "
            "turns it on anyway")
     elif a.kv_streaming == "on":
         warn("--kv-streaming on: a context under 64K is not streamed (the attention's window holds all of it): off")

tools/test_setup_golden.json

@@ -875,6 +875,8 @@
     "131072",
     "--kv",
     "int8",
+    "--kv-resident",
+    "32768",
     "--vision",
     "--vram-reserve-mib",
     "700"
@@ -1029,6 +1031,8 @@
     "131072",
     "--kv",
     "int8",
+    "--kv-resident",
+    "32768",
     "--vision",
     "--vram-reserve-mib",
     "700"

tools/test_setup_risk.py

@@ -455,10 +455,32 @@ class KvStreaming(unittest.TestCase):
         self.assertIn("KV streaming off, as you chose", out)
 
     def test_on_past_the_ram_test(self):
-        code, out, cfg, _ = self.go("IQ3_S", "--kv-streaming", "on")         # auto leaves it off on 64 GB
+        # auto leaves it off on 58 GB: IQ3_S needs 50.3 (experts) + 10 (room) + 1.8 (the cache) = 62.1 GB
+        code, out, cfg, _ = install(58.0, self.FOUND, ["--family", "qwen", "--model", "IQ3_S", "--no-start", "--kv-streaming", "on"])
         self.assertEqual(code, 0, out)
         self.assertEqual(arg(cfg, "--kv-resident"), "32768")
         self.assertIn("Kept as you chose (--kv-streaming on)", out)
+        self.assertIn("KV streaming needs ~62 GB of RAM for IQ3_S", out)
+
+    def test_auto_streams_a_64gb_pc_with_few_experts_in_vram(self):
+        """KV-128: the owner's PC reads 63.1 GB; the all-in-RAM rule asked 64.8 and left the cache in VRAM."""
+        card5070 = [card(0, "NVIDIA GeForce RTX 5070", 11.9, "120")]
+        code, out, cfg, _ = install(63.1, card5070, ["--family", "qwen", "--model", "IQ3_S", "--no-start", "--context", "131072"])
+        self.assertEqual(code, 0, out)
+        self.assertEqual(arg(cfg, "--kv-resident"), "32768")
+        self.assertIn("KV streaming on: the context's KV cache lives in RAM", out)
+        self.assertNotIn("Kept as you chose", out)
+
+    def test_auto_stays_off_just_below_the_rule(self):
+        code, out, cfg, _ = install(61.9, self.FOUND, ["--family", "qwen", "--model", "IQ3_S", "--no-start", "--context", "131072"])
+        self.assertEqual(code, 0, out)
+        self.assertNotIn("--kv-resident", cfg["args"])
+        self.assertIn("KV streaming off: it needs ~62 GB of RAM for IQ3_S", out)
+
+    def test_below_64k_is_never_streamed_on_auto(self):
+        code, out, cfg, _ = install(63.1, self.FOUND, ["--family", "qwen", "--model", "IQ3_S", "--no-start", "--context", "32768"])
+        self.assertEqual(code, 0, out)
+        self.assertNotIn("--kv-resident", cfg["args"])
 
     def test_k8v4_streams(self):
         code, out, cfg, _ = self.go("Q2_0", "--kv-streaming", "on", "--kv", "k8v4")
@@ -478,6 +500,37 @@ class KvStreaming(unittest.TestCase):
         self.assertIn("--resident-budget-gib is for UD-Q4_K_XL", out)
 
 
+class KvStreamingNeed(unittest.TestCase):
+    """KV-128: the RAM setup asks before it streams the KV cache by itself (`kv_streaming_need_gb`)."""
+
+    def test_the_numbers(self):
+        need = lambda m, kv="int8", ctx=131072: round(setup.kv_streaming_need_gb(m, ctx, kv), 1)
+        self.assertEqual(need("IQ3_S"), 62.1)       # 50.3 + 10 + 1.8; the all-in-RAM rule said 62 + 1.7 + 1 = 64.7
+        self.assertEqual(need("IQ3_XXS"), 54.7)     # 42.9 + 10 + 1.8
+        self.assertEqual(need("Q2_0"), 45.8)        # 34 + 10 + 1.8
+        self.assertEqual(need("IQ3_S", "q4_0"), 61.3)
+
+    def test_never_asks_more_than_before(self):
+        for model, ctx, kv in itertools.product(setup.MODELS, (65536, 131072, 262144), ("int8", "q4_0", "k8v4")):
+            before = setup.MODELS[model]["ram_gb"] + setup.kv_streaming_ram_gb(ctx, kv) + 1
+            self.assertLessEqual(setup.kv_streaming_need_gb(model, ctx, kv), before, (model, ctx, kv))
+
+    @mock.patch.object(setup, "is_wsl", lambda: False)
+    def test_the_rule(self):
+        want = lambda ram, ctx=131072, choice="auto": setup.kv_streaming_wanted("IQ3_S", ctx, "int8", ram, choice)
+        self.assertTrue(want(63.1))
+        self.assertTrue(want(62.1))
+        self.assertFalse(want(62.0))
+        self.assertFalse(want(31.9))                 # a PC in the low-RAM mode keeps the KV in VRAM
+        self.assertFalse(want(63.1, 32768))
+        self.assertFalse(want(63.1, choice="off"))
+        self.assertTrue(want(31.9, choice="on"))
+
+    @mock.patch.object(setup, "is_wsl", lambda: True)
+    def test_wsl_never_streams(self):
+        self.assertFalse(setup.kv_streaming_wanted("IQ3_S", 131072, "int8", 128.0))
+
+
 class RulesAgreeWithTheWrittenConfig(unittest.TestCase):
     """The launcher's preset diff asks setup's rules (`kv_streaming_wanted`, `low_ram_wanted`) what "auto" decides on
     this PC, instead of comparing the word "auto" with a config's flags.  That is only honest if those rules are