vram_fit: a streamed slot session at 128K is 748 MiB, as DF2-MEM measured on the CUDA engine (0.73 GiB with 32768 resident; 0.56 GiB at 32K): the per-cell extra a streamed session keeps is 1400 bytes, not the 380 derived from the code

strata_control/vram_fit.py

@@ -64,7 +64,7 @@ BASE_SLOTTED_MIB = {("intel", "flash-next"): 4500}      # (vendor, model family)
 SLOT_SLOTTED_MIB = 2.22
 SLOT_RUN_MIN = 1                                       # the engine's minimum: generate.cpp refuses a start whose auto sizing leaves the cache no slot
 KV_RESIDENT_MIN = 20480                                # qsa_kv_resident_min(): a smaller --kv-resident is raised to this
-STREAM_EXTRA_BYTES = 380                               # per cell of context a streamed session still keeps in VRAM (rope table, indexer rows, page table); derived from qsa_state_bytes
+STREAM_EXTRA_BYTES = 1400                              # per cell of context a streamed session still keeps in VRAM (rope table, indexer rows, page table); measured: the CUDA engine's slot session at 128K with 32768 resident is 0.73 GiB (RESULTS-DF2MEM.md section 17), the 32K session 0.56 GiB
 CTX_STEPS_SLOTTED = CTX_STEPS + (4096, 2048)     # below 8K only for a start with request slots: each slot's session is one more KV cache to pay for
 SLOTS_MAX = 8                                    # the engine's batch window (serve/server.py PARALLEL_MAX)
 

tests/terminal/test_vram_fit.py

@@ -163,7 +163,7 @@ class SlotSessions(unittest.TestCase):
         full = V.session_mib(131072)
         streamed = V.session_mib(131072, "int8", 32768)
         self.assertAlmostEqual(full, 1929, delta=15)
-        self.assertAlmostEqual(streamed, 621, delta=30)                  # the 32K pool (573 MiB with its fixed part) plus what scales with the whole context
+        self.assertAlmostEqual(streamed, 748, delta=15)                  # measured 0.73 GiB on the CUDA engine (DF2-MEM): the 32K pool (573 MiB with its fixed part) plus what scales with the whole context
         self.assertEqual(V.session_mib(8192, "int8", 32768), V.session_mib(8192))     # a resident window larger than the context changes nothing
         self.assertAlmostEqual(V.session_mib(131072, "int8", 4096), V.session_mib(131072, "int8", V.KV_RESIDENT_MIN), delta=0.01)   # the engine raises a small window to 20480
 
@@ -200,7 +200,7 @@ class SlotSessions(unittest.TestCase):
         self.assertIn("one request at a time", f.line)
 
     def test_a_12gb_card_keeps_128k_with_streamed_kv(self):
-        # setup writes --kv-resident 32768 at 128K: each of the 3 sessions keeps about 620 MiB, not 1.9 GB
+        # setup writes --kv-resident 32768 at 128K: each of the 3 sessions keeps about 750 MiB (measured), not 1.9 GB
         for vendor in ("intel", "other"):
             f = V.plan(FN, 12.0, 131072, "int8", 700, "auto", vendor, title=FN, reserve_fixed=True, prefill_fixed=True, slots=2, kv_resident=32768)
             self.assertFalse(f.changed, vendor)