DeepSeek engine: one default prompt chunk rule, prefill_chunk_default(): 1024 only for a single-card default run where the GPUs hold at least half of the routed experts (4 x R9700; the R9700 worker's gpu_expert_share() from 4cde62a35), 4096 when a GGUF pack's experts are copied into RAM (--mmap-experts off) on a non-HIP card with 10 GiB or more, 8192 for the experts read in place (5070, 3060, 1 x R9700: what a launch on one card already ran), 2048 for a pack that is not a GGUF; real launch logs on the RTX 5070 and the RTX 3060 show no --tp / --threads and 'prompts in chunks of 8192'; one unit test with the 5070, 3060, 5090-class, 1 x and 4 x R9700 cases

src/dsv41/ds4_experts_rule.hpp

@@ -24,6 +24,25 @@ inline int experts_per_word(int requested, uint64_t vram_bytes, int model_k) {
     return n < 1 ? 1 : n;
 }
 
+// Prompt chunk (--prefill-chunk) default, one rule decided once at load from bytes (no timing).  The chunk's buffers come out of the VRAM the
+// expert tier would have, and every chunk streams the experts the cards do not hold once, so a bigger chunk is fewer passes.  Measured (8K prompt):
+//   - the GPUs hold most of the experts (4 x R9700, share 0.86): 1024 370 tok/s, 8192 214  -> small chunk;
+//   - the experts are read in place and the CPU lane streams most of them (RTX 5070 / 3060 12 GB, 1 x R9700): 8192 best (5070 244 against 186
+//     at 1024, 3060 228 against 175, 1 x R9700 431 against 200) -> the big chunk (the GGUF pack's long-standing default);
+//   - the experts copied into RAM (--mmap-experts off): H100 32 GiB 2048 502, 4096 538, 8192 495 -> 4096 (not measured on HIP: no).
+// Reference KL against the original model does not move with the chunk (1024 0.1705, 2048 0.1692, 4096 0.1688, 8192 0.1695).
+// Alloc failure halves the chunk at load (4096 -> 2048 -> 1024 ...), KV and score buffers are allocated whole at the full --max-context.
+constexpr uint64_t kChunk4096MinVram = 10ull << 30;
+// own_default: a single-card run (no --threads, tp 1); gpu_share: the share of the routed experts the cards can hold (1 = all);
+// gguf_copy: a GGUF pack with --mmap-experts off; file_experts: the experts stay in the files; hip: a HIP build; min_vram_bytes: the
+// smallest card; big: the chunk of the streamed case (the engine's kVlRows, 8192)
+inline int prefill_chunk_default(bool own_default, double gpu_share, bool gguf_copy, bool file_experts, bool hip, uint64_t min_vram_bytes,
+                                 int big = 8192) {
+    if (own_default && gpu_share >= 0.5) return 1024;
+    if (gguf_copy && !hip && min_vram_bytes >= kChunk4096MinVram) return 4096;
+    return file_experts ? big : 2048;
+}
+
 // DS4_TOPK=N (a test, not for users): over the flag and the rule; returns the count to use
 inline int experts_test_override(int chosen, int model_k) {
     if (const char* v = std::getenv("DS4_TOPK")) {

src/dsv41/ds4_model.cu

@@ -1966,8 +1966,26 @@ struct Model::Impl {
                         o.vram_experts;
             file_experts = exp_b - std::max(0.0, room - other_b) + (double) kRamMargin > (double) ram_avail();
         }
-        const bool own_default = !(par || o.tp > 1);   // (the single-card default run the chunked path newly takes: chunks of 1024 - 8K prompt 370 tok/s on 4 x R9700, 214 at 8192; the threads / tp runs keep theirs)
-        chunk = prompt_par ? (o.prefill_chunk >= 0 ? o.prefill_chunk : own_default ? 1024 : file_experts ? kVlRows : 2048) : 0;
+        // The default prompt chunk, one rule (ds4_experts_rule.hpp: prefill_chunk_default).  Measured: the chunked pass streams the experts the GPUs
+        // do not hold once per chunk, so where they hold less than half of them a larger chunk is fewer passes (1 x R9700 + 251 GB, 8K prompt:
+        // 1024 200, 2048 277, 4096 353, 8192 431 tok/s; RTX 5070 / 3060 12 GB in place: 8192 244 / 228 against 1024 186 / 175); where they hold
+        // most (4 x R9700: 8192 214, 1024 370) the small one wins; with the experts copied into RAM (--mmap-experts off) 4096 beats 8192 by 7-9%
+        // (H100 32 GiB: 2048 502, 4096 538, 8192 495).  A GGUF pack always has file_experts set (copied only by --mmap-experts off).
+        const bool own_default = !(par || o.tp > 1);   // (a single-card run: no --threads, tp 1 - a launch with several cards passes --tp 1; one card passes neither)
+        const bool gguf_copy = pk.gguf && o.mmap_experts == 0;
+        uint64_t vmin_bytes = ~0ull;
+        for (const Dev& dv : devs) vmin_bytes = std::min<uint64_t>(vmin_bytes, (uint64_t) dv.total);
+        if (devs.empty()) vmin_bytes = 0;
+#if defined(__HIP_PLATFORM_AMD__) || defined(__HIPCC__)
+        const bool hip_build = true;
+#else
+        const bool hip_build = false;
+#endif
+        const int rule_chunk = prefill_chunk_default(own_default, gpu_expert_share(), gguf_copy, file_experts, hip_build, vmin_bytes, kVlRows);
+        chunk = prompt_par ? (o.prefill_chunk >= 0 ? o.prefill_chunk : rule_chunk) : 0;
+        if (prompt_par && o.prefill_chunk < 0 && rule_chunk == 4096)
+            std::fprintf(stderr, "ds4: prompts in chunks of 4096 tokens (the experts are copied into RAM, %.0f GiB of graphics memory; --prefill-chunk N changes it)\n",
+                         (double) vmin_bytes / (double) (1ull << 30));
         const char* dc = std::getenv("DS4_CHUNK");   // (tests: down to 16, a short prompt in several chunks)
         if (dc != nullptr) chunk = prompt_par ? std::atoi(dc) : 0;
         if (const char* v = std::getenv("DS4_CHUNK_MIN")) chunk_min = std::max(1, std::atoi(v));
@@ -2812,7 +2830,9 @@ struct Model::Impl {
     // the staging ring's size (MB) for the experts a chunk streams from the CPU lane: 400 where a good part of them are
     // streamed; 16 where the cards hold most of the experts (they are within 30% of fitting) (a 4-card box) - the ring's VRAM is then worth more
     // as expert slots, and the decode after a prompt keeps the windowed path's speed (R9700 x4: 29.6 -> 31.7 tok/s)
-    size_t ring_mb_auto() const {
+    // The share of the routed experts the GPUs can hold, from what is known at layout (their free memory less the reserve, the
+    // prompt path's buffers and the dense weights; the trunk's experts only when no drafter is loaded): 1 = all of them.
+    double gpu_expert_share(double* exp_out = nullptr, double* room_out = nullptr, double* other_out = nullptr) const {
         double exp_b = 0, other_b = 0, room = 0;
         for (const auto& kt : pk.t) {
             const std::string& n = kt.first;
@@ -2823,7 +2843,15 @@ struct Model::Impl {
         }
         for (const Dev& d : devs)
             room += std::max(0.0, (double) d.free0 - (double) ((size_t) o.reserve_mib << 20) - 1.5 * (1 << 30)) * o.vram_experts;
-        const bool fit = exp_b > 0 && exp_b <= 1.3 * std::max(1.0, room - other_b);   // (R9700 x4: 1.16 -> ring 16: decode -1.8% after a prompt, 64: -4.4%, 400: -4.6%)
+        if (exp_out) *exp_out = exp_b;
+        if (room_out) *room_out = room;
+        if (other_out) *other_out = other_b;
+        return exp_b > 0 ? std::min(1.0, std::max(0.0, room - other_b) / exp_b) : 1.0;
+    }
+    size_t ring_mb_auto() const {
+        double exp_b = 0, other_b = 0, room = 0;
+        const double share = gpu_expert_share(&exp_b, &room, &other_b);
+        const bool fit = exp_b > 0 && share >= 1.0 / 1.3;   // (R9700 x4: 0.86 -> ring 16: decode -1.8% after a prompt, 64: -4.4%, 400: -4.6%)
         if (o.verbose)
             std::fprintf(stderr, "ds4: staging ring: experts %.1f GB, VRAM room %.1f GB, dense %.1f GB -> %d MB\n", exp_b / 1e9,
                          room / 1e9, other_b / 1e9, fit ? 16 : 400);

tests/dsv41/experts_rule_test.cpp

@@ -63,6 +63,36 @@ int main() {
     unsetenv("DS4_TOPK");
 #endif
     CHECK_EQ(experts_test_override(5, 6), 5);
+    // prompt chunk: prefill_chunk_default(own_default, gpu_share, gguf_copy, file_experts, hip, min_vram, big=8192).  What `strata launch` runs on one card
+    // passes neither --tp nor --threads, but the engine's default tp is 2, so own_default is false there; a launch with several cards passes --tp 1.
+    using strata::ds4::prefill_chunk_default;
+    // RTX 5070 / RTX 3060 (12 GB), launch default: experts in place -> 8192 (their logs say "prompts in chunks of 8192")
+    CHECK_EQ(prefill_chunk_default(false, 0.06, false, true, false, 12227ull << 20), 8192);
+    CHECK_EQ(prefill_chunk_default(false, 0.06, false, true, false, 12 * GiB), 8192);
+    // the same cards with --tp 1 on one card: share 0.06 < 0.5 -> 8192 as well (the streamed case)
+    CHECK_EQ(prefill_chunk_default(true, 0.06, false, true, false, 12 * GiB), 8192);
+    // a 5090-class card (32 GB, share 0.19): in place 8192; with the experts copied into RAM 4096
+    CHECK_EQ(prefill_chunk_default(false, 0.19, false, true, false, 32 * GiB), 8192);
+    CHECK_EQ(prefill_chunk_default(false, 0.19, true, true, false, 32 * GiB), 4096);
+    CHECK_EQ(prefill_chunk_default(true, 0.19, true, true, false, 32 * GiB), 4096);
+    CHECK_EQ(prefill_chunk_default(false, 0.10, true, true, false, 16 * GiB), 4096);
+    CHECK_EQ(prefill_chunk_default(false, 0.05, true, true, false, 24 * GiB), 4096);
+    CHECK_EQ(prefill_chunk_default(false, 0.06, true, true, false, 12227ull << 20), 4096);
+    // copied into RAM on a card under 10 GiB (short VRAM): the streamed default, never the 4096 rule
+    CHECK_EQ(prefill_chunk_default(false, 0.0, true, true, false, 8 * GiB), 8192);
+    CHECK_EQ(prefill_chunk_default(false, 0.0, true, true, false, 0), 8192);
+    // 1 x R9700 (HIP, 32 GB, share 0.19), single-card default run (tp 1 passed by a multi-card box): 8192, also when copied (HIP is not measured at 4096)
+    CHECK_EQ(prefill_chunk_default(true, 0.19, false, true, true, 32 * GiB), 8192);
+    CHECK_EQ(prefill_chunk_default(true, 0.19, true, true, true, 32 * GiB), 8192);
+    // 4 x R9700 (share 0.86), single-card default run (--tp 1 --threads off): the small chunk
+    CHECK_EQ(prefill_chunk_default(true, 0.86, false, true, true, 32 * GiB), 1024);
+    CHECK_EQ(prefill_chunk_default(true, 0.50, false, true, true, 32 * GiB), 1024);
+    CHECK_EQ(prefill_chunk_default(true, 0.49, false, true, true, 32 * GiB), 8192);
+    // threads / tp runs (own_default false) keep their defaults whatever the share: 8192 streamed, 2048 for a pack that is not a GGUF
+    CHECK_EQ(prefill_chunk_default(false, 0.86, false, true, true, 32 * GiB), 8192);
+    CHECK_EQ(prefill_chunk_default(false, 0.30, false, false, false, 24 * GiB), 2048);
+    // the rule takes no --max-context: a 128K session on a 12 GB card gets the same chunk, because the chunk's buffers and the KV are
+    // allocated whole at load for that context and a chunk that does not fit there is halved before the first prompt (4096 -> 2048 -> 1024 ...)
     if (fails == 0) std::printf("experts rule: all checks passed\n");
     return fails == 0 ? 0 : 1;
 }