#4846: when RAM is still short at the expert arena, read the experts from the file like --mmap-experts

After the 20 s wait and the give-back of the PLE lock, a shortage no longer registers the arena anyway (a 60 GiB host stalled in reclaim until it was power-cycled). The run uses the --mmap-experts expert source, says so in one line with one next step, and the start never refuses. A hand-set --ple-io ram and --shared-expert-arena are kept as asked (warn only); a hard cgroup limit below the arena falls back instead of refusing; STRATA_ARENA_RAM_FALLBACK=0 is the old behaviour. If the file cannot be opened either, the arena loads the old way. Decision rule and tests in ple_residency.hpp / ple_residency_test.

include/strata/core/expert_source.hpp

@@ -1309,6 +1309,11 @@ public:
     /// The owner of a lock the engine took on its own (the PLE table's residency lock) gives it back here and returns the bytes
     /// it released (0: nothing to give); `open` then reads the available RAM again.  Unset: nothing is given back.
     std::function<uint64_t()> give_back_ram;
+    /// #4846 (owner 11-01): set before `open` when a RAM shortage that survives the wait and the give-back must not be loaded into
+    /// (arena_ram_verdict, ple_residency.hpp).  `open` then registers and allocates nothing, returns false with `ram_short()` true and the
+    /// numbers in `err`; the caller uses the --mmap-experts source for this run.  Unset: the shortage is only warned about, as before.
+    bool fall_back_when_ram_short = false;
+    bool ram_short() const { return ram_short_; }
     /// #633: set by `open` when the RAM available is less than the arena needs (a recommendation; it still loads)
     const std::string& ram_warning() const { return ram_warning_; }
     uint64_t pinned_arena_bytes() const { return pinned_bytes_; }   ///< what the driver registered (the pinned prefix, or the total by need)
@@ -1332,6 +1337,7 @@ private:
     std::atomic<int64_t> reads_{0};   ///< (a counter only; two threads can read blobs at once)
     std::string note_;
     std::string ram_warning_;
+    bool ram_short_ = false;
     double gib_per_s_ = 0.0;
     double load_seconds_ = 0.0;
     double load_read_s_ = 0.0;

include/strata/core/ple_residency.hpp

@@ -45,4 +45,30 @@ inline bool ple_lock_blocks_cache(std::optional<uint64_t> available, uint64_t ta
     return auto_conversation_cache_mib(*available + table_bytes, min_free_mib) > auto_conversation_cache_mib(available, min_free_mib);
 }
 
+/// #4846 (owner 11-01): what the expert arena does when the RAM is short.  The arena is loaded into anonymous memory; with less RAM
+/// available than it needs (after the wait for an engine still exiting, and after the engine gave back the locks it took by itself) the
+/// load pushes the host into reclaim, and a 60 GiB host was seen to stall until it was power-cycled (#4051, #4846).  The engine still
+/// starts, never refuses: it reads the experts from the file like --mmap-experts for this run and says so.
+///   Load      the RAM is there (or the arena is not priced: `need` 0)
+///   FallBack  short, and the fallback is allowed: use the --mmap-experts expert source, register nothing
+///   Warn      short, fallback not allowed (the user fixed the choice by hand): load as asked, with the warning
+///   Refuse    a hard memory limit (a cgroup / container) below the arena, fallback not allowed: certain to be killed while loading
+enum class ArenaRamVerdict { Load, FallBack, Warn, Refuse };
+
+inline ArenaRamVerdict arena_ram_verdict(uint64_t need, uint64_t available, uint64_t hard_limit, bool fallback_allowed) {
+    if (need == 0) return ArenaRamVerdict::Load;
+    if (hard_limit < need) return fallback_allowed ? ArenaRamVerdict::FallBack : ArenaRamVerdict::Refuse;
+    if (available < need) return fallback_allowed ? ArenaRamVerdict::FallBack : ArenaRamVerdict::Warn;
+    return ArenaRamVerdict::Load;
+}
+
+/// May the engine swap the arena for the --mmap-experts source on its own?  Not when the user fixed the RAM by hand: `--ple-io ram`
+/// asked for the PLE table locked (`ple_ram_by_hand`: the user's word, not the residency policy's), or a shared arena file
+/// (--shared-expert-arena) was named.  STRATA_ARENA_RAM_FALLBACK=0 turns it off (testers: the old behaviour, to compare).
+inline bool arena_ram_fallback_allowed(const char* env_value, bool ple_ram_by_hand, bool shared_arena) {
+    if (ple_ram_by_hand || shared_arena) return false;
+    if (env_value != nullptr && env_value[0] != '\0') return std::atoi(env_value) != 0;
+    return true;
+}
+
 } // namespace strata::core

src/core/expert_source.cpp

@@ -3,6 +3,7 @@
 #include "strata/core/expert_nibble.hpp"
 #include "strata/core/foresight_swap.hpp"
 #include "strata/core/on_device.hpp"
+#include "strata/core/ple_residency.hpp"
 #include "strata/platform/aux_cpus.hpp"
 #include "strata/core/remote_experts.hpp"
 #include "strata/core/peer_experts.hpp"
@@ -5860,12 +5861,20 @@ bool ArenaExpertSource::open(const std::string& pack_dir, int64_t n_layers, int6
     // certain to end that way: refused, with the numbers.  Less RAM available than the arena (another program, an
     // engine still exiting) is only a warning - the OS may make room - per the recommend-not-force rule.
     ram_warning_.clear();
+    ram_short_ = false;
     // (a shared arena, --shared-expert-arena, may already be in RAM for another engine: not checked)
     if (detail::HostMemory hm; shared_arena_file.empty() && detail::host_available_memory(hm)) {
         const uint64_t need = want + (uint64_t) blob;
         const double gib = 1073741824.0;
         char buf[512];
-        if (hm.cgroup_limit < need) {
+        if (arena_ram_verdict(need, ~uint64_t{0}, hm.cgroup_limit, fall_back_when_ram_short) != ArenaRamVerdict::Load) {
+            if (fall_back_when_ram_short) {   // #4846: the caller reads the experts from the file instead of refusing
+                std::snprintf(buf, sizeof buf, "it needs %.2f GiB and this process's memory limit is %.2f GiB",
+                              (double) need / gib, (double) hm.cgroup_limit / gib);
+                err = buf;
+                ram_short_ = true;
+                return false;
+            }
             std::snprintf(buf, sizeof buf,
                           "ArenaExpertSource: the expert arena needs %.2f GiB of RAM but this process's memory limit "
                           "(cgroup memory.max / memory.limit_in_bytes) is %.2f GiB: it would be killed while loading. "
@@ -5878,7 +5887,8 @@ bool ArenaExpertSource::open(const std::string& pack_dir, int64_t n_layers, int6
         if (hm.available < need) {
             // #4051: the shortfall is often an engine that was still exiting (the code's own example), a few seconds
             // from giving its RAM back: wait for it, not for ever, and say so.  STRATA_ARENA_RAM_WAIT_S changes the
-            // seconds (0 loads at once, as before this wait).  The load then goes on either way - it only warns.
+            // seconds (0 loads at once, as before this wait).  Still short after it (and after the give-back below): the caller's
+            // --mmap-experts fallback (#4846) when it asked for one, otherwise the load goes on and only warns.
             double wait_s = 20.0;
             if (const char* w = std::getenv("STRATA_ARENA_RAM_WAIT_S"); w != nullptr && w[0] != '\0') {
                 char* end = nullptr;
@@ -5918,6 +5928,17 @@ bool ArenaExpertSource::open(const std::string& pack_dir, int64_t n_layers, int6
                     }
                 }
             }
+            if (arena_ram_verdict(need, hm.available, ~uint64_t{0}, fall_back_when_ram_short) == ArenaRamVerdict::FallBack) {
+                // #4846 (owner 11-01): still short after the wait and the give-back.  Registering anyway can stall the whole host in
+                // reclaim until it is power-cycled: nothing is allocated, the caller takes the --mmap-experts source for this run.
+                std::snprintf(buf, sizeof buf,
+                              "it needs %.2f GiB and %.2f GiB is available (%.2f GiB short%s)",
+                              (double) need / gib, (double) hm.available / gib, (double) (need - hm.available) / gib,
+                              waited > 0.0 ? ", after waiting" : "");
+                err = buf;
+                ram_short_ = true;
+                return false;
+            }
             if (hm.available < need) {
                 std::snprintf(buf, sizeof buf,
                               "the expert arena needs %.2f GiB of RAM but %.2f GiB is available (%.2f GiB short%s): the "

src/core/ple_residency_test.cpp

@@ -55,6 +55,28 @@ int main() {
         check(!ple_lock_blocks_cache(1ull * GiB, 1ull << 20, floor_mib), "a tiny table cannot make a cache");
         check(!ple_lock_blocks_cache(std::numeric_limits<uint64_t>::max(), table, floor_mib), "no overflow");
     }
+    // #4846 (owner 11-01): RAM still short at the arena after the wait and the give-back: the --mmap-experts source, never a refusal,
+    // never registering anyway.  A hand-set --ple-io ram or a shared arena file is respected (warn only).
+    {
+        const uint64_t need = arena + (1ull << 20);   // the arena and one blob
+        const uint64_t none = std::numeric_limits<uint64_t>::max();   // no cgroup limit
+        check(arena_ram_verdict(need, need, none, true) == ArenaRamVerdict::Load, "RAM exactly there: load");
+        check(arena_ram_verdict(need, 240ull * GiB, none, true) == ArenaRamVerdict::Load, "plenty: load");
+        check(arena_ram_verdict(need, need - 1, none, true) == ArenaRamVerdict::FallBack, "one byte short: file source");
+        check(arena_ram_verdict(need, 20ull * GiB, none, true) == ArenaRamVerdict::FallBack, "20 GiB of 51: file source");
+        check(arena_ram_verdict(need, 20ull * GiB, none, false) == ArenaRamVerdict::Warn, "20 GiB of 51, fixed by hand: warn only");
+        check(arena_ram_verdict(need, 240ull * GiB, 30ull * GiB, true) == ArenaRamVerdict::FallBack, "a 30 GiB container limit: file source, not a refusal");
+        check(arena_ram_verdict(need, 240ull * GiB, 30ull * GiB, false) == ArenaRamVerdict::Refuse, "a 30 GiB container limit, fixed by hand: as before");
+        check(arena_ram_verdict(need, 240ull * GiB, need, true) == ArenaRamVerdict::Load, "a limit of exactly the arena: load");
+        check(arena_ram_verdict(0, 0, 0, true) == ArenaRamVerdict::Load, "an unpriced arena loads");
+        check(arena_ram_fallback_allowed(nullptr, false, false), "fallback on by default");
+        check(!arena_ram_fallback_allowed(nullptr, true, false), "a hand-set --ple-io ram is kept");
+        check(!arena_ram_fallback_allowed(nullptr, false, true), "a shared arena file is kept");
+        check(!arena_ram_fallback_allowed("0", false, false), "STRATA_ARENA_RAM_FALLBACK=0 is the old behaviour");
+        check(arena_ram_fallback_allowed("1", false, false), "STRATA_ARENA_RAM_FALLBACK=1");
+        check(arena_ram_fallback_allowed("", false, false), "an empty value is unset");
+        check(!arena_ram_fallback_allowed("1", true, false), "the user's --ple-io ram wins over the env value");
+    }
     std::printf("ple_residency_test: %d checks passed\n", checks);
     return 0;
 }

src/program/generate.cpp

@@ -6807,6 +6807,16 @@ int main(int argc, char** argv) {
     }
     strata::core::ArenaExpertSource arena_src;
     strata::core::ExpertSource* srcp = nullptr;
+    // #4846 (owner 11-01): the arena first, and when the RAM is still short after the wait and the give-back, the --mmap-experts source
+    // for this run (o.mmap_experts is set here, before anything below sizes the experts' tier from it; nothing was sized for the arena
+    // by now: the PLE table's lock, priced against it, was just given back).  A hand-set `--ple-io ram` and a shared arena file are kept
+    // as asked (warn only), STRATA_ARENA_RAM_FALLBACK=0 is the old behaviour.  Two passes at most; the second only after the first chose
+    // the file, and if the file cannot be opened the arena is tried again the old way (a start never ends in a refusal for this).
+    const bool ram_fallback_ok = strata::core::arena_ram_fallback_allowed(
+        std::getenv("STRATA_ARENA_RAM_FALLBACK"), o.ple_io_explicit && o.ple_io == "ram", !o.shared_expert_arena.empty());
+    bool ram_fallback_taken = false, ram_fallback_failed = false;
+    for (bool again = true; again;) {
+    again = false;
     if (o.mmap_experts) {
         // FileExpertSource maps the pack's experts.bin: a canonical pack has it; a native (IQ) pack has it when
         // built with `tools/iq_pack.py --experts-bin` (the per-layer blob sizes of its layout, PR #121).  The low-RAM
@@ -6818,10 +6828,20 @@ int main(int argc, char** argv) {
         if (const char* v = std::getenv("STRATA_FETCH_THREADS"); v != nullptr && std::atoi(v) > 0)
             src.set_fetch_threads(std::atoi(v));
         if (!src.open(o.pack, g.n_layers, g.n_expert, err)) {
+            if (ram_fallback_taken) {   // #4846: the file cannot be opened either; the arena, loaded the old way, is what is left
+                std::fprintf(stderr, "strata generate: the experts cannot be read from the file either (%s): loading the expert arena "
+                                     "as before, short of RAM\n", err.c_str());
+                o.mmap_experts = false;
+                ram_fallback_taken = false;
+                ram_fallback_failed = true;
+                again = true;
+                continue;
+            }
             std::fprintf(stderr, "strata generate: %s\n", err.c_str());
             return 1;
         }
-        std::fprintf(stderr, "strata generate: experts via mmap (--mmap-experts; %s)\n",
+        std::fprintf(stderr, "strata generate: experts via mmap (%s; %s)\n",
+                     ram_fallback_taken ? "not enough RAM for the expert arena: the same source as --mmap-experts" : "--mmap-experts",
                      src.gguf_mode() ? "the GGUF shards in place, no experts.bin" : "the A/B arm of R2.1");
         // #286: with a RAM budget the hottest experts live in it, and the rest are read from the drive unbuffered
         // when the file cache could not keep them beside the budget anyway (a 32 GB PC) - the mapped reads' page
@@ -7012,11 +7032,22 @@ int main(int argc, char** argv) {
                 return held;
             };
         strata::core::pin_reserve_take();
+        arena_src.fall_back_when_ram_short = ram_fallback_ok && !ram_fallback_failed;
         const bool arena_ok = arena_src.open(o.pack, g.n_layers, g.n_expert, /*threads=*/6, err, pin_limit,
                                              o.shared_expert_arena);
         strata::core::pin_note_arena(arena_src.pinned_arena_bytes());
         strata::core::pin_sim_report("after the expert arena");
         strata::core::pin_reserve_release_soft();
+        if (!arena_ok && arena_src.ram_short()) {
+            // one plain line, one next step; the numbers are in err
+            std::fprintf(stderr, "strata generate: not enough RAM for the expert arena (%s): this run reads the experts from the "
+                                 "file, like --mmap-experts (slower; the answers are the same). To get the arena back, free that RAM "
+                                 "and start again\n", err.c_str());
+            o.mmap_experts = true;
+            ram_fallback_taken = true;
+            again = true;
+            continue;
+        }
         if (!arena_ok) {
             std::fprintf(stderr, "strata generate: %s\n", err.c_str());
             return 1;
@@ -7061,6 +7092,7 @@ int main(int argc, char** argv) {
 #endif
         srcp = &arena_src;
     }
+    }   // the arena / file pass loop (#4846)
 #if defined(__linux__)
     // 0.1.43.1: on by default (STRATA_ADAPT_CALLBACK_WARMUP=0 turns it off).  Linux threads inherit the creator's CPU mask, and
     // the host pins itself to one CPU later; a callback thread the driver creates lazily after that would run on the