mirror of
https://github.com/Niko1221/Strata.git
synced 2026-11-07 22:59:00 -10:00
metal: the streaming arena's size rule - keep max(7 GB, 15% of the RAM), the n-gram table's memory out of it, re-checked just before the arena is committed; buffers past 8 GB when 4 of 8 GB are too few
arena_budget.h: the rule as plain arithmetic, tested by metal/tests/arena_budget_test.cpp (13 checks, run on the M4). The M5 Max froze (2026-10-13 02:08) with an 89 GB arena planned from 'available - 7 GB' beside a 50 GB n-gram table: macOS now keeps 15% of the RAM (20.6 GB there), the table's mapped bytes (--ple-io ram) or 2 GB (read row by row) come out too, and just before the arena is committed the free memory is asked again - less than planned: the arena is laid out again smaller instead of failing. STRATA_METAL_KEEP_FREE_GB / STRATA_METAL_ARENA_GB override. stream.cpp: arena buffers grow past 8 GB when 4 of 8 GB cannot hold the arena (STRATA_METAL_ARENA_CHUNK_GB). The coordinator suggested max(12 GB, 15%); on the 32 GB M4 (Q2_0 streamed, 200 tokens, memory_pressure every 3 s) 12 GB kept cut the arena 9.4 -> 4.4 GB and decode 7.8 -> 4.8 tok/s, while 7 GB kept left macOS 55% free the whole run (12 GB: 70%); same tokens. Hence 7 GB below ~47 GB of RAM, 15% above. Not yet run on the M5 (down).
This commit is contained in:
Niko1221 committed
1 parent
5cf2eb2788
commit
54c0de70ca
5 files changed
+167
-13
No files matched your search
@@ -0,0 +1,56 @@
|
||||
// metal/src/engine/arena_budget.h - how big the SSD-streaming expert arena may be (stream.h), as plain arithmetic so it
|
||||
// can be tested without a Mac (metal/tests/arena_budget_test.cpp).
|
||||
//
|
||||
// The arena is anonymous memory the GPU keeps resident: macOS cannot page it out or drop it like file cache. It must
|
||||
// leave room for macOS and the desktop, and for the other big memory the engine itself uses while it runs:
|
||||
// keep macOS and the desktop: max(7 GB, 15% of the RAM) - STRATA_METAL_KEEP_FREE_GB sets it. The M5 Max
|
||||
// (128 GB) froze with an 89 GB arena and 7 GB kept beside a 50 GB n-gram table; with 15% it keeps 20.6 GB.
|
||||
// On the 32 GB M4, 7 GB kept left macOS 55% of its memory free all run (memory_pressure) and decodes
|
||||
// 7.8 tok/s; 12 GB kept cut the arena from 9.4 to 4.4 GB and decode to 4.8 tok/s (Q2_0, 200 tokens).
|
||||
// ple the n-gram table: mapped whole in RAM (--ple-io ram) it is all of it; read row by row (the default)
|
||||
// its row cache and the OS's read-ahead, 2 GB.
|
||||
// GPU the Metal working set less what is already allocated, less 1 GB.
|
||||
// STRATA_METAL_ARENA_GB sets the arena outright. The arena is placed just before it is filled, and the memory macOS
|
||||
// can give is asked again then: when it has dropped (another program), the arena shrinks (arena_shrink).
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
|
||||
namespace strata::metal_engine {
|
||||
|
||||
struct ArenaMem {
|
||||
double phys = 0; // bytes of RAM
|
||||
double working_set = 0; // the Metal device's recommended working set
|
||||
double allocated = 0; // already allocated by the engine (Metal buffers)
|
||||
double available = 0; // what macOS can give now: free + inactive + speculative + purgeable pages
|
||||
double ple_mapped = 0; // the n-gram table's bytes when it is mapped whole in RAM, else 0
|
||||
double keep_gb = -1; // STRATA_METAL_KEEP_FREE_GB (< 0: unset)
|
||||
double arena_gb = -1; // STRATA_METAL_ARENA_GB (< 0: unset)
|
||||
};
|
||||
|
||||
constexpr double kGB = 1e9;
|
||||
|
||||
inline double arena_keep(const ArenaMem& m) {
|
||||
return m.keep_gb >= 0 ? m.keep_gb * kGB : std::max(7.0 * kGB, 0.15 * m.phys);
|
||||
}
|
||||
|
||||
inline double arena_ple_reserve(const ArenaMem& m) { return m.ple_mapped > 0 ? m.ple_mapped : 2.0 * kGB; }
|
||||
|
||||
/// The arena's bytes (at least 1 GB: a smaller arena still runs, at a low hit rate).
|
||||
inline double arena_budget(const ArenaMem& m) {
|
||||
if (m.arena_gb >= 0) return m.arena_gb * kGB;
|
||||
const double gpu = m.working_set - m.allocated - 1.0 * kGB;
|
||||
const double ram = m.available - arena_keep(m) - arena_ple_reserve(m);
|
||||
return std::max(1.0 * kGB, std::min(gpu, ram));
|
||||
}
|
||||
|
||||
/// Just before the arena is filled: the bytes it may still have when macOS can now give `available_now` (it shrinks,
|
||||
/// never grows; the explicit STRATA_METAL_ARENA_GB is kept as it is).
|
||||
inline double arena_shrink(const ArenaMem& m, double planned, double available_now) {
|
||||
if (m.arena_gb >= 0) return planned;
|
||||
ArenaMem now = m;
|
||||
now.available = available_now;
|
||||
return std::min(planned, arena_budget(now));
|
||||
}
|
||||
|
||||
} // namespace strata::metal_engine
|
||||
@@ -291,8 +291,11 @@ void ExpertStream::promote_slot(int s) {
|
||||
for (auto it = mine.rbegin(); it != mine.rend(); ++it) urgent_q_.push_front(*it);
|
||||
}
|
||||
|
||||
bool ExpertStream::make_arena(mtl::Context& cx, uint64_t budget, const std::string& profile, int min_slots, std::string& err) {
|
||||
bool ExpertStream::make_arena(mtl::Context& cx, uint64_t budget, const std::string& profile, int min_slots, std::string& err,
|
||||
const std::function<uint64_t()>& recheck) {
|
||||
const double t0 = now_ms();
|
||||
shrink_to_ = 0;
|
||||
for (auto& k : cls_) k.n = 0; // a second try (after a shrink) lays the arena out again
|
||||
// the ranking: the profile's pairs, then every pair it misses, layer-interleaved
|
||||
std::vector<std::pair<int, int>> rank;
|
||||
std::vector<uint8_t> seen((size_t) NL * NE, 0);
|
||||
@@ -338,7 +341,16 @@ bool ExpertStream::make_arena(mtl::Context& cx, uint64_t budget, const std::stri
|
||||
base += k.n;
|
||||
}
|
||||
// the slots placed in chunks of at most maxBufferLength (and 8 GB), up to 4 (the kernels' buffers 0, 7, 8, 9)
|
||||
const uint64_t cmax = std::min<uint64_t>(cx.max_buffer_length(), 8ull << 30) / PAGE * PAGE;
|
||||
// (8 GB each unless the arena needs bigger ones to fit 4: the M4 refused one 20.6 GB buffer, the M5 Max takes an
|
||||
// 89 GB arena for UD-Q5_K_XL in 4 of ~23 GB; STRATA_METAL_ARENA_CHUNK_GB sets it)
|
||||
uint64_t total = 0, biggest = 0;
|
||||
for (const auto& k : cls_) {
|
||||
total += (uint64_t) k.n * k.pages * PAGE;
|
||||
biggest = std::max<uint64_t>(biggest, k.pages * PAGE);
|
||||
}
|
||||
uint64_t want = std::max<uint64_t>(8ull << 30, total / 4 + 4 * biggest);
|
||||
if (const char* e = std::getenv("STRATA_METAL_ARENA_CHUNK_GB")) want = (uint64_t) (std::atof(e) * (1ull << 30));
|
||||
const uint64_t cmax = std::min<uint64_t>(cx.max_buffer_length(), want) / PAGE * PAGE;
|
||||
code_.assign((size_t) base, 0);
|
||||
host_.assign((size_t) base, nullptr);
|
||||
std::vector<uint64_t> csize;
|
||||
@@ -356,6 +368,16 @@ bool ExpertStream::make_arena(mtl::Context& cx, uint64_t budget, const std::stri
|
||||
}
|
||||
if (!csize.empty()) csize.back() = cur;
|
||||
if (csize.size() > 4) { err = "metal stream: the arena needs more than 4 buffers (lower STRATA_METAL_ARENA_GB)"; return false; }
|
||||
if (recheck) { // the memory macOS can give, asked again just before the arena is committed
|
||||
uint64_t total = 0;
|
||||
for (uint64_t b : csize) total += b;
|
||||
const uint64_t allowed = recheck();
|
||||
if (allowed + (64ull << 20) < total) {
|
||||
shrink_to_ = allowed;
|
||||
err = "metal stream: less memory than planned is free now: the arena shrinks";
|
||||
return false;
|
||||
}
|
||||
}
|
||||
for (uint64_t b : csize) { // anonymous memory wrapped no-copy: a prompt's buffers can alias a region of it
|
||||
void* m = mmap(nullptr, (size_t) b, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0);
|
||||
if (m == MAP_FAILED) { err = "metal stream: cannot map the arena"; return false; }
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
#include "gguf_lite.h"
|
||||
#include "mtl_rt.h"
|
||||
|
||||
#include <functional>
|
||||
#include <atomic>
|
||||
#include <condition_variable>
|
||||
#include <cstdint>
|
||||
@@ -45,7 +46,11 @@ public:
|
||||
std::string& err);
|
||||
// The arena: `budget` bytes of slots, filled from the profile's ranking (or layer-interleaved if none).
|
||||
// min_slots: each size class at least this many slots (576 with prompt chunks, else 64)
|
||||
bool make_arena(mtl::Context& cx, uint64_t budget, const std::string& profile, int min_slots, std::string& err);
|
||||
// recheck (optional): just before the arena is committed, the bytes it may have now (arena_shrink); when that is
|
||||
// less than the arena laid out, nothing is allocated, shrink_to() says how much, and the caller tries again
|
||||
bool make_arena(mtl::Context& cx, uint64_t budget, const std::string& profile, int min_slots, std::string& err,
|
||||
const std::function<uint64_t()>& recheck = {});
|
||||
uint64_t shrink_to() const { return shrink_to_; }
|
||||
bool ready() const { return !chunks_.empty(); }
|
||||
// the arena's buffers (at most 4, each within the device's maxBufferLength): chunk(i) for i >= count is chunk 0
|
||||
const mtl::Buffer& chunk(int i) const { return chunks_[(size_t) i < chunks_.size() ? (size_t) i : 0]; }
|
||||
@@ -65,6 +70,7 @@ public:
|
||||
void reset_stats();
|
||||
|
||||
private:
|
||||
uint64_t shrink_to_ = 0;
|
||||
struct Lay { uint64_t g, gs, u, us, d, ds; int cls; };
|
||||
struct Slot { int16_t layer = -1; int16_t e = -1; int prev = -1, next = -1; bool pre = false; int left = 0; uint64_t pin = 0; std::atomic<int> state{0}; }; // 0 idle, 1 queued, 2 reading
|
||||
struct Cls { uint64_t pages = 0; int n = 0; int base = 0; int head = -1, tail = -1; }; // slot range [base, base+n)
|
||||
|
||||
+30
-10
@@ -14,6 +14,7 @@
|
||||
#include "gguf_lite.h"
|
||||
#include "mtl_rt.h"
|
||||
#include "ref_quant.h"
|
||||
#include "arena_budget.h"
|
||||
#include "stream.h"
|
||||
#include "../../tests/coop_probe.h"
|
||||
|
||||
@@ -1061,7 +1062,9 @@ void stream_atexit() {
|
||||
} // namespace
|
||||
|
||||
// The arena, once the rest is allocated (the MTP layer loads after init): what the GPU working set and the RAM leave
|
||||
// with STRATA_METAL_KEEP_FREE_GB (default 7) kept for macOS; STRATA_METAL_ARENA_GB sets it.
|
||||
// with STRATA_METAL_KEEP_FREE_GB kept for macOS (default: 7 GB, or 15% of the RAM when that is more - the M5 Max
|
||||
// froze with an 89 GB arena and 7 GB left beside a 50 GB n-gram table read through the file cache);
|
||||
// STRATA_METAL_ARENA_GB sets it.
|
||||
bool Window::stream_arena(std::string& err) {
|
||||
uint64_t phys = 0;
|
||||
size_t len = sizeof phys;
|
||||
@@ -1073,18 +1076,26 @@ bool Window::stream_arena(std::string& err) {
|
||||
"Metal window)\n", g / 1e9);
|
||||
}
|
||||
const double used = (double) cx_->allocated();
|
||||
const char* ek = std::getenv("STRATA_METAL_KEEP_FREE_GB");
|
||||
const double keep = (ek ? std::atof(ek) : 7.0) * gb;
|
||||
// what macOS could give now: free + inactive + speculative + purgeable pages (the file cache is given back)
|
||||
double avail = (double) phys - used;
|
||||
{
|
||||
auto avail_now = [&]() -> double {
|
||||
vm_statistics64_data_t vs;
|
||||
mach_msg_type_number_t cnt = HOST_VM_INFO64_COUNT;
|
||||
if (host_statistics64(mach_host_self(), HOST_VM_INFO64, (host_info64_t) &vs, &cnt) == KERN_SUCCESS)
|
||||
avail = (double) (vs.free_count + vs.inactive_count + vs.speculative_count + vs.purgeable_count) * (double) vm_page_size;
|
||||
}
|
||||
double budget = std::min((double) cx_->recommended_working_set() - used - 1.0 * gb, avail - keep);
|
||||
if (const char* e = std::getenv("STRATA_METAL_ARENA_GB")) budget = std::atof(e) * gb;
|
||||
return (double) (vs.free_count + vs.inactive_count + vs.speculative_count + vs.purgeable_count) * (double) vm_page_size;
|
||||
return (double) phys - (double) cx_->allocated();
|
||||
};
|
||||
ArenaMem am; // arena_budget.h: the rule, testable
|
||||
am.phys = (double) phys;
|
||||
am.working_set = (double) cx_->recommended_working_set();
|
||||
am.allocated = used;
|
||||
am.available = avail_now();
|
||||
if (ss_ && ss_->ple.table && ss_->ple.table->mode() == strata::kernels::PleIo::Mmap)
|
||||
am.ple_mapped = (double) ss_->ple.table->rows() * 192.0; // a row is at most 192 bytes (Q8_0 + its scale)
|
||||
if (const char* e = std::getenv("STRATA_METAL_KEEP_FREE_GB")) am.keep_gb = std::atof(e);
|
||||
if (const char* e = std::getenv("STRATA_METAL_ARENA_GB")) am.arena_gb = std::atof(e);
|
||||
const double keep = arena_keep(am);
|
||||
const double avail = am.available;
|
||||
double budget = arena_budget(am);
|
||||
if (budget < 2 * gb) {
|
||||
std::fprintf(stderr, "strata metal stream: only %.1f GB left for the expert arena (the GPU working set %.1f GB, RAM %.1f GB, "
|
||||
"%.1f GB allocated): expect a low hit rate\n", budget / gb, cx_->recommended_working_set() / gb,
|
||||
@@ -1099,7 +1110,16 @@ bool Window::stream_arena(std::string& err) {
|
||||
cx_->recommended_working_set() / gb);
|
||||
const char* epg = std::getenv("STRATA_METAL_PROMPT_GEMM");
|
||||
const bool pg_on = !(epg && std::string(epg) == "0");
|
||||
if (!st_->make_arena(*cx_, (uint64_t) budget, prof, pg_on ? 576 : 64, err)) return false;
|
||||
for (int tries = 0;; ++tries) {
|
||||
const double planned = budget;
|
||||
if (st_->make_arena(*cx_, (uint64_t) budget, prof, pg_on ? 576 : 64, err,
|
||||
[&]() -> uint64_t { return (uint64_t) arena_shrink(am, planned, avail_now()); }))
|
||||
break;
|
||||
if (st_->shrink_to() == 0 || tries == 4) return false;
|
||||
std::fprintf(stderr, "strata metal stream: less memory is free than when the arena was planned: %.1f -> %.1f GB\n",
|
||||
budget / gb, st_->shrink_to() / gb);
|
||||
budget = (double) st_->shrink_to();
|
||||
}
|
||||
if (pg_on && !prompt_gemm_init(err)) return false;
|
||||
g_stream_window = this;
|
||||
std::atexit(stream_atexit);
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
// metal/tests/arena_budget_test.cpp - the SSD-streaming arena's size rule (src/engine/arena_budget.h), on the sizes of
|
||||
// the Macs it was measured on. No GPU: plain arithmetic. Exit 0 = every check passed.
|
||||
#include "../src/engine/arena_budget.h"
|
||||
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
|
||||
using namespace strata::metal_engine;
|
||||
|
||||
static int failures = 0;
|
||||
static void check(bool ok, const char* what, double got) {
|
||||
std::printf("%s %-70s %.2f GB\n", ok ? "ok " : "FAIL", what, got / kGB);
|
||||
failures += !ok;
|
||||
}
|
||||
|
||||
int main() {
|
||||
// the M5 Max as it froze: 137.4 GB RAM, 115.4 GB working set, 7.0 GB allocated, 97.7 GB available
|
||||
ArenaMem m5{137.4 * kGB, 115.4 * kGB, 7.0 * kGB, 97.7 * kGB};
|
||||
check(std::fabs(arena_keep(m5) - 0.15 * 137.4 * kGB) < 1e6, "M5 Max: macOS keeps 15% of the RAM (20.6 GB)", arena_keep(m5));
|
||||
const double b5 = arena_budget(m5);
|
||||
check(b5 <= 97.7 * kGB - 20.6 * kGB - 2.0 * kGB + 1e6, "M5 Max: the arena leaves >= 20 GB + the n-gram reserve", b5);
|
||||
check(b5 > 70 * kGB, "M5 Max: still a large arena (UD-Q5's 98 GB of experts mostly cached)", b5);
|
||||
// the n-gram table mapped whole (--ple-io ram, 50.7 GB): it comes out of the arena
|
||||
ArenaMem m5r = m5;
|
||||
m5r.ple_mapped = 50.7 * kGB;
|
||||
check(arena_budget(m5r) <= b5 - 48.0 * kGB, "M5 Max, table in RAM: the arena gives up the table's 50.7 GB", arena_budget(m5r));
|
||||
// the M4 Mac mini, 32 GB: 7 GB kept (15% would be 5 GB)
|
||||
ArenaMem m4{34.4 * kGB, 26.8 * kGB, 3.3 * kGB, 17.8 * kGB};
|
||||
check(std::fabs(arena_keep(m4) - 7.0 * kGB) < 1e6, "M4 32 GB: macOS keeps 7 GB", arena_keep(m4));
|
||||
check(std::fabs(arena_budget(m4) - (17.8 - 7.0 - 2.0) * kGB) < 1e6, "M4 32 GB: available - 7 - 2", arena_budget(m4));
|
||||
// the overrides
|
||||
ArenaMem o = m4;
|
||||
o.keep_gb = 12.0;
|
||||
check(std::fabs(arena_budget(o) - (17.8 - 12.0 - 2.0) * kGB) < 1e6, "STRATA_METAL_KEEP_FREE_GB=12", arena_budget(o));
|
||||
o.arena_gb = 10.0;
|
||||
check(std::fabs(arena_budget(o) - 10.0 * kGB) < 1e6, "STRATA_METAL_ARENA_GB=10 wins", arena_budget(o));
|
||||
check(std::fabs(arena_shrink(o, 10.0 * kGB, 1.0 * kGB) - 10.0 * kGB) < 1e6, "an explicit arena is not shrunk", 10.0 * kGB);
|
||||
// never below 1 GB; the GPU working set bounds it too
|
||||
ArenaMem tiny{16 * kGB, 10 * kGB, 9.5 * kGB, 8 * kGB};
|
||||
check(std::fabs(arena_budget(tiny) - 1.0 * kGB) < 1e6, "a 16 GB Mac with nothing left: 1 GB (low hit rate)", arena_budget(tiny));
|
||||
ArenaMem gpu = m5;
|
||||
gpu.available = 200 * kGB;
|
||||
check(std::fabs(arena_budget(gpu) - (115.4 - 7.0 - 1.0) * kGB) < 1e6, "the working set bounds it", arena_budget(gpu));
|
||||
// memory taken between planning and committing: it shrinks, never grows
|
||||
const double s1 = arena_shrink(m5, b5, 60.0 * kGB);
|
||||
check(s1 < b5 && std::fabs(s1 - (60.0 - 20.61 - 2.0) * kGB) < 0.1 * kGB, "less free when committing: it shrinks", s1);
|
||||
check(std::fabs(arena_shrink(m5, b5, 130.0 * kGB) - b5) < 1e6, "more free when committing: it stays as planned", b5);
|
||||
std::printf("%s\n", failures ? "FAILED" : "all passed");
|
||||
return failures ? 1 : 0;
|
||||
}
|
||||
Reference in new issue
Block a user