mirror of
https://github.com/Niko1221/Strata.git
synced 2026-11-07 14:21:06 -10:00
dsv41: Maya-port defaults from the owner-PC A/Bs (ON: win_submit, flow pump, Stop, lane-free; OFF with switch: timer, Engram readers, unbuffered Engram, cold prefetch, DSpark cum)
Owner PC (RTX 5070 12 GB, Ryzen 5 7600, official 2.00 pack, shipped profile-2.00.bin, 3 alternating pairs, medians, off = every switch off): 4K prompt, off 92.64 tok/s: win_submit 99.12 (+7.0%, 3/3 pairs), flow pump 97.37 (+5.1%, 3/3), 64 Engram readers 96.64 (+4.3%, 3/3), unbuffered Engram 85.56 (-7.6%, 0/3), all on 86.84 (-6.3%). code decode, off 4.51: timer 4.51 (1/3), 64 readers 4.41 (2/3), unbuffered 4.67 (2/3, one pair 3.83), all 4.57. mmap decode, off 1.95: cold-page prefetch 1.71 (-12%, 0/3). Cold (no profile) agreed on win_submit and flow pump (+4.5%, +2.7%, 2/2) and showed the timer +12%, readers + unbuffered +22% on decode (2/2, off arm noisy). Identity: --vram-experts 0, off vs all IDENTICAL (4K prompt, code decode); text equal to off in every warm arm. Stop during a long prompt (H100, 8K and 16K): STOP answered in 0.12 / 0.16 s, the conversation before the prompt kept, the next request identical. DSpark cum cut: H100 code 20.13 tok/s off vs 19.60 at 0.5, no gain. ON: DS4_WIN_SUBMIT, DS4_FLOW_PUMP_WAIT, Stop, STRATA_DS4_LANE_FREE (a no-op on the physical-core lane). OFF, switch turns it on: DS4_TIMER_MS=1, DS4_ENGRAM_THREADS=64, DS4_EG_DIRECT=1, DS4_IQ_PF_COLD=1, STRATA_DS4_DSPARK_CUM=0.5.
This commit is contained in:
1 parent
0d38517888
commit
6c6267f33c
3 files changed
+13
-13
No files matched your search
@@ -141,11 +141,11 @@ static int g_round_nd = 0;
|
||||
// The drafts' cut on the product of their confidences (DeepInfer's cumulative rule): a draft is verified while the chance
|
||||
// that it and every draft before it are kept stays at least this - a long window whose later drafts are likely lost is
|
||||
// not verified whole. STRATA_DS4_DSPARK_CUM=F, a request's dspark_cum key; 0: off, only the floor on each draft's own
|
||||
// confidence (dspark_conf). Project Maya's default 0.5 (a Radeon 8065S: decode +6.5%, the same tokens)
|
||||
// confidence (dspark_conf). Off by default (not measured faster on code: H100 / owner PC); Project Maya runs 0.5 (a Radeon 8065S: decode +6.5%, the same tokens)
|
||||
float dspark_cum_default() {
|
||||
static const float v = [] {
|
||||
const char* e = std::getenv("STRATA_DS4_DSPARK_CUM");
|
||||
return e != nullptr ? (float) std::atof(e) : 0.5f;
|
||||
return e != nullptr ? (float) std::atof(e) : 0.0f;
|
||||
}();
|
||||
return v;
|
||||
}
|
||||
|
||||
@@ -2420,13 +2420,12 @@ struct Model::Impl {
|
||||
// the row cache a small share of the RAM free (DS4_ENGRAM_CACHE_MB: its size, 0 off; DS4_ENGRAM_THREADS)
|
||||
const size_t row = (size_t) pk.eg_hd + pk.eg_hd / 32;
|
||||
#if defined(_WIN32)
|
||||
// (Windows: each thread's ReadFile is synchronous, so the threads are the queue depth - a 28K-token prompt
|
||||
// in one chunk, 1.36M rows before layer 1: 10 threads 17.1 s, 32 14.1 s, 64 13.4 s on a 20-core PC)
|
||||
int nth = 64;
|
||||
#else
|
||||
// (Windows: each thread's ReadFile is synchronous, so the threads are the queue depth - Project Maya: a 28K-token
|
||||
// prompt in one chunk, 10 threads 17.1 s, 64 13.4 s on a 20-core PC. Here (6 cores, shipped profile) 64 readers
|
||||
// gave +4.5% on a 4K prompt but no decode gain in every pair: DS4_ENGRAM_THREADS=64 turns them on)
|
||||
#endif
|
||||
const unsigned hw = std::max(1u, std::thread::hardware_concurrency());
|
||||
int nth = (int) std::min(16u, std::max(4u, hw / 2));
|
||||
#endif
|
||||
if (const char* e = std::getenv("DS4_ENGRAM_THREADS")) nth = std::max(1, std::atoi(e));
|
||||
const uint64_t avail = ram_avail();
|
||||
uint64_t mb = avail == ~uint64_t{0} ? 64 : std::min<uint64_t>(256, std::max<uint64_t>(16, (avail >> 20) / 64));
|
||||
@@ -2437,10 +2436,10 @@ struct Model::Impl {
|
||||
// Windows: the Engram tables' files that hold nothing else are unmapped and read unbuffered - an SSD's
|
||||
// queue kept by the reader threads instead of the page cache's (whose reads of one file from many threads
|
||||
// went ~255K rows a second here); unbuffered reads of a mapped file take a slow path, so only unmapped
|
||||
// ones (DS4_EG_DIRECT=0: buffered, mapped, as before)
|
||||
// ones (off by default: -8% on a 4K prompt here; DS4_EG_DIRECT=1 turns it on)
|
||||
std::vector<char> eg_direct(pk.files.size(), 0);
|
||||
#if defined(_WIN32)
|
||||
if (pk.gguf && (std::getenv("DS4_EG_DIRECT") == nullptr || std::atoi(std::getenv("DS4_EG_DIRECT")) != 0)) {
|
||||
if (pk.gguf && std::getenv("DS4_EG_DIRECT") != nullptr && std::atoi(std::getenv("DS4_EG_DIRECT")) != 0) {
|
||||
std::vector<char> eg_file(pk.files.size(), 0), other(pk.files.size(), 0);
|
||||
for (const auto& kt : pk.t)
|
||||
if (kt.second.file >= 0 && (size_t) kt.second.file < pk.files.size())
|
||||
@@ -5084,12 +5083,12 @@ struct Model::Impl {
|
||||
const float lim = g.lim;
|
||||
{ // (os::prefetch_cold, as above: the resident ones are left alone - on Windows the hint walks every page of a
|
||||
// range, ~0.13 ms an IQ2_XXS expert, on the lane's critical path before the first entry computes.
|
||||
// DS4_IQ_PF_ALL=1: every range, as before)
|
||||
// off by default (the mmap path, owner PC: not faster in every pair); DS4_IQ_PF_COLD=1 turns it on)
|
||||
std::vector<std::pair<const void*, size_t>> want;
|
||||
for (const Ent& en : ents)
|
||||
if (!store && (l.ram.empty() || l.ram[(size_t) en.e] < 0))
|
||||
for (int r = 0; r < 3; ++r) want.push_back({mx_mat(l, en.e, r), mx_part_bytes(l, r)});
|
||||
static const bool pf_all = std::getenv("DS4_IQ_PF_ALL") != nullptr && std::atoi(std::getenv("DS4_IQ_PF_ALL")) != 0;
|
||||
static const bool pf_all = std::getenv("DS4_IQ_PF_COLD") == nullptr || std::atoi(std::getenv("DS4_IQ_PF_COLD")) == 0;
|
||||
if (pf_all) os::prefetch(want);
|
||||
else os::prefetch_cold(want);
|
||||
}
|
||||
|
||||
@@ -40,11 +40,12 @@ namespace strata::ds4::os {
|
||||
|
||||
/// Windows sleeps in 15.6 ms ticks unless a process asks for finer ones: the CPU lane's service thread naps (sleep_for 20 us)
|
||||
/// between tokens, and the device's first wait of the next token then waits out the tick. 1 ms for the process's life
|
||||
/// (Project Maya #358); DS4_TIMER_MS=0 leaves the system's resolution alone. Elsewhere nothing.
|
||||
/// (Project Maya #358). Off by default (owner PC, shipped profile, 3 pairs: not faster in every pair); DS4_TIMER_MS=1 turns it on.
|
||||
/// Elsewhere nothing.
|
||||
inline void fine_timer() {
|
||||
#if defined(_WIN32)
|
||||
const char* v = std::getenv("DS4_TIMER_MS");
|
||||
const int ms = v != nullptr ? std::atoi(v) : 1;
|
||||
const int ms = v != nullptr ? std::atoi(v) : 0;
|
||||
if (ms > 0) timeBeginPeriod((UINT) ms);
|
||||
#endif
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user