mirror of
https://github.com/Heretek-AI/Strata.git
synced 2026-11-08 08:07:26 -10:00
Problem. On a layer split the later cards are left with very little VRAM after loading (#3702: 68 MiB on the last card; #3638: 268 and 666 MiB, then every batched prompt read of more than ~400 tokens ends non-finite after hours; #3797: the weight arena fails with 320 MiB free and the message does not say who holds the rest). The window-graph eviction (12b0bce95) and the copy stream pool (#3350) are in already; this is the rest of the code audit of what is allocated after load. Audit. The table of every lazy allocation after load (what, when, size, how a failure is handled) is in phaseA-tests/notes-027-gh-eng-d.md. Bugs found and fixed here, each small: - verify.cpp capture / capture_batch and mtp.cpp finish_capture: the cudaGraphUpload and its sync were only printed. The upload is a warm-up, but a failure (no VRAM) stayed as the thread's pending error and the next kernel wrapper's cudaGetLastError ended the engine. The error is now cleared and the first launch uploads the graph. - verify.cpp capture_commit and mtp.cpp finish_capture: the commit graph called cudaGraphInstantiate directly (no eviction of older layouts, unlike the window graphs) and reported the status of the end-of-capture call, which is success, so the message read "no error". capture_commit now goes through instantiate_evicting and reports the instantiate's status. - instantiate_evicting instantiates into a local and sets the member only on success (what a failed instantiate leaves in its output is not specified, and a non-null member means "captured"). - prefill.cpp carve: after a failed take of the shared region the sub-allocators had no base and each cudaMalloc'd its buffer (hundreds of MiB in all) on the card that had just run out. carve now returns at once; the caller reports the failure as before. - generate.cpp: a failed set_stage_helper / set_peer is reported and the run goes on, but its CUDA error stayed pending; cleared. Found and not changed: relend() re-plans only the first participant (#3638 comment). Read: it runs only from the VRAM command, which is off on a layer split, so it cannot touch a later stage's loan. Other candidates (ignored cudaEventCreate for the pipelined fences, unchecked expert copies, cudaFuncSetAttribute flags set before success) are in the notes. Changes for the report of #3797 and the reserve (#3702). - include/strata/program/gpu_report.hpp: on a failed weight-arena allocation (first card and each stage) the engine now also prints, for every card of the run, the free and total memory and the largest other programs that hold memory on it (NVML, loaded at run time; where the driver does not list them it says so). Text only; the failure is the same. tests/program/gpu_report_test.cpp pins the text. - include/strata/program/stage_reserve.hpp: the arithmetic of stage_room (what the split keeps free on a card) moved out of generate.cpp unchanged; tests/program/stage_reserve_test.cpp compares it with the old expression over a grid. The file also holds a PROPOSAL for a larger window term on a later card (96 MiB + 6 MiB per further window size + 48 MiB, 174 MiB for --spec 4) which nothing calls: the default placement is not changed. gates/3702-later-reserve.sh measures it. - The low-VRAM warning of a later card now adds how many MiB the plan meant to keep free there and how far off that was. Tests: gpu_report_test, stage_reserve_test (host only, built on z440). Not covered: no model was run. What the ~700 MiB the plan does not count on the last card is (window graph sets, kernel code loads) needs the run in gates/3702-later-reserve.sh.
97 lines
5.1 KiB
C++
97 lines
5.1 KiB
C++
// tests/program/gpu_report_test.cpp - #3797: the out-of-memory report at start (pure text; the NVML query is only
|
|
// checked to be safe to call: it must return, not throw, and leave the list empty when it answers false).
|
|
#include "strata/program/gpu_report.hpp"
|
|
|
|
#include <cstdio>
|
|
#include <string>
|
|
|
|
using namespace strata::program::gpu_report;
|
|
|
|
namespace {
|
|
int g_fail = 0;
|
|
void check(bool ok, const char* what, const std::string& got = {}) {
|
|
if (ok) return;
|
|
std::fprintf(stderr, "FAIL: %s%s%s\n", what, got.empty() ? "" : "\n got: ", got.c_str());
|
|
++g_fail;
|
|
}
|
|
bool has(const std::string& s, const char* part) { return s.find(part) != std::string::npos; }
|
|
} // namespace
|
|
|
|
int main() {
|
|
{ // the case of #3797: 320 MiB free of a 22 GB card, another program holds the rest
|
|
Card c;
|
|
c.index = 1;
|
|
c.name = "NVIDIA GeForce RTX 2080 Ti";
|
|
c.free_mib = 320;
|
|
c.total_mib = 21998;
|
|
c.processes_known = true;
|
|
c.processes = {{4242, "/usr/bin/python3.12", 20100}, {777, "/opt/strata/strata", 900}, {1, "Xorg", 12}, {5, "", 0}};
|
|
const std::string r = format_gpu_memory_report({c}, 777);
|
|
check(has(r, "CUDA1 (NVIDIA GeForce RTX 2080 Ti): 320 MiB free of 21998 MiB"), "free and total", r);
|
|
check(has(r, "python3.12 (pid 4242) 19.6 GiB"), "the largest other program, by file name", r);
|
|
check(!has(r, "strata (pid 777)"), "this process is not listed as another program", r);
|
|
check(has(r, "Xorg (pid 1) 12 MiB"), "a small one is still named", r);
|
|
check(has(r, "1 program whose memory the driver does not report"), "a process of unknown size is counted", r);
|
|
check(r.find("python3.12") < r.find("Xorg"), "largest first", r);
|
|
}
|
|
{ // more than three: the rest is counted
|
|
Card c;
|
|
c.index = 0;
|
|
c.free_mib = 100;
|
|
c.total_mib = 12288;
|
|
c.processes_known = true;
|
|
for (uint32_t i = 1; i <= 6; ++i) c.processes.push_back({i, "p" + std::to_string(i), 100u * i});
|
|
const std::string r = format_gpu_memory_report({c}, 0);
|
|
check(has(r, "p6 (pid 6) 600 MiB, p5 (pid 5) 500 MiB, p4 (pid 4) 400 MiB, and 3 more"), "three named, three counted", r);
|
|
check(!has(r, "p3 (pid 3)"), "the fourth is not named", r);
|
|
}
|
|
{ // the driver would not say: no claim about other programs
|
|
Card c;
|
|
c.index = 0;
|
|
c.free_mib = 512;
|
|
c.total_mib = 8192;
|
|
c.processes_known = false;
|
|
const std::string r = format_gpu_memory_report({c}, 1);
|
|
check(has(r, "512 MiB free of 8192 MiB; the driver does not say which programs use it here"), "unknown processes", r);
|
|
check(!has(r, "other programs on it"), "no list", r);
|
|
}
|
|
{ // the driver answered with nothing but ourselves: said carefully (Windows often lists no program at all)
|
|
Card c;
|
|
c.index = 0;
|
|
c.free_mib = 700;
|
|
c.total_mib = 12288;
|
|
c.processes_known = true;
|
|
c.processes = {{99, "strata.exe", 11000}};
|
|
const std::string r = format_gpu_memory_report({c}, 99);
|
|
check(has(r, "the driver lists no other program using it"), "only ourselves", r);
|
|
}
|
|
{ // two cards, a pid with no name, a Windows path
|
|
Card a, b;
|
|
a.index = 0; a.free_mib = 2000; a.total_mib = 16000; a.processes_known = true;
|
|
b.index = 1; b.free_mib = 50; b.total_mib = 12000; b.processes_known = true;
|
|
b.processes = {{31, "", 11000}, {32, "C:\\Program Files\\Obs\\obs64.exe", 300}};
|
|
const std::string r = format_gpu_memory_report({a, b}, 5);
|
|
check(has(r, "CUDA0: 2000 MiB free of 16000 MiB; the driver lists no other program using it\n"), "first card line", r);
|
|
check(has(r, "CUDA1: 50 MiB free of 12000 MiB; other programs on it: pid 31 10.7 GiB, obs64.exe (pid 32) 300 MiB"), "second card line", r);
|
|
}
|
|
check(format_gpu_memory_report({}, 1).empty(), "no cards, no text");
|
|
{ // planned versus free
|
|
check(planned_vs_free_text(796, 68) ==
|
|
"the layer split planned to keep 796 MiB free here after loading; 68 MiB are free, so 728 MiB went to something the plan did not count",
|
|
"planned versus free", planned_vs_free_text(796, 68));
|
|
check(planned_vs_free_text(796, 796).empty() && planned_vs_free_text(796, 900).empty(), "enough free: nothing to say");
|
|
check(planned_vs_free_text(0, 10).empty() && planned_vs_free_text(-5, 10).empty(), "no plan, no text");
|
|
check(has(planned_vs_free_text(300, -4), "0 MiB are free, so 300 MiB"), "a negative free count is read as 0");
|
|
}
|
|
{ // the query never throws and never leaves a half-filled list behind a false
|
|
std::vector<Process> p{{1, "x", 1}};
|
|
const bool ok = query_other_processes("", p);
|
|
check(!ok && p.empty(), "an empty bus id is refused");
|
|
std::vector<Process> q{{1, "x", 1}};
|
|
const bool r = query_other_processes("ffff:ff:ff.f", q); // no such card: false, or true with a list; both are fine
|
|
check(r || q.empty(), "a failed query leaves the list empty");
|
|
}
|
|
std::printf(g_fail ? "FAIL\n" : "PASS\n");
|
|
return g_fail ? 1 : 0;
|
|
}
|