mirror of
https://github.com/Niko1221/Strata.git
synced 2026-11-07 14:21:06 -10:00
NVIDIA: STRATA_MMQ_KQ=1 is on by default for compute capability 8.0 and up (owner yes 11-03): K-quant experts (UD-Q4/Q5/Q6_K_XL) read the prompt on the MMQ kernels
kNvKqDefaultOn is true; STRATA_MMQ_KQ=0, STRATA_NV_KQ_DEFAULT=0 and STRATA_DEFAULTS=026 turn it off, a STRATA_MMQ_KQ the user sets is kept, Volta / Turing / Pascal keep the FP16 prompt path. A pack without K-quant experts never reads the variable. Start check (RTX PRO 6000, UD-Q4_K_XL, no STRATA variable set): the start prints the K-quant MMQ line and loads libstrata_kq; 4K prompt GPU timeline 1,047 ms (dequant 7.3%) against 1,500 ms (dequant 25.1%) with STRATA_MMQ_KQ=0, prompt 2,854 vs 2,233 tok/s (one request each).
This commit is contained in:
1 parent
e2f7c47447
commit
52ac4e7337
3 files changed
+9
-7
No files matched your search
@@ -36,12 +36,13 @@ std::vector<std::string> apply_nv_defaults(int major, int minor);
|
||||
|
||||
namespace strata::core {
|
||||
|
||||
/// NVIDIA candidate default that changes the answers (round 0212, NVSINGLE; NOT on until the owner says yes): STRATA_MMQ_KQ=1, the K-quant MMQ prompt
|
||||
/// NVIDIA default that changes the answers (round 0212, NVSINGLE; the owner said yes 11-03): STRATA_MMQ_KQ=1, the K-quant MMQ prompt
|
||||
/// kernels for the experts of Unsloth's UD-Q4/Q5/Q6_K_XL files (Q4_K / Q5_K / Q5_1 / Q6_K), which otherwise are dequantized to FP16 and multiplied by
|
||||
/// cuBLAS. A pack without K-quant experts never reads the variable. Cards of compute capability 8.0 and up only (measured on sm_86 and sm_120).
|
||||
/// Today the rule is behind STRATA_NV_KQ_DEFAULT=1; flipping kNvKqDefaultOn makes it a default with STRATA_MMQ_KQ=0, STRATA_NV_KQ_DEFAULT=0 and
|
||||
/// STRATA_DEFAULTS=026 as the off switches. min_major / min_minor are the lowest compute capability among the visible cards.
|
||||
constexpr bool kNvKqDefaultOn = false;
|
||||
/// On by default; STRATA_MMQ_KQ=0, STRATA_NV_KQ_DEFAULT=0 and STRATA_DEFAULTS=026 are the off switches (kNvKqDefaultOn=false would leave it behind
|
||||
/// STRATA_NV_KQ_DEFAULT=1). Measured: RTX PRO 6000 prompt +26..38%, A6000 +2..11%, first-token KL 0.024 / 0.035, 61-task suite 54 vs 53.
|
||||
/// min_major / min_minor are the lowest compute capability among the visible cards.
|
||||
constexpr bool kNvKqDefaultOn = true;
|
||||
std::vector<std::pair<std::string, std::string>> nv_kq_default_env(int min_major, int min_minor, bool defaults026);
|
||||
std::vector<std::pair<std::string, std::string>> nv_kq_default_env(int min_major, int min_minor);
|
||||
/// Sets what nv_kq_default_env returns where the user has not set the variable; prints one line to stderr when it set anything.
|
||||
|
||||
@@ -66,9 +66,10 @@ int main() {
|
||||
CHECK(!p.empty() && p == q);
|
||||
CHECK(!find(a, "STRATA_PF_FUSED") && !find(p, "STRATA_PF_FUSED"));
|
||||
}
|
||||
// round 0212 candidate: STRATA_MMQ_KQ=1 for compute capability 8.0+, behind STRATA_NV_KQ_DEFAULT=1 until kNvKqDefaultOn flips
|
||||
// round 0212: STRATA_MMQ_KQ=1 is a default for compute capability 8.0+ (owner yes 11-03), with its own off switches
|
||||
unset("STRATA_NV_KQ_DEFAULT");
|
||||
CHECK(kNvKqDefaultOn || nv_kq_default_env(12, 0, false).empty());
|
||||
CHECK(kNvKqDefaultOn);
|
||||
for (int major : {8, 9, 10, 12}) CHECK(nv_kq_default_env(major, 0, false).size() == 1); // on without any switch
|
||||
put("STRATA_NV_KQ_DEFAULT", "1");
|
||||
for (int major : {8, 9, 10, 12}) {
|
||||
const auto k = nv_kq_default_env(major, 0, false);
|
||||
|
||||
@@ -3799,7 +3799,7 @@ int main(int argc, char** argv) {
|
||||
cudaDeviceGetAttribute(&n, cudaDevAttrComputeCapabilityMinor, d) == cudaSuccess && m == m0 && n == n0;
|
||||
}
|
||||
if (same) { strata::core::apply_nv_defaults(m0, n0); strata::core::apply_spec_adapt_default(m0, n0); }
|
||||
{ // round 0212: the K-quant MMQ prompt kernels (flag STRATA_NV_KQ_DEFAULT=1 until the owner says yes); the lowest compute capability decides
|
||||
{ // round 0212: the K-quant MMQ prompt kernels, on by default for compute capability 8.0+ (STRATA_MMQ_KQ=0 or STRATA_NV_KQ_DEFAULT=0 turns them off); the lowest compute capability decides
|
||||
int lo_m = m0, lo_n = n0;
|
||||
for (int d = 1; d < nd; ++d) {
|
||||
int m = 0, n = 0;
|
||||
|
||||
Reference in new issue
Block a user