NVIDIA: STRATA_MMQ_KQ=1 is on by default for compute capability 8.0 and up (owner yes 11-03): K-quant experts (UD-Q4/Q5/Q6_K_XL) read the prompt on the MMQ kernels

kNvKqDefaultOn is true; STRATA_MMQ_KQ=0, STRATA_NV_KQ_DEFAULT=0 and STRATA_DEFAULTS=026 turn it off, a STRATA_MMQ_KQ the user sets is kept, Volta / Turing / Pascal keep the FP16 prompt path. A pack without K-quant experts never reads the variable.
Start check (RTX PRO 6000, UD-Q4_K_XL, no STRATA variable set): the start prints the K-quant MMQ line and loads libstrata_kq; 4K prompt GPU timeline 1,047 ms (dequant 7.3%) against 1,500 ms (dequant 25.1%) with STRATA_MMQ_KQ=0, prompt 2,854 vs 2,233 tok/s (one request each).
This commit is contained in:
Niko1221 committed 2026-11-03 12:20:34 +01:00
1 parent e2f7c47447
commit 52ac4e7337
3 files changed
+9 -7

No files matched your search

+5 -4
View File
@@ -36,12 +36,13 @@ std::vector<std::string> apply_nv_defaults(int major, int minor);
namespace strata::core {
/// NVIDIA candidate default that changes the answers (round 0212, NVSINGLE; NOT on until the owner says yes): STRATA_MMQ_KQ=1, the K-quant MMQ prompt
/// NVIDIA default that changes the answers (round 0212, NVSINGLE; the owner said yes 11-03): STRATA_MMQ_KQ=1, the K-quant MMQ prompt
/// kernels for the experts of Unsloth's UD-Q4/Q5/Q6_K_XL files (Q4_K / Q5_K / Q5_1 / Q6_K), which otherwise are dequantized to FP16 and multiplied by
/// cuBLAS. A pack without K-quant experts never reads the variable. Cards of compute capability 8.0 and up only (measured on sm_86 and sm_120).
/// Today the rule is behind STRATA_NV_KQ_DEFAULT=1; flipping kNvKqDefaultOn makes it a default with STRATA_MMQ_KQ=0, STRATA_NV_KQ_DEFAULT=0 and
/// STRATA_DEFAULTS=026 as the off switches. min_major / min_minor are the lowest compute capability among the visible cards.
constexpr bool kNvKqDefaultOn = false;
/// On by default; STRATA_MMQ_KQ=0, STRATA_NV_KQ_DEFAULT=0 and STRATA_DEFAULTS=026 are the off switches (kNvKqDefaultOn=false would leave it behind
/// STRATA_NV_KQ_DEFAULT=1). Measured: RTX PRO 6000 prompt +26..38%, A6000 +2..11%, first-token KL 0.024 / 0.035, 61-task suite 54 vs 53.
/// min_major / min_minor are the lowest compute capability among the visible cards.
constexpr bool kNvKqDefaultOn = true;
std::vector<std::pair<std::string, std::string>> nv_kq_default_env(int min_major, int min_minor, bool defaults026);
std::vector<std::pair<std::string, std::string>> nv_kq_default_env(int min_major, int min_minor);
/// Sets what nv_kq_default_env returns where the user has not set the variable; prints one line to stderr when it set anything.
+3 -2
View File
@@ -66,9 +66,10 @@ int main() {
CHECK(!p.empty() && p == q);
CHECK(!find(a, "STRATA_PF_FUSED") && !find(p, "STRATA_PF_FUSED"));
}
// round 0212 candidate: STRATA_MMQ_KQ=1 for compute capability 8.0+, behind STRATA_NV_KQ_DEFAULT=1 until kNvKqDefaultOn flips
// round 0212: STRATA_MMQ_KQ=1 is a default for compute capability 8.0+ (owner yes 11-03), with its own off switches
unset("STRATA_NV_KQ_DEFAULT");
CHECK(kNvKqDefaultOn || nv_kq_default_env(12, 0, false).empty());
CHECK(kNvKqDefaultOn);
for (int major : {8, 9, 10, 12}) CHECK(nv_kq_default_env(major, 0, false).size() == 1); // on without any switch
put("STRATA_NV_KQ_DEFAULT", "1");
for (int major : {8, 9, 10, 12}) {
const auto k = nv_kq_default_env(major, 0, false);
+1 -1
View File
@@ -3799,7 +3799,7 @@ int main(int argc, char** argv) {
cudaDeviceGetAttribute(&n, cudaDevAttrComputeCapabilityMinor, d) == cudaSuccess && m == m0 && n == n0;
}
if (same) { strata::core::apply_nv_defaults(m0, n0); strata::core::apply_spec_adapt_default(m0, n0); }
{ // round 0212: the K-quant MMQ prompt kernels (flag STRATA_NV_KQ_DEFAULT=1 until the owner says yes); the lowest compute capability decides
{ // round 0212: the K-quant MMQ prompt kernels, on by default for compute capability 8.0+ (STRATA_MMQ_KQ=0 or STRATA_NV_KQ_DEFAULT=0 turns them off); the lowest compute capability decides
int lo_m = m0, lo_n = n0;
for (int d = 1; d < nd; ++d) {
int m = 0, n = 0;