gfx12: opt-in switch STRATA_PF_FUSED_IQ2S=1 for the native fused prompt kernels on the IQ2_S and IQ4_XS gate/up layers (Linux; the IQ3_S pack's 21 layers that keep MMQ), and the CPU share limit drops from 3,072 to 1,536 tokens only when it is on (share_rules::gfx12_share_limit), so a chunk from 1,536 up streams all and runs them. OFF by default: nothing changes without the variable. R9700 (p620, IQ3_S, 54% resident), on against the share-only default: prompt 2K +10.8%, 4K +12.2%, 8K +13.2%, 32K +10.0% (5 pairs each, 5/5), 1K and 1.5K unchanged; values change (first-token KL against the FP16 prompt path 0.0150 vs MMQ 0.0263). The quick quality suite lost 2 of 19 tasks with it on (summ-M-d14, code-S-d04) where off and off again agree, so it stays opt-in until the full 61-task suite has run. Separate commit: drop it alone to go back to the share-only default

This commit is contained in:
Niko1221 committed 2026-11-01 01:16:23 +01:00
1 parent d2068bde56
commit b9e3e2b86a
5 files changed
+30 -3

No files matched your search

+4
View File
@@ -29,6 +29,10 @@ struct NativeGeom {
/// enabled() (STRATA_PF_FUSED=1 on sm_80+), and the kernels cover this gate/up and down pair on this device.
bool native_supported(int gu_type, int d_type);
/// AMD-Q: true when the gfx12 build also takes the IQ2_S and IQ4_XS gate/up layers on the fused kernels (Linux; OPT-IN: STRATA_PF_FUSED_IQ2S=1; STRATA_PF_FUSED=0 turns all fused kernels off).
/// prefill.cpp picks the CPU share's limit from it (share_rules::gfx12_share_limit).
bool gfx12_iq2s_default();
/// x [rows][cols] FP32 -> `xa` (act_bytes(rows, cols)): int8 per 32 values in natural order, the native kernels' form.
void quantize_act_native(const float* x, int64_t rows, int64_t cols, void* xa, void* stream);
+5
View File
@@ -22,6 +22,11 @@ constexpr int64_t kShareLimitOld = 1024;
constexpr int64_t kShareLimitNew = 3072;
/// `raise`: the card class may take the new limit (a class whose A/B lost at 1-3K keeps 1,024, automatically).
constexpr int64_t default_share_limit(bool raise = true) { return raise ? kShareLimitNew : kShareLimitOld; }
/// AMD-Q: a gfx12 card whose fused IQ2_S / IQ4_XS gate/up kernels are on (fused::gfx12_iq2s_default): chunks below this are staged and take the CPU share, a chunk
/// from it up streams every expert and runs the fused kernels. R9700 (p620, IQ3_S, 54% resident), prompt speed against the share-only default (limit 3,072):
/// limit 1,536: 1.3K 1.00, 1.5K 1.00, 1.8K +7.1%, 2K +10.0%, 2.6K +16.2%, 3K +11.1% (two pairs each); limit 2,048: 1.8K +1.4%, 2K +9.7%; limit 3,072 (fused only above): no gain at 1.5K-2.6K.
constexpr int64_t kShareLimitGfx12Fused = 1536;
constexpr int64_t gfx12_share_limit(bool fused_iq2s) { return fused_iq2s ? kShareLimitGfx12Fused : kShareLimitNew; }
/// resident_slots: the expert cache's live slots; total_pairs: layers x routed experts per layer (<= 0: unknown, arm).
constexpr bool default_share_has_work(int64_t resident_slots, int64_t total_pairs) {
+17 -2
View File
@@ -1973,6 +1973,17 @@ int pick_ww(int64_t n, int n_expert) {
} // namespace
bool gfx12_iq2s_default() {
static const bool on = [] {
const char* v = std::getenv("STRATA_PF_FUSED_IQ2S");
const char* all = std::getenv("STRATA_PF_FUSED");
// OPT-IN (STRATA_PF_FUSED_IQ2S=1): the quick quality suite lost 2 of 19 tasks with it on (summ-M-d14 one unsupported name, code-S-d04 a syntax error)
// while the same two OFF runs agree; the owner decides on the full suite. Flip the default here: `!(v != nullptr && v[0] == '0')`.
return strata::core::kGfx12PromptDefaultsOn && v != nullptr && v[0] == '1' && !(all != nullptr && all[0] == '0');
}();
return on;
}
bool native_supported(int gu_type, int d_type) {
static const bool off = [] { // STRATA_PF_FUSED_NATIVE=0: the native packs keep MMQ under STRATA_PF_FUSED=1 (A/B)
const char* v = std::getenv("STRATA_PF_FUSED_NATIVE");
@@ -2006,8 +2017,12 @@ bool native_supported(int gu_type, int d_type) {
}
if (gu_type == T_Q2_0 && d_type != T_Q2_0) return false; // the opt-in (STRATA_PF_FUSED=1 as well) takes the Q2_0 / Q2_0 pair only
#endif
// default: only these three; the Q2_0 one (148-amd) is on Linux only: on Windows it is opt-in (STRATA_PF_FUSED=1) until tested there
if (env < 0 && gu_type != T_IQ3_S && gu_type != T_IQ3_XXS && !(gu_type == T_Q2_0 && strata::core::kGfx12PromptDefaultsOn)) return false;
// default: only these three; the Q2_0 one (148-amd) is on Linux only: on Windows it is opt-in (STRATA_PF_FUSED=1) until tested there.
// AMD-Q (owner decision 10-31): the IQ2_S and IQ4_XS gate/up layers too (the IQ3_S pack's 21 layers that kept MMQ), Linux only like the Q2_0 entry.
// R9700 (p620, IQ3_S, serve, 3 interleaved pairs): prompt 2K +9.7%, 8K +12.2%, 32K +8.8%; first-token KL against the MMQ default 0.017 (29 prompts of
// 1,025 tokens, top-1 28 of 29), the answers are not the MMQ bytes. STRATA_PF_FUSED_IQ2S=1 turns it on (opt-in until the full quality suite has run); STRATA_PF_FUSED=0 turns all fused kernels off.
if (env < 0 && gu_type != T_IQ3_S && gu_type != T_IQ3_XXS && !(gu_type == T_Q2_0 && strata::core::kGfx12PromptDefaultsOn) &&
!(gfx12_iq2s_default() && (gu_type == T_IQ2_S || gu_type == T_IQ4_XS))) return false;
const DevInfo& d = dev_info();
if (!d.ok || !d_covered_on(d_type, d)) return false;
if (d.t75 && (gu_type != T_Q2_0 || d_type != T_Q2_0)) return false;
+3 -1
View File
@@ -258,6 +258,7 @@ inline bool cpu_share_on() { return cpu_share_env() != 0.0; }
// STRATA_PREFILL_CPU_SHARE_MAX (default 3072, at least 1024): with the share on, chunks below it are staged after their
// routing - so they can hand the CPU its share - instead of streaming every expert the cards do not hold. The share pays
// on chunks up to ~3K tokens, where few tokens route to many of the streamed experts; 1024 keeps the old limit.
int64_t g_share_limit_arch = 0; // AMD-Q: the default limit of a gfx12 card (share_rules::gfx12_share_limit), set by arm_cpu_share
inline int64_t cpu_share_max() {
static const int64_t v = [] {
const char* e = std::getenv("STRATA_PREFILL_CPU_SHARE_MAX");
@@ -265,7 +266,7 @@ inline int64_t cpu_share_max() {
}();
static const bool explicit_max = std::getenv("STRATA_PREFILL_CPU_SHARE_MAX") != nullptr;
if (g_share_default && cpu_share_explicit() == -2.0 && !explicit_max) // the default-armed share: share_rules::default_share_limit (STRATA_PREFILL_CPU_SHARE_MAX=1024 restores the old 1,024)
return strata::prefill::share_rules::default_share_limit();
return g_share_limit_arch > 0 ? g_share_limit_arch : strata::prefill::share_rules::default_share_limit();
return v;
}
// On a layer split every stage may have the pool (set_cpu_pool), and the stages read different chunks at the same time:
@@ -2642,6 +2643,7 @@ void Prefill::arm_cpu_share(bool applies, bool by_default) {
if (cudaGetDevice(&dev) == cudaSuccess && cudaGetDeviceProperties(&prop, dev) == cudaSuccess &&
strata::core::gfx12_prefill_share_default(prop.gcnArchName)) {
g_share_default = true;
g_share_limit_arch = strata::prefill::share_rules::gfx12_share_limit(fused::gfx12_iq2s_default());
std::fprintf(stderr, "prefill: the CPU share is ON by default for prompt chunks up to %lld tokens on this gfx12 card (the idle CPU "
"takes some of the experts the GPU would stream; answers can differ slightly: first-token KL ~0.02 at 1K tokens, "
"prompts of 500-1,000 tokens +18..+31%%, 2K and up unchanged). STRATA_PREFILL_CPU_SHARE=0 or STRATA_GFX12_DEFAULTS=0 turns it off, "
+1
View File
@@ -23,6 +23,7 @@ int main() {
CHECK(strata::prefill::share_rules::default_share_limit() == 3072);
CHECK(strata::prefill::share_rules::default_share_limit(true) == strata::prefill::share_rules::kShareLimitNew);
CHECK(strata::prefill::share_rules::default_share_limit(false) == 1024);
CHECK(strata::prefill::share_rules::gfx12_share_limit(true) == 1536 && strata::prefill::share_rules::gfx12_share_limit(false) == 3072); // AMD-Q
static_assert(strata::prefill::share_rules::kShareLimitOld == 1024, "the old limit (the way back is STRATA_PREFILL_CPU_SHARE_MAX=1024)");
// H11: memory verdicts