SYCL: debug-only hash dumps to find where two runs of the same prompt first differ (all off unless set)

STRATA_PF_HASH_FILE: per row hashes of the first GDN layer's stages in the prompt path. STRATA_LAYER_HASH_FILE: per layer hashes of the payload the host sees for the CPU experts and of their output. STRATA_STATE_HASH_FILE / STRATA_STATE_DUMP: hashes (and a raw dump of the first four layers) of the GDN states and filled int8 KV before a window. They found the oneMKL bf16 GEMM repeatability problem fixed in the previous commit.

sycl/src/core/verify.cpp

@@ -96,6 +96,38 @@ bool env_on(const char* name) {
     const char* e = std::getenv(name);
     return e != nullptr && e[0] != 0 && e[0] != '0';
 }
+
+// STRATA_LAYER_HASH_FILE=<path> (debug, off by default): after every CPU-expert service the host writes one text line
+// "window layer rows xhash idshash whash yhash" (FNV-1a over the host-visible payload the GPU published for the CPU
+// experts - the activation rows, the routed ids, the weights - and over the CPU experts' output rows).  Two runs of the
+// same teacher-forced window are compared line by line: the first line that differs names the first layer whose input
+// differs (CLEAN-6.0).  The activation rows are published only when a routed expert is not resident, or with
+// STRATA_DOORBELL_CHECK=1 for every layer.
+std::FILE* layer_hash_file() {
+    static std::FILE* f = [] {
+        const char* p = std::getenv("STRATA_LAYER_HASH_FILE");
+        return p != nullptr && p[0] != 0 ? std::fopen(p, "w") : nullptr;
+    }();
+    return f;
+}
+uint64_t layer_hash_fnv(const void* p, size_t bytes) {
+    uint64_t h = 1469598103934665603ull;
+    const auto* b = static_cast<const uint8_t*>(p);
+    for (size_t i = 0; i < bytes; ++i) h = (h ^ b[i]) * 1099511628211ull;
+    return h;
+}
+
+// STRATA_STATE_HASH_FILE=<path> (debug, off by default): before every window the host waits for the queue and writes
+// "window gdn <ordinal> <fnv>" for every GDN layer's recurrent+conv state and "window qsa <ordinal> <fnv>" for the
+// int8 K/V codes and scales of the pages the window's position already filled (page table taken as the identity),
+// so two runs can be compared at the first window to see whether the prompt path left the same state (CLEAN-6.0).
+std::FILE* state_hash_file() {
+    static std::FILE* f = [] {
+        const char* p = std::getenv("STRATA_STATE_HASH_FILE");
+        return p != nullptr && p[0] != 0 ? std::fopen(p, "w") : nullptr;
+    }();
+    return f;
+}
 #if defined(STRATA_USE_HIP)
 const bool g_coherent = env_on("STRATA_VERIFY_COHERENT");
 const bool g_doorbell_store = env_on("STRATA_DOORBELL_STORE");
@@ -2159,6 +2191,36 @@ bool Verifier::run(int T, const int32_t *tokens, int64_t pos0, PoolMultiFn pool,
         }
     }
     if (trace_h_ != nullptr) std::memset(trace_h_, 0, trace_n_ * 8);   // #649: this window's breadcrumbs only
+    if (std::FILE* sf = state_hash_file()) {   // debug only (STRATA_STATE_HASH_FILE)
+        cs_->wait();
+        const uint64_t gf = (uint64_t) g.ssm_state_size * g.ssm_v_heads * g.ssm_state_size +
+                            (uint64_t) g.ssm_conv_channels * (g.ssm_d_conv - 1);
+        std::vector<char> hb(std::max<uint64_t>(gf * 4, 1u << 20));
+        for (int64_t i = 0; i < ss.gdn_alloc; ++i) {
+            cs_->memcpy(hb.data(), ss.gdn_state + (size_t) i * gf, gf * 4).wait();
+            if (const char* dp = std::getenv("STRATA_STATE_DUMP"); dp != nullptr && windows == 0 && i < 4) {   // raw state of the first GDN layers
+                const std::string fn = std::string(dp) + "_gdn" + std::to_string(ss.gdn_ord0 + i) + ".bin";
+                if (std::FILE* df = std::fopen(fn.c_str(), "wb")) { std::fwrite(hb.data(), 1, gf * 4, df); std::fclose(df); }
+            }
+            std::fprintf(sf, "%lld gdn %lld %016llx\n", (long long) windows, (long long) (ss.gdn_ord0 + i),
+                         (unsigned long long) layer_hash_fnv(hb.data(), gf * 4));
+        }
+        const int64_t pages = (pos0 + 3) / 4;
+        for (int64_t i = 0; i < ss.qsa_alloc; ++i) {
+            const QsaState& q = ss.qsa_states[i];
+            if (!q.kv_int8 || q.k_q == nullptr) continue;
+            const size_t kb = (size_t) pages * g.n_head_kv * 4 * g.head_dim;
+            uint64_t h = 1469598103934665603ull;
+            auto mix = [&](const void* dev, size_t bytes) {
+                if (bytes > hb.size()) hb.resize(bytes);
+                cs_->memcpy(hb.data(), dev, bytes).wait();
+                h ^= layer_hash_fnv(hb.data(), bytes); h *= 1099511628211ull;
+            };
+            mix(q.k_q, kb); mix(q.v_q, kb); mix(q.k_scale, kb / 64 * 2); mix(q.v_scale, kb / 64 * 2);
+            std::fprintf(sf, "%lld qsa %lld %016llx\n", (long long) windows, (long long) (ss.qsa_ord0 + i), (unsigned long long) h);
+        }
+        std::fflush(sf);
+    }
     std::atomic_thread_fence(std::memory_order_seq_cst);
     trace_ev("WINDOW", -1, -1, pos0 * 16 + T);
     ms_host += ms_since(t0);
@@ -2412,6 +2474,14 @@ bool Verifier::run(int T, const int32_t *tokens, int64_t pos0, PoolMultiFn pool,
             pool(user, h_x_ + (size_t) tb * g.n_embd, h_ids_ + (size_t) tb * ss.k, n, ss.k,
                  h_ymiss_ + (size_t) tb * ss.k * g.n_embd, l);
         if (remote_opt_) remote_opt_->end();
+        if (std::FILE* lf = layer_hash_file()) {   // debug only (STRATA_LAYER_HASH_FILE)
+            std::fprintf(lf, "%lld %lld %d %016llx %016llx %016llx %016llx\n", (long long) windows, (long long) l, n,
+                         (unsigned long long) layer_hash_fnv(h_x_ + (size_t) tb * g.n_embd, (size_t) n * g.n_embd * 4),
+                         (unsigned long long) layer_hash_fnv(h_ids_ + (size_t) tb * ss.k, (size_t) n * ss.k * 4),
+                         (unsigned long long) layer_hash_fnv(h_w_ + (size_t) tb * ss.k, (size_t) n * ss.k * 4),
+                         (unsigned long long) layer_hash_fnv(h_ymiss_ + (size_t) tb * ss.k * g.n_embd, (size_t) n * ss.k * g.n_embd * 4));
+            std::fflush(lf);
+        }
         VDBG("layer %lld served\n", (long long) l);
         if (g_trace) trace_ev(*(volatile uint32_t*) h_flagA_ == want ? "SERVED" : "SERVED-NO-PLAN-YET", k, l,
                               (int64_t) ms_since(b));   // aux: ms the CPU experts took

sycl/src/prefill/prefill.cpp

@@ -146,6 +146,24 @@ inline bool dma_direct(const core::ExpertSource* src, int64_t layer, int64_t exp
 constexpr float EPS = 1e-6f;
 constexpr int64_t N = 2560, HC = 4, D = N * HC, LR = 320, K = 10, NE = 512;
 constexpr int64_t C = 10240, ZV = 6144, HV = 48;
+
+// STRATA_PF_HASH_FILE=<path> (debug, off by default): for the first GDN layer of every prompt chunk the host waits for
+// the queue after each stage and writes one line per row "chunk stage row fnv" (the layer input, the projections, the
+// gates, the conv output, the recurrence output) plus the layer's recurrent state, so two runs show which stage of
+// the prefill first differs (CLEAN-6.0).
+inline std::FILE* pf_hash_file() {
+    static std::FILE* f = [] {
+        const char* p = std::getenv("STRATA_PF_HASH_FILE");
+        return p != nullptr && p[0] != 0 ? std::fopen(p, "w") : nullptr;
+    }();
+    return f;
+}
+inline uint64_t pf_hash_fnv(const void* p, size_t bytes) {
+    uint64_t h = 1469598103934665603ull;
+    const auto* b = static_cast<const uint8_t*>(p);
+    for (size_t i = 0; i < bytes; ++i) h = (h ^ b[i]) * 1099511628211ull;
+    return h;
+}
 // plan v0.3 P6: staging holds the largest blob of the pack (a native pack's blobs differ per layer)
 inline int64_t MAXBLOB() { return (int64_t) strata::kernels::cpu::expert_layout().max_blob; }
 constexpr int STAGE = 8;           // host->device expert staging ring (chunks below stream_all_min())
@@ -3227,17 +3245,39 @@ bool Prefill::run_impl(const int64_t *tokens, int64_t n, int64_t pos0,
                     pt.mark(kPfGdn, cs);
                     float* state = ss.gdn_state + (size_t) (gdn_index - ss.gdn_ord0) * gdn_floats;
                     float* conv = state + (uint64_t) g.ssm_state_size * g.ssm_v_heads * g.ssm_state_size;
+                    static int64_t pfh_chunk = -1;
+                    const bool pfh_on = pf_hash_file() != nullptr && gdn_index == 0;
+                    if (pfh_on) ++pfh_chunk;
+                    auto pfh = [&](const char* nm, const void* dev, int64_t rows, size_t rowbytes) {
+                        if (!pfh_on) return;
+                        m.cs->wait();
+                        std::vector<char> hb((size_t) rows * rowbytes);
+                        m.cs->memcpy(hb.data(), dev, hb.size()).wait();
+                        for (int64_t r = 0; r < rows; ++r)
+                            std::fprintf(pf_hash_file(), "%lld %s %lld %016llx\n", (long long) pfh_chunk, nm, (long long) r,
+                                         (unsigned long long) pf_hash_fnv(hb.data() + (size_t) r * rowbytes, rowbytes));
+                        std::fflush(pf_hash_file());
+                    };
+                    pfh("mixed", m.mixed, T, N * 4);
                     if (!native_proj(m.gemm, wqkv, m.mixed_h, m.qkv, T, v.name("attn_qkv.weight"), err)) return false;
+                    pfh("qkv", m.qkv, T, C * 4);
                     if (!native_proj(m.gemm, wg, m.mixed_h, m.z, T, v.name("attn_gate.weight"), err)) return false;
+                    pfh("z", m.z, T, ZV * 4);
                     if (!bf16_proj(m.gemm, wa, m.mixed_bf, m.ab, T, v.name("ssm_alpha.weight"), err, 2 * HV, m.mixed_bf_lo)) return false;
                     if (!bf16_proj(m.gemm, wb, m.mixed_bf, m.ab + HV, T, v.name("ssm_beta.weight"), err, 2 * HV, m.mixed_bf_lo)) return false;
+                    pfh("ab", m.ab, T, 2 * HV * 4);
                     pt.mark(kPfGdnConv, cs);   // "gdn" is the projections in; the rest on their own lines
                     gdn_gates(m.ab, (const float*) wdt->data, (const float*) wsa->data, m.gate, m.beta, T, m.cs);
                     gdn_conv(conv, m.qkv, (const float*) wc->data, m.hbuf, T, EPS, m.cs);
+                    pfh("gate", m.gate, T, HV * 4);
+                    pfh("beta", m.beta, T, HV * 4);
+                    pfh("hbuf", m.hbuf, T, C * 4);
                     pt.mark(kPfGdnRec, cs);
                     const int64_t ld_y = pf_pad() && T >= std::max<int64_t>(pf_switch_min_t(), 64) ? ZV + ZV_PAD : 0;
                     gdn_recurrence(state, m.hbuf, m.gate, m.beta, m.z, (const float*) wnm->data, EPS, m.y, m.y_h, T, m.cs,
                                    ld_y);
+                    pfh("y", m.y, T, ZV * 4);
+                    pfh("state_rec", state, HV, (size_t) g.ssm_state_size * g.ssm_state_size * 4);
                     pt.mark(kPfGdnOut, cs);
                     if (!native_proj(m.gemm, wo, m.y_h, m.bo, T, v.name("ssm_out.weight"), err, 0, ld_y)) return false;
                     ++gdn_index;