feat(ext): add optional text hidden-state encoding to text runners (#1073)

Co-authored-by: D3rPapts <[email protected]>

include/gufo/ext/changes/0.2/text-hidden-state-encoding.md

@@ -0,0 +1,16 @@
+# Text runner hidden-state encoding
+
+Text (`src/ext/text.hpp`): add `TextHiddenStateEncoding`,
+`TextHiddenStateRequest`, `TextHiddenStates`, `TextHiddenStateStatus` and the
+optional `TextModelRunner::HiddenStateEncoding()` /
+`TextModelRunner::EncodeHiddenStates(...)` pair, so another extension can use a
+text model created through `FindTextProvider(...)->create` as a frozen encoder.
+A call encodes one complete, caller-framed prompt from position zero,
+independent of every session state, and returns owned row-major FP32
+`[tokens, width]` states in token order: the residual stream after the last
+decoder layer, before the final output norm, which consumers apply exactly
+once. `HiddenStateEncoding()` returning `std::nullopt` means unsupported, and
+`EncodeHiddenStates` then throws `std::logic_error`; execution outcomes are
+reported as `kOk`, `kCancelled`, `kInvalidInput` or `kFailed`. The Qwen text
+runner implements it on a private executor. Additive; existing runners keep
+the unsupported defaults.

src/ext/text.hpp

@@ -72,6 +72,52 @@ struct TextPreparedPrompt {
   std::size_t cache_prefix_tokens{0};
 };
 
+/// Optional hidden-state encoding, which lets another model use a text
+/// runner's backbone as a frozen encoder (for example, an image model's text
+/// conditioning). Discovered through `TextModelRunner::HiddenStateEncoding()`.
+struct TextHiddenStateEncoding {
+  /// Scalars per row: the model's hidden size.
+  std::size_t width{0};
+  /// Most tokens one `EncodeHiddenStates` call accepts.
+  std::size_t max_tokens{0};
+};
+
+/// One complete prompt, framed and tokenized by the caller.
+struct TextHiddenStateRequest {
+  /// Encoded as a whole from position zero with the model's causal attention.
+  std::span<const TextRunnerToken> tokens;
+  /// Polled during execution; returning true stops the call with
+  /// `TextHiddenStateStatus::kCancelled`.
+  std::function<bool()> is_cancelled;
+};
+
+/// Hidden states of one request. `values` holds `rows * width` FP32 scalars
+/// in row-major order; row `i` belongs to request token `i`, so rows follow
+/// token order and `rows` equals the request's token count.
+///
+/// Each row is the residual stream after the last decoder layer, before the
+/// model's final output norm. Runners apply no normalization: a consumer that
+/// conditions on normalized states applies that final norm exactly once.
+///
+/// The result owns its storage; it never aliases runner memory and remains
+/// valid after the runner is destroyed.
+struct TextHiddenStates {
+  std::size_t rows{0};
+  std::size_t width{0};
+  std::vector<float> values;
+};
+
+enum class TextHiddenStateStatus : std::uint8_t {
+  kOk,
+  /// `is_cancelled` returned true; no states are returned.
+  kCancelled,
+  /// Empty prompt, more than `max_tokens` tokens, or a token outside the
+  /// model's vocabulary. Nothing was executed.
+  kInvalidInput,
+  /// Execution failed. A later call may succeed.
+  kFailed,
+};
+
 struct TextRunnerRamCacheOptions {
   static constexpr std::size_t kAutomaticMaxBytes = std::size_t{32} << 30;
   std::size_t capacity_bytes{0};
@@ -428,6 +474,31 @@ public:
   [[nodiscard]] virtual std::string Decode(
       std::span<const TextRunnerToken> tokens) const = 0;
 
+  /// The runner's hidden-state encoder, or std::nullopt when it offers none.
+  [[nodiscard]] virtual std::optional<TextHiddenStateEncoding>
+  HiddenStateEncoding() const {
+    return std::nullopt;
+  }
+  /// Encodes `request.tokens` as one complete prompt (see `TextHiddenStates`
+  /// for the returned layout and normalization). Every call starts from an
+  /// empty context and is independent of all `TextRunnerState`s: it neither
+  /// reads nor changes sessions, prefix caches or snapshots, so repeating a
+  /// request returns the same states. Concurrent calls on one runner
+  /// serialize. Any working memory belongs to the runner and is released with
+  /// it. `states` must not be null. On `kOk`, `*states` holds the result;
+  /// otherwise it is empty and `*error` (when not null) explains why. Throws
+  /// std::logic_error when
+  /// `HiddenStateEncoding()` is std::nullopt, so an unsupported capability is
+  /// never reported as an execution failure.
+  [[nodiscard]] virtual TextHiddenStateStatus EncodeHiddenStates(
+      const TextHiddenStateRequest& request, TextHiddenStates* states,
+      std::string* error) const {
+    (void)request;
+    (void)states;
+    (void)error;
+    throw std::logic_error("runner does not support hidden-state encoding");
+  }
+
   /// Continuation contract. The cache owns every execution slot: it creates
   /// the resources once, binds a state to each slot, and reconciles a state
   /// after restoring rows into its slot at the given position.

src/models/qwen/runner.cpp

@@ -9,6 +9,7 @@
 #include <cstddef>
 #include <cstdint>
 #include <memory>
+#include <mutex>
 #include <optional>
 #include <span>
 #include <stdexcept>
@@ -839,6 +840,83 @@ public:
     return model_->GetTokenizer().Decode(tokens);
   }
 
+  [[nodiscard]] std::optional<TextHiddenStateEncoding> HiddenStateEncoding()
+      const override {
+    return TextHiddenStateEncoding{
+        .width = model_->GetConfig().hidden_size,
+        .max_tokens = max_context_,
+    };
+  }
+
+  // Runs on a private executor, never on a component-cache slot, so sessions,
+  // prefix caches and snapshots are untouched. The capture is the last
+  // decoder layer's residual output before output_norm (prefill_chunk.cpp).
+  [[nodiscard]] TextHiddenStateStatus EncodeHiddenStates(
+      const TextHiddenStateRequest& request, TextHiddenStates* states,
+      std::string* error) const override {
+    const auto finish = [&](TextHiddenStateStatus status, std::string message) {
+      if (states != nullptr)
+        *states = {};
+      if (error != nullptr)
+        *error = std::move(message);
+      return status;
+    };
+    if (states == nullptr)
+      throw std::invalid_argument("hidden-state encoding needs an output");
+    const auto tokens = request.tokens;
+    if (tokens.empty())
+      return finish(TextHiddenStateStatus::kInvalidInput,
+                    "hidden-state encoding needs at least one token");
+    if (tokens.size() > max_context_)
+      return finish(TextHiddenStateStatus::kInvalidInput,
+                    "prompt exceeds the hidden-state encoding limit");
+    const std::size_t vocabulary = model_->GetConfig().vocab_size;
+    if (std::ranges::any_of(tokens, [vocabulary](TextRunnerToken token) {
+          return token >= vocabulary;
+        }))
+      return finish(TextHiddenStateStatus::kInvalidInput,
+                    "prompt token is outside the model vocabulary");
+
+    std::lock_guard lock(hidden_state_mutex_);
+    try {
+      auto& executor = HiddenStateExecutor(tokens.size());
+      // Leave the executor empty and unarmed however the call ends.
+      struct Release {
+        hip::QwenGpuExecutor& executor;
+        ~Release() {
+          executor.SetCancellationCheck({});
+          try {
+            executor.SetPromptHiddenCapture(false);
+          } catch (...) {
+          }
+          executor.Reset();
+        }
+      } release{executor};
+      executor.Reset();
+      executor.SetCancellationCheck(request.is_cancelled);
+      executor.SetPromptHiddenCapture(true);
+      (void)executor.ForwardPromptBatch(tokens, 0, false);
+      const std::size_t width = model_->GetConfig().hidden_size;
+      const auto hidden = executor.GetPromptHiddenStates();
+      if (hidden.size() != tokens.size() * width)
+        throw std::logic_error("Qwen hidden-state capture size mismatch");
+      states->rows = tokens.size();
+      states->width = width;
+      states->values.assign(hidden.begin(), hidden.end());
+    } catch (const hip::QwenPrefillCancelled&) {
+      return finish(TextHiddenStateStatus::kCancelled,
+                    "hidden-state encoding cancelled");
+    } catch (const std::exception& e) {
+      // A failed device pass may leave the executor unusable; the next call
+      // builds a fresh one.
+      hidden_state_executor_.reset();
+      return finish(TextHiddenStateStatus::kFailed, e.what());
+    }
+    if (error != nullptr)
+      error->clear();
+    return TextHiddenStateStatus::kOk;
+  }
+
   void PreparePrefixReuse(
       TextRunnerState& state,
       std::span<const TextRunnerToken> prefix) const override {
@@ -1056,6 +1134,25 @@ public:
   }
 
 private:
+  // Shares the loaded weights; only the attention/recurrent state is private.
+  // Sized to the longest prompt encoded so far in 1,024-token steps, so a
+  // runner with a long serving context does not reserve a second full-length
+  // KV cache for short encoder prompts.
+  [[nodiscard]] hip::QwenGpuExecutor& HiddenStateExecutor(
+      std::size_t tokens) const {
+    constexpr std::size_t kStep = 1024;
+    if (hidden_state_executor_ == nullptr ||
+        hidden_state_executor_->GetMaxContext() < tokens) {
+      const auto context = static_cast<std::uint32_t>(std::min<std::size_t>(
+          max_context_,
+          std::max<std::size_t>(2, (tokens + kStep - 1) / kStep * kStep)));
+      hidden_state_executor_.reset();
+      hidden_state_executor_ = std::make_unique<hip::QwenGpuExecutor>(
+          model_, context, execution_policy_);
+    }
+    return *hidden_state_executor_;
+  }
+
   [[nodiscard]] std::size_t MaximumDecodeBatchWidth() const noexcept {
     const std::size_t rows_per_session = dflash_model_ != nullptr ? 8 : 1;
     return std::clamp<std::size_t>(max_context_ / rows_per_session, 1, 8);
@@ -1073,6 +1170,8 @@ private:
   std::uint32_t copy_rows_{static_cast<std::uint32_t>(
       hip::QwenGpuExecutor::kMaxWideVerificationRows)};
   cache::Identity component_identity_;
+  mutable std::mutex hidden_state_mutex_;
+  mutable std::unique_ptr<hip::QwenGpuExecutor> hidden_state_executor_;
 };
 
 }  // namespace

tests/extensions/CMakeLists.txt

@@ -17,19 +17,24 @@ if(NOT GUFO_EXTENSION_FIXTURE)
       "-DOPENSSL_CRYPTO_LIBRARY=${OPENSSL_CRYPTO_LIBRARY}"
     BUILD_COMMAND ${CMAKE_COMMAND} --build <BINARY_DIR> --target
       extension_runtime_test extension_role_test
-      ext_runtime_neutral_contracts_test
+      ext_runtime_neutral_contracts_test ext_text_hidden_states_contract_test
     INSTALL_COMMAND ""
     BUILD_ALWAYS TRUE)
   add_custom_target(extension_runtime_test DEPENDS extension_fixture_build)
   add_custom_target(extension_role_test DEPENDS extension_fixture_build)
   add_custom_target(ext_runtime_neutral_contracts_test
     DEPENDS extension_fixture_build)
+  add_custom_target(ext_text_hidden_states_contract_test
+    DEPENDS extension_fixture_build)
   add_test(NAME extension_runtime_test COMMAND "${fixture_build}/extensions/extension_runtime_test${CMAKE_EXECUTABLE_SUFFIX}")
   add_test(NAME extension_role_test COMMAND "${fixture_build}/extensions/extension_role_test${CMAKE_EXECUTABLE_SUFFIX}")
   add_test(NAME ext_runtime_neutral_contracts_test
     COMMAND "${fixture_build}/extensions/ext_runtime_neutral_contracts_test${CMAKE_EXECUTABLE_SUFFIX}")
+  add_test(NAME ext_text_hidden_states_contract_test
+    COMMAND "${fixture_build}/extensions/ext_text_hidden_states_contract_test${CMAKE_EXECUTABLE_SUFFIX}")
   set_tests_properties(extension_runtime_test extension_role_test
-    ext_runtime_neutral_contracts_test PROPERTIES LABELS "cpu;extensions")
+    ext_runtime_neutral_contracts_test ext_text_hidden_states_contract_test
+    PROPERTIES LABELS "cpu;extensions")
   # The fixture project has no gufo_core, so the implementation behind the public
   # services links here. Like an external target, this one drops the directory's
   # repository include path.
@@ -98,6 +103,17 @@ add_test(NAME ext_runtime_neutral_contracts_test
 set_tests_properties(ext_runtime_neutral_contracts_test PROPERTIES
   LABELS "cpu;extensions")
 
+# An image extension using a registered text model as its encoder sees only
+# this public contract; see tests/models/qwen/hip for the Qwen numerics.
+add_executable(ext_text_hidden_states_contract_test
+  ext_text_hidden_states_contract_test.cpp)
+target_link_libraries(ext_text_hidden_states_contract_test PRIVATE
+  gufo_ext_runtime)
+add_test(NAME ext_text_hidden_states_contract_test
+  COMMAND ext_text_hidden_states_contract_test)
+set_tests_properties(ext_text_hidden_states_contract_test PROPERTIES
+  LABELS "cpu;extensions")
+
 add_executable(extension_runtime_test runtime_test.cpp)
 # This in-tree integration test also exercises the private CLI logging facade.
 # The public runtime deliberately no longer exports the repository root.

tests/extensions/ext_text_hidden_states_contract_test.cpp

@@ -0,0 +1,237 @@
+// Text hidden-state encoding contract, as an external consumer sees it.
+//
+// Links only `gufo_ext_runtime` and includes only <gufo/ext/...>, like an
+// image extension that uses a registered text model as its frozen encoder.
+// Numerical evidence for the Qwen adapter lives in
+// tests/models/qwen/hip/text_hidden_states_test.cpp.
+
+#include <cstddef>
+#include <cstdio>
+#include <functional>
+#include <gufo/ext/text.hpp>
+#include <memory>
+#include <optional>
+#include <span>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <utility>
+#include <vector>
+
+namespace {
+
+using gufo::ext::TextHiddenStateEncoding;
+using gufo::ext::TextHiddenStateRequest;
+using gufo::ext::TextHiddenStates;
+using gufo::ext::TextHiddenStateStatus;
+using gufo::ext::TextRunnerToken;
+
+int failures = 0;
+
+void Expect(bool condition, const char* what) {
+  if (!condition) {
+    ++failures;
+    std::fprintf(stderr, "FAILED: %s\n", what);
+  }
+}
+
+// The smallest complete text runner: every required operation and nothing
+// else, so the hidden-state methods keep their defaults.
+class PlainRunner : public gufo::ext::TextModelRunner {
+public:
+  [[nodiscard]] gufo::ext::TextRunnerDescriptor Descriptor() const override {
+    gufo::ext::TextRunnerDescriptor descriptor;
+    descriptor.model_id = "plain";
+    descriptor.state_abi = "plain";
+    descriptor.max_context = 8;
+    return descriptor;
+  }
+  [[nodiscard]] gufo::ext::TextRunnerResourceClaim ResourceClaim()
+      const override {
+    return {};
+  }
+  [[nodiscard]] std::vector<gufo::ext::TextExecutionPlan> SupportedPlans()
+      const override {
+    return {{}};
+  }
+  [[nodiscard]] std::vector<TextRunnerToken> Tokenize(
+      std::string_view text) const override {
+    return std::vector<TextRunnerToken>(text.size(), 1);
+  }
+  [[nodiscard]] std::optional<std::vector<TextRunnerToken>> RenderAndTokenize(
+      const gufo::ext::ChatRequest&) const override {
+    return std::nullopt;
+  }
+  [[nodiscard]] std::string Decode(
+      std::span<const TextRunnerToken>) const override {
+    return {};
+  }
+  [[nodiscard]] std::unique_ptr<gufo::ext::ComponentCacheResources>
+  CreateComponentCacheResources(gufo::cache::ResourceLedger&,
+                                std::size_t) const override {
+    throw std::logic_error("not used");
+  }
+  [[nodiscard]] std::unique_ptr<gufo::ext::TextRunnerState> BindComponentState(
+      gufo::cache::Adapter&, gufo::cache::Slot&) const override {
+    throw std::logic_error("not used");
+  }
+  void ReconcileComponentState(gufo::ext::TextRunnerState&,
+                               std::size_t) const override {
+    throw std::logic_error("not used");
+  }
+  [[nodiscard]] gufo::ext::TextPrefillStep Prefill(
+      gufo::ext::TextRunnerState&, std::span<const TextRunnerToken>,
+      std::size_t, std::size_t) const override {
+    throw std::logic_error("not used");
+  }
+  [[nodiscard]] gufo::ext::TextDecodeSelection SelectNext(
+      gufo::ext::TextRunnerState&,
+      gufo::sampling::SamplerState&) const override {
+    throw std::logic_error("not used");
+  }
+  void Advance(gufo::ext::TextRunnerState&, TextRunnerToken) const override {
+    throw std::logic_error("not used");
+  }
+  [[nodiscard]] std::size_t CheckpointPosition(
+      const gufo::ext::TextRunnerState&) const override {
+    return 0;
+  }
+};
+
+// A conforming encoder with a known answer: row i, column c holds
+// token_i * 100 + c, so order and layout are observable.
+class EncoderRunner final : public PlainRunner {
+public:
+  static constexpr std::size_t kWidth = 3;
+  static constexpr std::size_t kMaxTokens = 4;
+  static constexpr TextRunnerToken kVocabulary = 50;
+
+  [[nodiscard]] std::optional<TextHiddenStateEncoding> HiddenStateEncoding()
+      const override {
+    return TextHiddenStateEncoding{.width = kWidth, .max_tokens = kMaxTokens};
+  }
+  [[nodiscard]] TextHiddenStateStatus EncodeHiddenStates(
+      const TextHiddenStateRequest& request, TextHiddenStates* states,
+      std::string* error) const override {
+    *states = {};
+    const auto reject = [&](TextHiddenStateStatus status, const char* why) {
+      if (error != nullptr)
+        *error = why;
+      return status;
+    };
+    if (request.tokens.empty() || request.tokens.size() > kMaxTokens)
+      return reject(TextHiddenStateStatus::kInvalidInput, "token count");
+    for (const auto token : request.tokens) {
+      if (token >= kVocabulary)
+        return reject(TextHiddenStateStatus::kInvalidInput, "vocabulary");
+    }
+    if (request.is_cancelled && request.is_cancelled())
+      return reject(TextHiddenStateStatus::kCancelled, "cancelled");
+    states->rows = request.tokens.size();
+    states->width = kWidth;
+    for (const auto token : request.tokens) {
+      for (std::size_t column = 0; column < kWidth; ++column)
+        states->values.push_back(static_cast<float>(token * 100 + column));
+    }
+    return TextHiddenStateStatus::kOk;
+  }
+};
+
+std::shared_ptr<gufo::ext::TextModelRunner> CreateEncoder(
+    const gufo::ext::TextRunnerOptions&, std::string*) {
+  return std::make_shared<EncoderRunner>();
+}
+
+void CheckUnsupportedIsNotAFailure() {
+  const PlainRunner runner;
+  Expect(!runner.HiddenStateEncoding().has_value(),
+         "runners do not offer hidden-state encoding by default");
+  const std::vector<TextRunnerToken> tokens{1, 2};
+  TextHiddenStates states;
+  bool unsupported = false;
+  try {
+    (void)runner.EncodeHiddenStates({.tokens = tokens, .is_cancelled = {}},
+                                    &states, nullptr);
+  } catch (const std::logic_error&) {
+    unsupported = true;
+  }
+  Expect(unsupported,
+         "encoding on an unsupported runner throws instead of returning a "
+         "failure status");
+}
+
+void CheckLayoutOrderAndOwnership() {
+  gufo::ext::RegisterTextProvider(
+      {.id = "test-hidden-encoder", .create = &CreateEncoder});
+  const auto* provider = gufo::ext::FindTextProvider("test-hidden-encoder");
+  Expect(provider != nullptr && provider->create != nullptr,
+         "an encoder is created through the public provider factory");
+  if (provider == nullptr || provider->create == nullptr)
+    return;
+  std::string error;
+  auto runner = provider->create({}, &error);
+  const auto encoding = runner->HiddenStateEncoding();
+  Expect(encoding.has_value() && encoding->width == EncoderRunner::kWidth &&
+             encoding->max_tokens == EncoderRunner::kMaxTokens,
+         "the capability reports width and prompt limit before encoding");
+
+  const std::vector<TextRunnerToken> tokens{7, 3, 9};
+  TextHiddenStates states;
+  Expect(
+      runner->EncodeHiddenStates({.tokens = tokens, .is_cancelled = {}},
+                                 &states, &error) == TextHiddenStateStatus::kOk,
+      "a valid prompt encodes");
+  // The result must stay usable after its producer is gone.
+  runner.reset();
+  Expect(states.rows == tokens.size() &&
+             states.width == EncoderRunner::kWidth &&
+             states.values.size() == states.rows * states.width,
+         "states are rows x width");
+  bool ordered = true;
+  for (std::size_t row = 0; row < states.rows; ++row) {
+    for (std::size_t column = 0; column < states.width; ++column) {
+      ordered = ordered && states.values[row * states.width + column] ==
+                               static_cast<float>(tokens[row] * 100 + column);
+    }
+  }
+  Expect(ordered, "row i is token i, stored row-major");
+  const TextHiddenStates copy = states;
+  Expect(copy.values == states.values && copy.values.data() != nullptr &&
+             copy.values.data() != states.values.data(),
+         "states are an owned value");
+}
+
+void CheckInvalidInputAndCancellation() {
+  const EncoderRunner runner;
+  TextHiddenStates states{.rows = 1, .width = 1, .values = {1.0F}};
+  std::string error;
+  Expect(runner.EncodeHiddenStates({}, &states, &error) ==
+                 TextHiddenStateStatus::kInvalidInput &&
+             states.values.empty() && !error.empty(),
+         "an empty prompt is invalid input and leaves no states");
+  const std::vector<TextRunnerToken> too_long(EncoderRunner::kMaxTokens + 1, 1);
+  Expect(runner.EncodeHiddenStates({.tokens = too_long, .is_cancelled = {}},
+                                   &states, nullptr) ==
+             TextHiddenStateStatus::kInvalidInput,
+         "a prompt above max_tokens is invalid input");
+  const std::vector<TextRunnerToken> outside{EncoderRunner::kVocabulary};
+  Expect(runner.EncodeHiddenStates({.tokens = outside, .is_cancelled = {}},
+                                   &states, nullptr) ==
+             TextHiddenStateStatus::kInvalidInput,
+         "a token outside the vocabulary is invalid input");
+  const std::vector<TextRunnerToken> tokens{1};
+  Expect(runner.EncodeHiddenStates(
+             {.tokens = tokens, .is_cancelled = [] { return true; }}, &states,
+             nullptr) == TextHiddenStateStatus::kCancelled &&
+             states.values.empty(),
+         "cancellation is its own status and leaves no states");
+}
+
+}  // namespace
+
+int main() {
+  CheckUnsupportedIsNotAFailure();
+  CheckLayoutOrderAndOwnership();
+  CheckInvalidInputAndCancellation();
+  return failures == 0 ? 0 : 1;
+}

tests/models/qwen/CMakeLists.txt

@@ -85,6 +85,11 @@ if (ENGINE_ENABLE_HIP)
   add_qwen_hip_test(qwen_prompt_hidden_buffer_test
     tests/models/qwen/hip/prompt_hidden_buffer_test.cpp
     "hip;gpu;qwen")
+  # TextModelRunner::EncodeHiddenStates on GUFO_QWEN_HIDDEN_MODEL, optionally
+  # against tools/qwen27b/hidden_state_reference.py output.
+  add_qwen_hip_test(qwen_text_hidden_states_test
+    tests/models/qwen/hip/text_hidden_states_test.cpp
+    "hip;gpu;qwen;external-model")
 
   add_qwen_hip_test(qwen_gpu_ops_test
     tests/models/qwen/hip/basic/blas_ops_test.cpp

tests/models/qwen/hip/text_hidden_states_test.cpp

@@ -0,0 +1,345 @@
+// Qwen hidden-state encoding (`TextModelRunner::EncodeHiddenStates`) on a
+// real checkpoint.
+//
+// GUFO_QWEN_HIDDEN_MODEL names a qwen35 GGUF (MOSS-Image uses its Qwen3.5-9B
+// text encoder); without it the test skips with 77. It checks shape, repeated
+// positive/empty-prompt isolation, cancellation, invalid input and that a
+// concurrent session on the same weights is unaffected.
+//
+// GUFO_QWEN_HIDDEN_REFERENCE optionally names a file written by
+// tools/qwen27b/hidden_state_reference.py from the same checkpoint's
+// unquantized Hugging Face weights. The adapter's rows are then compared with
+// the reference's pre-norm rows, and the final norm, applied once to the
+// adapter's rows, with the reference's normalized rows. That pins the
+// documented semantics: no normalization inside the runner, exactly one in
+// the consumer. GUFO_QWEN_HIDDEN_MIN_COSINE overrides the per-row cosine
+// floor (default 0.99, chosen for BF16/F16 checkpoints).
+
+#include <algorithm>
+#include <array>
+#include <cmath>
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+#include <fstream>
+#include <functional>
+#include <iostream>
+#include <memory>
+#include <span>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <utility>
+#include <vector>
+
+#include "src/core/gguf_reader.hpp"
+#include "src/core/speculative/speculative_verifier.hpp"
+#include "src/ext/text.hpp"
+#include "src/models/qwen/hip/executor.hpp"
+#include "src/models/qwen/runner.hpp"
+
+namespace {
+
+using gufo::ext::TextHiddenStates;
+using gufo::ext::TextHiddenStateStatus;
+using gufo::ext::TextRunnerToken;
+
+constexpr std::uint32_t kContext = 4096;
+constexpr std::size_t kDecodeSteps = 4;
+
+void Expect(bool condition, std::string_view message) {
+  if (!condition)
+    throw std::runtime_error(std::string(message));
+}
+
+// MOSS-Image framing; enable_thinking=True leaves the think block open.
+std::string Frame(std::string_view prompt) {
+  return "<|im_start|>user\n" + std::string(prompt) +
+         "<|im_end|>\n<|im_start|>assistant\n<think>\n";
+}
+
+struct ReferenceCase {
+  std::vector<TextRunnerToken> tokens;
+  std::vector<float> pre_norm;
+  std::vector<float> normalized;
+};
+
+struct Reference {
+  std::size_t width{0};
+  float epsilon{0};
+  std::vector<float> scale;
+  std::vector<ReferenceCase> cases;
+};
+
+template<typename T>
+void Read(std::ifstream& in, T* values, std::size_t count) {
+  in.read(reinterpret_cast<char*>(values),
+          static_cast<std::streamsize>(count * sizeof(T)));
+  Expect(static_cast<bool>(in), "hidden-state reference is truncated");
+}
+
+// Layout written by tools/qwen27b/hidden_state_reference.py, little endian:
+// "GQHS", u32 version 1, u32 cases, u32 width, f32 epsilon, f32 scale[width],
+// then per case u32 rows, u32 tokens[rows], f32 pre_norm[rows * width],
+// f32 normalized[rows * width].
+Reference LoadReference(const char* path) {
+  std::ifstream in(path, std::ios::binary);
+  Expect(static_cast<bool>(in), "cannot open the hidden-state reference");
+  std::array<char, 4> magic{};
+  Read(in, magic.data(), magic.size());
+  Expect(std::string_view(magic.data(), magic.size()) == "GQHS",
+         "not a Gufo Qwen hidden-state reference");
+  std::uint32_t header[3]{};
+  Read(in, header, 3);
+  Expect(header[0] == 1, "unsupported hidden-state reference version");
+  Reference reference;
+  reference.width = header[2];
+  Read(in, &reference.epsilon, 1);
+  reference.scale.resize(reference.width);
+  Read(in, reference.scale.data(), reference.width);
+  for (std::uint32_t c = 0; c < header[1]; ++c) {
+    std::uint32_t rows = 0;
+    Read(in, &rows, 1);
+    ReferenceCase item;
+    item.tokens.resize(rows);
+    Read(in, item.tokens.data(), rows);
+    item.pre_norm.resize(std::size_t{rows} * reference.width);
+    Read(in, item.pre_norm.data(), item.pre_norm.size());
+    item.normalized.resize(item.pre_norm.size());
+    Read(in, item.normalized.data(), item.normalized.size());
+    reference.cases.push_back(std::move(item));
+  }
+  return reference;
+}
+
+TextHiddenStates Encode(const gufo::ext::TextModelRunner& runner,
+                        std::span<const TextRunnerToken> tokens) {
+  TextHiddenStates states;
+  std::string error;
+  const auto status = runner.EncodeHiddenStates(
+      {.tokens = tokens, .is_cancelled = {}}, &states, &error);
+  Expect(status == TextHiddenStateStatus::kOk, "encoding failed: " + error);
+  Expect(states.rows == tokens.size() &&
+             states.values.size() == states.rows * states.width,
+         "encoded states are not rows x width");
+  Expect(std::ranges::all_of(states.values,
+                             [](float v) { return std::isfinite(v); }),
+         "encoded states must be finite");
+  return states;
+}
+
+bool SameBytes(std::span<const float> a, std::span<const float> b) {
+  return a.size() == b.size() &&
+         (a.empty() || std::memcmp(a.data(), b.data(), a.size_bytes()) == 0);
+}
+
+std::vector<float> Logits(gufo::hip::QwenGpuExecutor& executor) {
+  const auto view = executor.CopyLastLogits();
+  return {view.begin(), view.end()};
+}
+
+// Greedy prefill plus a few decode steps on a session executor, optionally
+// encoding on the runner between prefill and decode.
+std::vector<std::vector<float>> RunSession(
+    gufo::hip::QwenGpuExecutor& session,
+    std::span<const TextRunnerToken> prompt,
+    const std::function<void()>& between) {
+  std::vector<std::vector<float>> logits;
+  session.Reset();
+  auto token = session.ForwardPromptBatch(prompt, 0, true);
+  logits.push_back(Logits(session));
+  if (between)
+    between();
+  for (std::size_t step = 0; step < kDecodeSteps; ++step) {
+    token = session.ForwardToken(
+        token, static_cast<std::uint32_t>(prompt.size() + step));
+    logits.push_back(Logits(session));
+  }
+  return logits;
+}
+
+double Cosine(std::span<const float> a, std::span<const float> b) {
+  double dot = 0, aa = 0, bb = 0;
+  for (std::size_t i = 0; i < a.size(); ++i) {
+    dot += double{a[i]} * b[i];
+    aa += double{a[i]} * a[i];
+    bb += double{b[i]} * b[i];
+  }
+  return dot / std::sqrt(aa * bb);
+}
+
+double Rms(std::span<const float> a) {
+  double sum = 0;
+  for (const float v : a)
+    sum += double{v} * v;
+  return std::sqrt(sum / static_cast<double>(a.size()));
+}
+
+std::vector<float> NormalizeOnce(std::span<const float> row,
+                                 const Reference& reference) {
+  const double inverse =
+      1.0 / std::sqrt(Rms(row) * Rms(row) + reference.epsilon);
+  std::vector<float> out(row.size());
+  for (std::size_t i = 0; i < row.size(); ++i)
+    out[i] = static_cast<float>(row[i] * inverse * reference.scale[i]);
+  return out;
+}
+
+void CompareWithReference(const gufo::ext::TextModelRunner& runner,
+                          const Reference& reference, std::size_t width) {
+  Expect(reference.width == width,
+         "hidden-state reference width does not match the GGUF");
+  const char* floor_text = std::getenv("GUFO_QWEN_HIDDEN_MIN_COSINE");
+  const double floor = floor_text != nullptr ? std::atof(floor_text) : 0.99;
+  for (std::size_t c = 0; c < reference.cases.size(); ++c) {
+    const auto& item = reference.cases[c];
+    const auto states = Encode(runner, item.tokens);
+    double min_pre = 1, min_once = 1, min_twice = 1, max_rms_ratio = 0;
+    double min_rms_ratio = 1e30, min_unnormalized = 1;
+    for (std::size_t row = 0; row < states.rows; ++row) {
+      const auto at = [&](std::span<const float> values) {
+        return values.subspan(row * width, width);
+      };
+      const auto ours = at(states.values);
+      const auto pre = at(item.pre_norm);
+      const auto normalized = at(item.normalized);
+      const auto once = NormalizeOnce(ours, reference);
+      const auto twice = NormalizeOnce(once, reference);
+      min_pre = std::min(min_pre, Cosine(ours, pre));
+      min_once = std::min(min_once, Cosine(once, normalized));
+      min_twice = std::min(min_twice, Cosine(twice, normalized));
+      min_unnormalized = std::min(min_unnormalized, Cosine(ours, normalized));
+      const double ratio = Rms(ours) / Rms(pre);
+      min_rms_ratio = std::min(min_rms_ratio, ratio);
+      max_rms_ratio = std::max(max_rms_ratio, ratio);
+    }
+    std::cout << "reference case=" << c << " rows=" << states.rows
+              << " min_cos_pre_norm=" << min_pre
+              << " min_cos_norm_once=" << min_once
+              << " min_cos_norm_twice=" << min_twice
+              << " min_cos_raw_vs_normalized=" << min_unnormalized
+              << " rms_ratio=[" << min_rms_ratio << ", " << max_rms_ratio
+              << "]\n";
+    Expect(min_pre >= floor, "adapter rows differ from the pre-norm reference");
+    Expect(min_once >= floor,
+           "adapter rows normalized once differ from the normalized "
+           "reference");
+    // Unnormalized rows keep the residual stream's magnitude; a runner that
+    // already applied the norm would land near the normalized scale instead.
+    Expect(min_rms_ratio > 0.9 && max_rms_ratio < 1.1,
+           "adapter rows are not on the pre-norm scale");
+  }
+}
+
+void Run(const char* model_path) {
+  std::string error;
+  auto owner = gufo::core::GgufReader::OpenFile(model_path, &error);
+  Expect(owner != nullptr, error);
+  auto model = gufo::hip::QwenGpuModel::CreateFromGguf(
+      std::shared_ptr<const gufo::core::GgufReader>(std::move(owner)), &error);
+  Expect(model != nullptr, error);
+  const std::size_t width = model->GetConfig().hidden_size;
+  const auto runner = gufo::models::qwen::CreateQwenTextRunner(
+      model, kContext, nullptr, gufo::speculative::SpeculativeOptions{},
+      gufo::speculative::DFlashDraftPolicy::kAdaptive, {}, {});
+
+  const auto encoding = runner->HiddenStateEncoding();
+  Expect(encoding.has_value() && encoding->width == width &&
+             encoding->max_tokens == kContext,
+         "Qwen runner must report its hidden-state width and limit");
+
+  const auto positive = runner->Tokenize(
+      Frame("A red fox reading a newspaper on a park bench, watercolor."));
+  const auto empty = runner->Tokenize(Frame(""));
+  const auto a = Encode(*runner, positive);
+  const auto e = Encode(*runner, empty);
+  Expect(a.width == width && e.width == width, "row width is hidden_size");
+  Expect(SameBytes(Encode(*runner, positive).values, a.values) &&
+             SameBytes(Encode(*runner, empty).values, e.values) &&
+             SameBytes(Encode(*runner, positive).values, a.values),
+         "repeated positive/empty encoding must be bit-identical");
+
+  // A session on the same weights: encoding between its prefill and decode
+  // must not change a single logit.
+  gufo::hip::QwenGpuExecutor session(model, kContext);
+  const auto baseline = RunSession(session, positive, {});
+  const auto interleaved = RunSession(session, positive, [&] {
+    (void)Encode(*runner, empty);
+    (void)Encode(*runner, positive);
+  });
+  Expect(baseline.size() == interleaved.size(), "session step count");
+  for (std::size_t step = 0; step < baseline.size(); ++step)
+    Expect(SameBytes(baseline[step], interleaved[step]),
+           "hidden-state encoding changed a concurrent session's logits");
+  Expect(SameBytes(Encode(*runner, positive).values, a.values),
+         "session activity changed hidden-state encoding");
+
+  // Past the first 1,024-token step the private executor is rebuilt larger
+  // and the capture spans several prefill chunks; neither may change results.
+  std::vector<TextRunnerToken> long_prompt;
+  while (long_prompt.size() < 2500)
+    long_prompt.insert(long_prompt.end(), positive.begin(), positive.end());
+  const auto long_states = Encode(*runner, long_prompt);
+  Expect(SameBytes(Encode(*runner, positive).values, a.values) &&
+             SameBytes(Encode(*runner, long_prompt).values, long_states.values),
+         "a larger private executor changed hidden-state encoding");
+
+  TextHiddenStates states;
+  Expect(runner->EncodeHiddenStates(
+             {.tokens = positive, .is_cancelled = [] { return true; }}, &states,
+             &error) == TextHiddenStateStatus::kCancelled &&
+             states.values.empty(),
+         "cancelled encoding reports kCancelled without states");
+  int polls = 0;
+  const auto status = runner->EncodeHiddenStates(
+      {.tokens = positive, .is_cancelled = [&polls] { return ++polls > 3; }},
+      &states, &error);
+  Expect(status == TextHiddenStateStatus::kCancelled ||
+             status == TextHiddenStateStatus::kOk,
+         "mid-pass cancellation either stops or completes");
+  Expect(SameBytes(Encode(*runner, positive).values, a.values),
+         "encoding after cancellation must be unaffected");
+
+  Expect(runner->EncodeHiddenStates({}, &states, &error) ==
+             TextHiddenStateStatus::kInvalidInput,
+         "an empty prompt is invalid input");
+  const std::vector<TextRunnerToken> outside{
+      static_cast<TextRunnerToken>(model->GetConfig().vocab_size)};
+  Expect(runner->EncodeHiddenStates({.tokens = outside, .is_cancelled = {}},
+                                    &states, &error) ==
+             TextHiddenStateStatus::kInvalidInput,
+         "a token outside the vocabulary is invalid input");
+  const std::vector<TextRunnerToken> too_long(kContext + 1, positive.front());
+  Expect(runner->EncodeHiddenStates({.tokens = too_long, .is_cancelled = {}},
+                                    &states, &error) ==
+             TextHiddenStateStatus::kInvalidInput,
+         "a prompt above max_tokens is invalid input");
+
+  if (const char* path = std::getenv("GUFO_QWEN_HIDDEN_REFERENCE"))
+    CompareWithReference(*runner, LoadReference(path), width);
+  else
+    std::cout << "GUFO_QWEN_HIDDEN_REFERENCE unset: reference comparison "
+                 "skipped\n";
+  std::cout << "positive rows=" << a.rows << " empty rows=" << e.rows
+            << " width=" << width << '\n';
+}
+
+}  // namespace
+
+int main(int argc, const char* const* argv) {
+  try {
+    const char* model =
+        argc > 1 ? argv[1] : std::getenv("GUFO_QWEN_HIDDEN_MODEL");
+    if (model == nullptr) {
+      std::cout << "Set GUFO_QWEN_HIDDEN_MODEL for the Qwen hidden-state "
+                   "check.\n";
+      return 77;
+    }
+    Run(model);
+    std::cout << "Qwen hidden-state checks passed.\n";
+    return 0;
+  } catch (const std::exception& error) {
+    std::cerr << error.what() << '\n';
+    return 1;
+  }
+}

tools/README.md

@@ -80,5 +80,10 @@ It replaces the old standalone W8A8 kernel copies.
 dispatch (exact outputs, weights rotated beyond the MALL cache); build the
 `qwen27b_verify_sweep` target and run `sweep.py`, see its README.
 `qwen27b/dflash_reference.py` checks a GPU trace against
-pinned upstream PyTorch operators using the same GGUF weights. See
+pinned upstream PyTorch operators using the same GGUF weights.
+`qwen27b/hidden_state_reference.py` records unquantized Hugging Face pre- and
+post-norm last-layer states for `qwen_text_hidden_states_test`, which checks
+`TextModelRunner::EncodeHiddenStates` (`GUFO_QWEN_HIDDEN_REFERENCE`);
+`--device cuda --dtype bfloat16` loads the checkpoint onto the GPU instead of
+host RAM (needs `accelerate`). See
 `docs/models/qwen3.8-27b/BENCHMARKS.md` and its quality report for commands and evidence.

tools/qwen27b/hidden_state_reference.py

@@ -0,0 +1,116 @@
+#!/usr/bin/env python3
+"""Write an independent reference for TextModelRunner::EncodeHiddenStates.
+
+Runs an unquantized Hugging Face Qwen3.5/3.8 checkpoint with transformers on
+fixed, MOSS-Image-framed prompts and records per token:
+
+- the last decoder layer's output (the residual stream before the final norm),
+- the same rows after the checkpoint's final norm, and
+- the final norm's effective per-channel scale and epsilon.
+
+tests/models/qwen/hip/text_hidden_states_test.cpp (GUFO_QWEN_HIDDEN_REFERENCE)
+compares Gufo's Qwen adapter with the first and, after applying the norm once
+itself, with the second. Token IDs are stored, so both sides encode identical
+inputs. No download is implicit: pass local directories. The default float32
+CPU run needs about four bytes of RAM per parameter; --device cuda (ROCm or
+CUDA) loads the weights straight onto the GPU, normally with --dtype bfloat16,
+without staging them in host RAM. Recorded rows are float32 either way.
+"""
+from __future__ import annotations
+
+import argparse
+import math
+import struct
+from pathlib import Path
+
+import numpy as np
+import torch
+from transformers import AutoModel, AutoTokenizer
+
+MAGIC = b"GQHS"
+VERSION = 1
+# MOSS-Image framing; enable_thinking=True leaves the think block open.
+FRAME = "<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n<think>\n"
+PROMPTS = ("A red fox reading a newspaper on a park bench, watercolor.", "")
+
+
+def text_backbone(model):
+    """The decoder owning embed_tokens, layers and the final norm."""
+    for name, module in model.named_modules():
+        if all(hasattr(module, attr) for attr in ("embed_tokens", "layers", "norm")):
+            return name, module
+    raise SystemExit("no decoder with embed_tokens, layers and norm found")
+
+
+def norm_epsilon(norm) -> float:
+    for attr in ("variance_epsilon", "eps"):
+        value = getattr(norm, attr, None)
+        if value is not None:
+            return float(value)
+    raise SystemExit("cannot read the final norm epsilon")
+
+
+@torch.no_grad()
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("checkpoint", type=Path,
+                        help="Hugging Face model directory (for MOSS-Image: text_encoder)")
+    parser.add_argument("output", type=Path)
+    parser.add_argument("--tokenizer", type=Path,
+                        help="tokenizer directory (default: the checkpoint)")
+    parser.add_argument("--dtype", choices=("float32", "bfloat16"), default="float32")
+    parser.add_argument("--device", default="cpu",
+                        help="torch device, e.g. cuda (also ROCm) to load the weights "
+                             "directly onto the GPU (default: cpu); needs accelerate")
+    args = parser.parse_args()
+
+    tokenizer = AutoTokenizer.from_pretrained(args.tokenizer or args.checkpoint)
+    # A device map streams each tensor straight to its device instead of
+    # materializing the whole model in host RAM first.
+    placement = ({} if torch.device(args.device).type == "cpu" else
+                 {"low_cpu_mem_usage": True, "device_map": {"": args.device}})
+    model = AutoModel.from_pretrained(args.checkpoint, torch_dtype=getattr(torch, args.dtype),
+                                      **placement)
+    model.eval()
+    name, backbone = text_backbone(model)
+    norm = backbone.norm
+    epsilon = norm_epsilon(norm)
+    width = backbone.embed_tokens.embedding_dim
+    # For x = 1 an RMS norm returns scale / sqrt(1 + eps), whatever its weight
+    # parameterization (plain or zero-centered).
+    ones = torch.ones(1, 1, width, dtype=norm.weight.dtype, device=norm.weight.device)
+    scale = (norm(ones).float() * math.sqrt(1.0 + epsilon)).reshape(width).cpu()
+
+    captured = []
+    hook = backbone.layers[-1].register_forward_hook(
+        lambda _module, _inputs, output: captured.append(
+            output[0] if isinstance(output, tuple) else output))
+    cases = []
+    for prompt in PROMPTS:
+        ids = tokenizer(FRAME.format(prompt), add_special_tokens=False,
+                        return_tensors="pt").input_ids
+        captured.clear()
+        backbone(input_ids=ids.to(backbone.embed_tokens.weight.device), use_cache=False)
+        if len(captured) != 1:
+            raise SystemExit(f"expected one last-layer output, got {len(captured)}")
+        pre = captured[0]
+        cases.append((ids.reshape(-1).numpy().astype("<u4"),
+                      pre.float().reshape(-1, width).cpu().numpy().astype("<f4"),
+                      norm(pre).float().reshape(-1, width).cpu().numpy().astype("<f4")))
+    hook.remove()
+
+    with args.output.open("wb") as out:
+        out.write(MAGIC)
+        out.write(struct.pack("<IIIf", VERSION, len(cases), width, epsilon))
+        out.write(scale.numpy().astype("<f4").tobytes())
+        for ids, pre, normalized in cases:
+            out.write(struct.pack("<I", ids.size))
+            out.write(ids.tobytes())
+            out.write(np.ascontiguousarray(pre).tobytes())
+            out.write(np.ascontiguousarray(normalized).tobytes())
+    print(f"{args.output}: decoder={name or '<root>'} width={width} "
+          f"epsilon={epsilon} cases={[int(ids.size) for ids, _, _ in cases]}")
+
+
+if __name__ == "__main__":
+    main()