From aac3f843f9348dcf474e8bc4c126dbcb3a0e9d75 Mon Sep 17 00:00:00 2001 From: Federico Izzo Date: Sat, 7 Nov 2026 13:48:34 +0100 Subject: [PATCH] feat(ext): add optional text hidden-state encoding to text runners (#1073) Co-authored-by: D3rPapts <138111743+D3rPapts@users.noreply.github.com> --- .../changes/0.2/text-hidden-state-encoding.md | 16 + src/ext/text.hpp | 71 ++++ src/models/qwen/runner.cpp | 99 +++++ tests/extensions/CMakeLists.txt | 20 +- .../ext_text_hidden_states_contract_test.cpp | 237 ++++++++++++ tests/models/qwen/CMakeLists.txt | 5 + .../qwen/hip/text_hidden_states_test.cpp | 345 ++++++++++++++++++ tools/README.md | 7 +- tools/qwen27b/hidden_state_reference.py | 116 ++++++ 9 files changed, 913 insertions(+), 3 deletions(-) create mode 100644 include/gufo/ext/changes/0.2/text-hidden-state-encoding.md create mode 100644 tests/extensions/ext_text_hidden_states_contract_test.cpp create mode 100644 tests/models/qwen/hip/text_hidden_states_test.cpp create mode 100644 tools/qwen27b/hidden_state_reference.py diff --git a/include/gufo/ext/changes/0.2/text-hidden-state-encoding.md b/include/gufo/ext/changes/0.2/text-hidden-state-encoding.md new file mode 100644 index 00000000..b8d47c69 --- /dev/null +++ b/include/gufo/ext/changes/0.2/text-hidden-state-encoding.md @@ -0,0 +1,16 @@ +# Text runner hidden-state encoding + +Text (`src/ext/text.hpp`): add `TextHiddenStateEncoding`, +`TextHiddenStateRequest`, `TextHiddenStates`, `TextHiddenStateStatus` and the +optional `TextModelRunner::HiddenStateEncoding()` / +`TextModelRunner::EncodeHiddenStates(...)` pair, so another extension can use a +text model created through `FindTextProvider(...)->create` as a frozen encoder. +A call encodes one complete, caller-framed prompt from position zero, +independent of every session state, and returns owned row-major FP32 +`[tokens, width]` states in token order: the residual stream after the last +decoder layer, before the final output norm, which consumers apply exactly +once. `HiddenStateEncoding()` returning `std::nullopt` means unsupported, and +`EncodeHiddenStates` then throws `std::logic_error`; execution outcomes are +reported as `kOk`, `kCancelled`, `kInvalidInput` or `kFailed`. The Qwen text +runner implements it on a private executor. Additive; existing runners keep +the unsupported defaults. diff --git a/src/ext/text.hpp b/src/ext/text.hpp index d383168f..988d774c 100644 --- a/src/ext/text.hpp +++ b/src/ext/text.hpp @@ -72,6 +72,52 @@ struct TextPreparedPrompt { std::size_t cache_prefix_tokens{0}; }; +/// Optional hidden-state encoding, which lets another model use a text +/// runner's backbone as a frozen encoder (for example, an image model's text +/// conditioning). Discovered through `TextModelRunner::HiddenStateEncoding()`. +struct TextHiddenStateEncoding { + /// Scalars per row: the model's hidden size. + std::size_t width{0}; + /// Most tokens one `EncodeHiddenStates` call accepts. + std::size_t max_tokens{0}; +}; + +/// One complete prompt, framed and tokenized by the caller. +struct TextHiddenStateRequest { + /// Encoded as a whole from position zero with the model's causal attention. + std::span tokens; + /// Polled during execution; returning true stops the call with + /// `TextHiddenStateStatus::kCancelled`. + std::function is_cancelled; +}; + +/// Hidden states of one request. `values` holds `rows * width` FP32 scalars +/// in row-major order; row `i` belongs to request token `i`, so rows follow +/// token order and `rows` equals the request's token count. +/// +/// Each row is the residual stream after the last decoder layer, before the +/// model's final output norm. Runners apply no normalization: a consumer that +/// conditions on normalized states applies that final norm exactly once. +/// +/// The result owns its storage; it never aliases runner memory and remains +/// valid after the runner is destroyed. +struct TextHiddenStates { + std::size_t rows{0}; + std::size_t width{0}; + std::vector values; +}; + +enum class TextHiddenStateStatus : std::uint8_t { + kOk, + /// `is_cancelled` returned true; no states are returned. + kCancelled, + /// Empty prompt, more than `max_tokens` tokens, or a token outside the + /// model's vocabulary. Nothing was executed. + kInvalidInput, + /// Execution failed. A later call may succeed. + kFailed, +}; + struct TextRunnerRamCacheOptions { static constexpr std::size_t kAutomaticMaxBytes = std::size_t{32} << 30; std::size_t capacity_bytes{0}; @@ -428,6 +474,31 @@ public: [[nodiscard]] virtual std::string Decode( std::span tokens) const = 0; + /// The runner's hidden-state encoder, or std::nullopt when it offers none. + [[nodiscard]] virtual std::optional + HiddenStateEncoding() const { + return std::nullopt; + } + /// Encodes `request.tokens` as one complete prompt (see `TextHiddenStates` + /// for the returned layout and normalization). Every call starts from an + /// empty context and is independent of all `TextRunnerState`s: it neither + /// reads nor changes sessions, prefix caches or snapshots, so repeating a + /// request returns the same states. Concurrent calls on one runner + /// serialize. Any working memory belongs to the runner and is released with + /// it. `states` must not be null. On `kOk`, `*states` holds the result; + /// otherwise it is empty and `*error` (when not null) explains why. Throws + /// std::logic_error when + /// `HiddenStateEncoding()` is std::nullopt, so an unsupported capability is + /// never reported as an execution failure. + [[nodiscard]] virtual TextHiddenStateStatus EncodeHiddenStates( + const TextHiddenStateRequest& request, TextHiddenStates* states, + std::string* error) const { + (void)request; + (void)states; + (void)error; + throw std::logic_error("runner does not support hidden-state encoding"); + } + /// Continuation contract. The cache owns every execution slot: it creates /// the resources once, binds a state to each slot, and reconciles a state /// after restoring rows into its slot at the given position. diff --git a/src/models/qwen/runner.cpp b/src/models/qwen/runner.cpp index 4ee83f76..0de4f3ac 100644 --- a/src/models/qwen/runner.cpp +++ b/src/models/qwen/runner.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -839,6 +840,83 @@ public: return model_->GetTokenizer().Decode(tokens); } + [[nodiscard]] std::optional HiddenStateEncoding() + const override { + return TextHiddenStateEncoding{ + .width = model_->GetConfig().hidden_size, + .max_tokens = max_context_, + }; + } + + // Runs on a private executor, never on a component-cache slot, so sessions, + // prefix caches and snapshots are untouched. The capture is the last + // decoder layer's residual output before output_norm (prefill_chunk.cpp). + [[nodiscard]] TextHiddenStateStatus EncodeHiddenStates( + const TextHiddenStateRequest& request, TextHiddenStates* states, + std::string* error) const override { + const auto finish = [&](TextHiddenStateStatus status, std::string message) { + if (states != nullptr) + *states = {}; + if (error != nullptr) + *error = std::move(message); + return status; + }; + if (states == nullptr) + throw std::invalid_argument("hidden-state encoding needs an output"); + const auto tokens = request.tokens; + if (tokens.empty()) + return finish(TextHiddenStateStatus::kInvalidInput, + "hidden-state encoding needs at least one token"); + if (tokens.size() > max_context_) + return finish(TextHiddenStateStatus::kInvalidInput, + "prompt exceeds the hidden-state encoding limit"); + const std::size_t vocabulary = model_->GetConfig().vocab_size; + if (std::ranges::any_of(tokens, [vocabulary](TextRunnerToken token) { + return token >= vocabulary; + })) + return finish(TextHiddenStateStatus::kInvalidInput, + "prompt token is outside the model vocabulary"); + + std::lock_guard lock(hidden_state_mutex_); + try { + auto& executor = HiddenStateExecutor(tokens.size()); + // Leave the executor empty and unarmed however the call ends. + struct Release { + hip::QwenGpuExecutor& executor; + ~Release() { + executor.SetCancellationCheck({}); + try { + executor.SetPromptHiddenCapture(false); + } catch (...) { + } + executor.Reset(); + } + } release{executor}; + executor.Reset(); + executor.SetCancellationCheck(request.is_cancelled); + executor.SetPromptHiddenCapture(true); + (void)executor.ForwardPromptBatch(tokens, 0, false); + const std::size_t width = model_->GetConfig().hidden_size; + const auto hidden = executor.GetPromptHiddenStates(); + if (hidden.size() != tokens.size() * width) + throw std::logic_error("Qwen hidden-state capture size mismatch"); + states->rows = tokens.size(); + states->width = width; + states->values.assign(hidden.begin(), hidden.end()); + } catch (const hip::QwenPrefillCancelled&) { + return finish(TextHiddenStateStatus::kCancelled, + "hidden-state encoding cancelled"); + } catch (const std::exception& e) { + // A failed device pass may leave the executor unusable; the next call + // builds a fresh one. + hidden_state_executor_.reset(); + return finish(TextHiddenStateStatus::kFailed, e.what()); + } + if (error != nullptr) + error->clear(); + return TextHiddenStateStatus::kOk; + } + void PreparePrefixReuse( TextRunnerState& state, std::span prefix) const override { @@ -1056,6 +1134,25 @@ public: } private: + // Shares the loaded weights; only the attention/recurrent state is private. + // Sized to the longest prompt encoded so far in 1,024-token steps, so a + // runner with a long serving context does not reserve a second full-length + // KV cache for short encoder prompts. + [[nodiscard]] hip::QwenGpuExecutor& HiddenStateExecutor( + std::size_t tokens) const { + constexpr std::size_t kStep = 1024; + if (hidden_state_executor_ == nullptr || + hidden_state_executor_->GetMaxContext() < tokens) { + const auto context = static_cast(std::min( + max_context_, + std::max(2, (tokens + kStep - 1) / kStep * kStep))); + hidden_state_executor_.reset(); + hidden_state_executor_ = std::make_unique( + model_, context, execution_policy_); + } + return *hidden_state_executor_; + } + [[nodiscard]] std::size_t MaximumDecodeBatchWidth() const noexcept { const std::size_t rows_per_session = dflash_model_ != nullptr ? 8 : 1; return std::clamp(max_context_ / rows_per_session, 1, 8); @@ -1073,6 +1170,8 @@ private: std::uint32_t copy_rows_{static_cast( hip::QwenGpuExecutor::kMaxWideVerificationRows)}; cache::Identity component_identity_; + mutable std::mutex hidden_state_mutex_; + mutable std::unique_ptr hidden_state_executor_; }; } // namespace diff --git a/tests/extensions/CMakeLists.txt b/tests/extensions/CMakeLists.txt index f6947158..7a70c63b 100644 --- a/tests/extensions/CMakeLists.txt +++ b/tests/extensions/CMakeLists.txt @@ -17,19 +17,24 @@ if(NOT GUFO_EXTENSION_FIXTURE) "-DOPENSSL_CRYPTO_LIBRARY=${OPENSSL_CRYPTO_LIBRARY}" BUILD_COMMAND ${CMAKE_COMMAND} --build --target extension_runtime_test extension_role_test - ext_runtime_neutral_contracts_test + ext_runtime_neutral_contracts_test ext_text_hidden_states_contract_test INSTALL_COMMAND "" BUILD_ALWAYS TRUE) add_custom_target(extension_runtime_test DEPENDS extension_fixture_build) add_custom_target(extension_role_test DEPENDS extension_fixture_build) add_custom_target(ext_runtime_neutral_contracts_test DEPENDS extension_fixture_build) + add_custom_target(ext_text_hidden_states_contract_test + DEPENDS extension_fixture_build) add_test(NAME extension_runtime_test COMMAND "${fixture_build}/extensions/extension_runtime_test${CMAKE_EXECUTABLE_SUFFIX}") add_test(NAME extension_role_test COMMAND "${fixture_build}/extensions/extension_role_test${CMAKE_EXECUTABLE_SUFFIX}") add_test(NAME ext_runtime_neutral_contracts_test COMMAND "${fixture_build}/extensions/ext_runtime_neutral_contracts_test${CMAKE_EXECUTABLE_SUFFIX}") + add_test(NAME ext_text_hidden_states_contract_test + COMMAND "${fixture_build}/extensions/ext_text_hidden_states_contract_test${CMAKE_EXECUTABLE_SUFFIX}") set_tests_properties(extension_runtime_test extension_role_test - ext_runtime_neutral_contracts_test PROPERTIES LABELS "cpu;extensions") + ext_runtime_neutral_contracts_test ext_text_hidden_states_contract_test + PROPERTIES LABELS "cpu;extensions") # The fixture project has no gufo_core, so the implementation behind the public # services links here. Like an external target, this one drops the directory's # repository include path. @@ -98,6 +103,17 @@ add_test(NAME ext_runtime_neutral_contracts_test set_tests_properties(ext_runtime_neutral_contracts_test PROPERTIES LABELS "cpu;extensions") +# An image extension using a registered text model as its encoder sees only +# this public contract; see tests/models/qwen/hip for the Qwen numerics. +add_executable(ext_text_hidden_states_contract_test + ext_text_hidden_states_contract_test.cpp) +target_link_libraries(ext_text_hidden_states_contract_test PRIVATE + gufo_ext_runtime) +add_test(NAME ext_text_hidden_states_contract_test + COMMAND ext_text_hidden_states_contract_test) +set_tests_properties(ext_text_hidden_states_contract_test PROPERTIES + LABELS "cpu;extensions") + add_executable(extension_runtime_test runtime_test.cpp) # This in-tree integration test also exercises the private CLI logging facade. # The public runtime deliberately no longer exports the repository root. diff --git a/tests/extensions/ext_text_hidden_states_contract_test.cpp b/tests/extensions/ext_text_hidden_states_contract_test.cpp new file mode 100644 index 00000000..712062ea --- /dev/null +++ b/tests/extensions/ext_text_hidden_states_contract_test.cpp @@ -0,0 +1,237 @@ +// Text hidden-state encoding contract, as an external consumer sees it. +// +// Links only `gufo_ext_runtime` and includes only , like an +// image extension that uses a registered text model as its frozen encoder. +// Numerical evidence for the Qwen adapter lives in +// tests/models/qwen/hip/text_hidden_states_test.cpp. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using gufo::ext::TextHiddenStateEncoding; +using gufo::ext::TextHiddenStateRequest; +using gufo::ext::TextHiddenStates; +using gufo::ext::TextHiddenStateStatus; +using gufo::ext::TextRunnerToken; + +int failures = 0; + +void Expect(bool condition, const char* what) { + if (!condition) { + ++failures; + std::fprintf(stderr, "FAILED: %s\n", what); + } +} + +// The smallest complete text runner: every required operation and nothing +// else, so the hidden-state methods keep their defaults. +class PlainRunner : public gufo::ext::TextModelRunner { +public: + [[nodiscard]] gufo::ext::TextRunnerDescriptor Descriptor() const override { + gufo::ext::TextRunnerDescriptor descriptor; + descriptor.model_id = "plain"; + descriptor.state_abi = "plain"; + descriptor.max_context = 8; + return descriptor; + } + [[nodiscard]] gufo::ext::TextRunnerResourceClaim ResourceClaim() + const override { + return {}; + } + [[nodiscard]] std::vector SupportedPlans() + const override { + return {{}}; + } + [[nodiscard]] std::vector Tokenize( + std::string_view text) const override { + return std::vector(text.size(), 1); + } + [[nodiscard]] std::optional> RenderAndTokenize( + const gufo::ext::ChatRequest&) const override { + return std::nullopt; + } + [[nodiscard]] std::string Decode( + std::span) const override { + return {}; + } + [[nodiscard]] std::unique_ptr + CreateComponentCacheResources(gufo::cache::ResourceLedger&, + std::size_t) const override { + throw std::logic_error("not used"); + } + [[nodiscard]] std::unique_ptr BindComponentState( + gufo::cache::Adapter&, gufo::cache::Slot&) const override { + throw std::logic_error("not used"); + } + void ReconcileComponentState(gufo::ext::TextRunnerState&, + std::size_t) const override { + throw std::logic_error("not used"); + } + [[nodiscard]] gufo::ext::TextPrefillStep Prefill( + gufo::ext::TextRunnerState&, std::span, + std::size_t, std::size_t) const override { + throw std::logic_error("not used"); + } + [[nodiscard]] gufo::ext::TextDecodeSelection SelectNext( + gufo::ext::TextRunnerState&, + gufo::sampling::SamplerState&) const override { + throw std::logic_error("not used"); + } + void Advance(gufo::ext::TextRunnerState&, TextRunnerToken) const override { + throw std::logic_error("not used"); + } + [[nodiscard]] std::size_t CheckpointPosition( + const gufo::ext::TextRunnerState&) const override { + return 0; + } +}; + +// A conforming encoder with a known answer: row i, column c holds +// token_i * 100 + c, so order and layout are observable. +class EncoderRunner final : public PlainRunner { +public: + static constexpr std::size_t kWidth = 3; + static constexpr std::size_t kMaxTokens = 4; + static constexpr TextRunnerToken kVocabulary = 50; + + [[nodiscard]] std::optional HiddenStateEncoding() + const override { + return TextHiddenStateEncoding{.width = kWidth, .max_tokens = kMaxTokens}; + } + [[nodiscard]] TextHiddenStateStatus EncodeHiddenStates( + const TextHiddenStateRequest& request, TextHiddenStates* states, + std::string* error) const override { + *states = {}; + const auto reject = [&](TextHiddenStateStatus status, const char* why) { + if (error != nullptr) + *error = why; + return status; + }; + if (request.tokens.empty() || request.tokens.size() > kMaxTokens) + return reject(TextHiddenStateStatus::kInvalidInput, "token count"); + for (const auto token : request.tokens) { + if (token >= kVocabulary) + return reject(TextHiddenStateStatus::kInvalidInput, "vocabulary"); + } + if (request.is_cancelled && request.is_cancelled()) + return reject(TextHiddenStateStatus::kCancelled, "cancelled"); + states->rows = request.tokens.size(); + states->width = kWidth; + for (const auto token : request.tokens) { + for (std::size_t column = 0; column < kWidth; ++column) + states->values.push_back(static_cast(token * 100 + column)); + } + return TextHiddenStateStatus::kOk; + } +}; + +std::shared_ptr CreateEncoder( + const gufo::ext::TextRunnerOptions&, std::string*) { + return std::make_shared(); +} + +void CheckUnsupportedIsNotAFailure() { + const PlainRunner runner; + Expect(!runner.HiddenStateEncoding().has_value(), + "runners do not offer hidden-state encoding by default"); + const std::vector tokens{1, 2}; + TextHiddenStates states; + bool unsupported = false; + try { + (void)runner.EncodeHiddenStates({.tokens = tokens, .is_cancelled = {}}, + &states, nullptr); + } catch (const std::logic_error&) { + unsupported = true; + } + Expect(unsupported, + "encoding on an unsupported runner throws instead of returning a " + "failure status"); +} + +void CheckLayoutOrderAndOwnership() { + gufo::ext::RegisterTextProvider( + {.id = "test-hidden-encoder", .create = &CreateEncoder}); + const auto* provider = gufo::ext::FindTextProvider("test-hidden-encoder"); + Expect(provider != nullptr && provider->create != nullptr, + "an encoder is created through the public provider factory"); + if (provider == nullptr || provider->create == nullptr) + return; + std::string error; + auto runner = provider->create({}, &error); + const auto encoding = runner->HiddenStateEncoding(); + Expect(encoding.has_value() && encoding->width == EncoderRunner::kWidth && + encoding->max_tokens == EncoderRunner::kMaxTokens, + "the capability reports width and prompt limit before encoding"); + + const std::vector tokens{7, 3, 9}; + TextHiddenStates states; + Expect( + runner->EncodeHiddenStates({.tokens = tokens, .is_cancelled = {}}, + &states, &error) == TextHiddenStateStatus::kOk, + "a valid prompt encodes"); + // The result must stay usable after its producer is gone. + runner.reset(); + Expect(states.rows == tokens.size() && + states.width == EncoderRunner::kWidth && + states.values.size() == states.rows * states.width, + "states are rows x width"); + bool ordered = true; + for (std::size_t row = 0; row < states.rows; ++row) { + for (std::size_t column = 0; column < states.width; ++column) { + ordered = ordered && states.values[row * states.width + column] == + static_cast(tokens[row] * 100 + column); + } + } + Expect(ordered, "row i is token i, stored row-major"); + const TextHiddenStates copy = states; + Expect(copy.values == states.values && copy.values.data() != nullptr && + copy.values.data() != states.values.data(), + "states are an owned value"); +} + +void CheckInvalidInputAndCancellation() { + const EncoderRunner runner; + TextHiddenStates states{.rows = 1, .width = 1, .values = {1.0F}}; + std::string error; + Expect(runner.EncodeHiddenStates({}, &states, &error) == + TextHiddenStateStatus::kInvalidInput && + states.values.empty() && !error.empty(), + "an empty prompt is invalid input and leaves no states"); + const std::vector too_long(EncoderRunner::kMaxTokens + 1, 1); + Expect(runner.EncodeHiddenStates({.tokens = too_long, .is_cancelled = {}}, + &states, nullptr) == + TextHiddenStateStatus::kInvalidInput, + "a prompt above max_tokens is invalid input"); + const std::vector outside{EncoderRunner::kVocabulary}; + Expect(runner.EncodeHiddenStates({.tokens = outside, .is_cancelled = {}}, + &states, nullptr) == + TextHiddenStateStatus::kInvalidInput, + "a token outside the vocabulary is invalid input"); + const std::vector tokens{1}; + Expect(runner.EncodeHiddenStates( + {.tokens = tokens, .is_cancelled = [] { return true; }}, &states, + nullptr) == TextHiddenStateStatus::kCancelled && + states.values.empty(), + "cancellation is its own status and leaves no states"); +} + +} // namespace + +int main() { + CheckUnsupportedIsNotAFailure(); + CheckLayoutOrderAndOwnership(); + CheckInvalidInputAndCancellation(); + return failures == 0 ? 0 : 1; +} diff --git a/tests/models/qwen/CMakeLists.txt b/tests/models/qwen/CMakeLists.txt index 1b650c2f..75aed581 100644 --- a/tests/models/qwen/CMakeLists.txt +++ b/tests/models/qwen/CMakeLists.txt @@ -85,6 +85,11 @@ if (ENGINE_ENABLE_HIP) add_qwen_hip_test(qwen_prompt_hidden_buffer_test tests/models/qwen/hip/prompt_hidden_buffer_test.cpp "hip;gpu;qwen") + # TextModelRunner::EncodeHiddenStates on GUFO_QWEN_HIDDEN_MODEL, optionally + # against tools/qwen27b/hidden_state_reference.py output. + add_qwen_hip_test(qwen_text_hidden_states_test + tests/models/qwen/hip/text_hidden_states_test.cpp + "hip;gpu;qwen;external-model") add_qwen_hip_test(qwen_gpu_ops_test tests/models/qwen/hip/basic/blas_ops_test.cpp diff --git a/tests/models/qwen/hip/text_hidden_states_test.cpp b/tests/models/qwen/hip/text_hidden_states_test.cpp new file mode 100644 index 00000000..e6e77ea2 --- /dev/null +++ b/tests/models/qwen/hip/text_hidden_states_test.cpp @@ -0,0 +1,345 @@ +// Qwen hidden-state encoding (`TextModelRunner::EncodeHiddenStates`) on a +// real checkpoint. +// +// GUFO_QWEN_HIDDEN_MODEL names a qwen35 GGUF (MOSS-Image uses its Qwen3.5-9B +// text encoder); without it the test skips with 77. It checks shape, repeated +// positive/empty-prompt isolation, cancellation, invalid input and that a +// concurrent session on the same weights is unaffected. +// +// GUFO_QWEN_HIDDEN_REFERENCE optionally names a file written by +// tools/qwen27b/hidden_state_reference.py from the same checkpoint's +// unquantized Hugging Face weights. The adapter's rows are then compared with +// the reference's pre-norm rows, and the final norm, applied once to the +// adapter's rows, with the reference's normalized rows. That pins the +// documented semantics: no normalization inside the runner, exactly one in +// the consumer. GUFO_QWEN_HIDDEN_MIN_COSINE overrides the per-row cosine +// floor (default 0.99, chosen for BF16/F16 checkpoints). + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "src/core/gguf_reader.hpp" +#include "src/core/speculative/speculative_verifier.hpp" +#include "src/ext/text.hpp" +#include "src/models/qwen/hip/executor.hpp" +#include "src/models/qwen/runner.hpp" + +namespace { + +using gufo::ext::TextHiddenStates; +using gufo::ext::TextHiddenStateStatus; +using gufo::ext::TextRunnerToken; + +constexpr std::uint32_t kContext = 4096; +constexpr std::size_t kDecodeSteps = 4; + +void Expect(bool condition, std::string_view message) { + if (!condition) + throw std::runtime_error(std::string(message)); +} + +// MOSS-Image framing; enable_thinking=True leaves the think block open. +std::string Frame(std::string_view prompt) { + return "<|im_start|>user\n" + std::string(prompt) + + "<|im_end|>\n<|im_start|>assistant\n\n"; +} + +struct ReferenceCase { + std::vector tokens; + std::vector pre_norm; + std::vector normalized; +}; + +struct Reference { + std::size_t width{0}; + float epsilon{0}; + std::vector scale; + std::vector cases; +}; + +template +void Read(std::ifstream& in, T* values, std::size_t count) { + in.read(reinterpret_cast(values), + static_cast(count * sizeof(T))); + Expect(static_cast(in), "hidden-state reference is truncated"); +} + +// Layout written by tools/qwen27b/hidden_state_reference.py, little endian: +// "GQHS", u32 version 1, u32 cases, u32 width, f32 epsilon, f32 scale[width], +// then per case u32 rows, u32 tokens[rows], f32 pre_norm[rows * width], +// f32 normalized[rows * width]. +Reference LoadReference(const char* path) { + std::ifstream in(path, std::ios::binary); + Expect(static_cast(in), "cannot open the hidden-state reference"); + std::array magic{}; + Read(in, magic.data(), magic.size()); + Expect(std::string_view(magic.data(), magic.size()) == "GQHS", + "not a Gufo Qwen hidden-state reference"); + std::uint32_t header[3]{}; + Read(in, header, 3); + Expect(header[0] == 1, "unsupported hidden-state reference version"); + Reference reference; + reference.width = header[2]; + Read(in, &reference.epsilon, 1); + reference.scale.resize(reference.width); + Read(in, reference.scale.data(), reference.width); + for (std::uint32_t c = 0; c < header[1]; ++c) { + std::uint32_t rows = 0; + Read(in, &rows, 1); + ReferenceCase item; + item.tokens.resize(rows); + Read(in, item.tokens.data(), rows); + item.pre_norm.resize(std::size_t{rows} * reference.width); + Read(in, item.pre_norm.data(), item.pre_norm.size()); + item.normalized.resize(item.pre_norm.size()); + Read(in, item.normalized.data(), item.normalized.size()); + reference.cases.push_back(std::move(item)); + } + return reference; +} + +TextHiddenStates Encode(const gufo::ext::TextModelRunner& runner, + std::span tokens) { + TextHiddenStates states; + std::string error; + const auto status = runner.EncodeHiddenStates( + {.tokens = tokens, .is_cancelled = {}}, &states, &error); + Expect(status == TextHiddenStateStatus::kOk, "encoding failed: " + error); + Expect(states.rows == tokens.size() && + states.values.size() == states.rows * states.width, + "encoded states are not rows x width"); + Expect(std::ranges::all_of(states.values, + [](float v) { return std::isfinite(v); }), + "encoded states must be finite"); + return states; +} + +bool SameBytes(std::span a, std::span b) { + return a.size() == b.size() && + (a.empty() || std::memcmp(a.data(), b.data(), a.size_bytes()) == 0); +} + +std::vector Logits(gufo::hip::QwenGpuExecutor& executor) { + const auto view = executor.CopyLastLogits(); + return {view.begin(), view.end()}; +} + +// Greedy prefill plus a few decode steps on a session executor, optionally +// encoding on the runner between prefill and decode. +std::vector> RunSession( + gufo::hip::QwenGpuExecutor& session, + std::span prompt, + const std::function& between) { + std::vector> logits; + session.Reset(); + auto token = session.ForwardPromptBatch(prompt, 0, true); + logits.push_back(Logits(session)); + if (between) + between(); + for (std::size_t step = 0; step < kDecodeSteps; ++step) { + token = session.ForwardToken( + token, static_cast(prompt.size() + step)); + logits.push_back(Logits(session)); + } + return logits; +} + +double Cosine(std::span a, std::span b) { + double dot = 0, aa = 0, bb = 0; + for (std::size_t i = 0; i < a.size(); ++i) { + dot += double{a[i]} * b[i]; + aa += double{a[i]} * a[i]; + bb += double{b[i]} * b[i]; + } + return dot / std::sqrt(aa * bb); +} + +double Rms(std::span a) { + double sum = 0; + for (const float v : a) + sum += double{v} * v; + return std::sqrt(sum / static_cast(a.size())); +} + +std::vector NormalizeOnce(std::span row, + const Reference& reference) { + const double inverse = + 1.0 / std::sqrt(Rms(row) * Rms(row) + reference.epsilon); + std::vector out(row.size()); + for (std::size_t i = 0; i < row.size(); ++i) + out[i] = static_cast(row[i] * inverse * reference.scale[i]); + return out; +} + +void CompareWithReference(const gufo::ext::TextModelRunner& runner, + const Reference& reference, std::size_t width) { + Expect(reference.width == width, + "hidden-state reference width does not match the GGUF"); + const char* floor_text = std::getenv("GUFO_QWEN_HIDDEN_MIN_COSINE"); + const double floor = floor_text != nullptr ? std::atof(floor_text) : 0.99; + for (std::size_t c = 0; c < reference.cases.size(); ++c) { + const auto& item = reference.cases[c]; + const auto states = Encode(runner, item.tokens); + double min_pre = 1, min_once = 1, min_twice = 1, max_rms_ratio = 0; + double min_rms_ratio = 1e30, min_unnormalized = 1; + for (std::size_t row = 0; row < states.rows; ++row) { + const auto at = [&](std::span values) { + return values.subspan(row * width, width); + }; + const auto ours = at(states.values); + const auto pre = at(item.pre_norm); + const auto normalized = at(item.normalized); + const auto once = NormalizeOnce(ours, reference); + const auto twice = NormalizeOnce(once, reference); + min_pre = std::min(min_pre, Cosine(ours, pre)); + min_once = std::min(min_once, Cosine(once, normalized)); + min_twice = std::min(min_twice, Cosine(twice, normalized)); + min_unnormalized = std::min(min_unnormalized, Cosine(ours, normalized)); + const double ratio = Rms(ours) / Rms(pre); + min_rms_ratio = std::min(min_rms_ratio, ratio); + max_rms_ratio = std::max(max_rms_ratio, ratio); + } + std::cout << "reference case=" << c << " rows=" << states.rows + << " min_cos_pre_norm=" << min_pre + << " min_cos_norm_once=" << min_once + << " min_cos_norm_twice=" << min_twice + << " min_cos_raw_vs_normalized=" << min_unnormalized + << " rms_ratio=[" << min_rms_ratio << ", " << max_rms_ratio + << "]\n"; + Expect(min_pre >= floor, "adapter rows differ from the pre-norm reference"); + Expect(min_once >= floor, + "adapter rows normalized once differ from the normalized " + "reference"); + // Unnormalized rows keep the residual stream's magnitude; a runner that + // already applied the norm would land near the normalized scale instead. + Expect(min_rms_ratio > 0.9 && max_rms_ratio < 1.1, + "adapter rows are not on the pre-norm scale"); + } +} + +void Run(const char* model_path) { + std::string error; + auto owner = gufo::core::GgufReader::OpenFile(model_path, &error); + Expect(owner != nullptr, error); + auto model = gufo::hip::QwenGpuModel::CreateFromGguf( + std::shared_ptr(std::move(owner)), &error); + Expect(model != nullptr, error); + const std::size_t width = model->GetConfig().hidden_size; + const auto runner = gufo::models::qwen::CreateQwenTextRunner( + model, kContext, nullptr, gufo::speculative::SpeculativeOptions{}, + gufo::speculative::DFlashDraftPolicy::kAdaptive, {}, {}); + + const auto encoding = runner->HiddenStateEncoding(); + Expect(encoding.has_value() && encoding->width == width && + encoding->max_tokens == kContext, + "Qwen runner must report its hidden-state width and limit"); + + const auto positive = runner->Tokenize( + Frame("A red fox reading a newspaper on a park bench, watercolor.")); + const auto empty = runner->Tokenize(Frame("")); + const auto a = Encode(*runner, positive); + const auto e = Encode(*runner, empty); + Expect(a.width == width && e.width == width, "row width is hidden_size"); + Expect(SameBytes(Encode(*runner, positive).values, a.values) && + SameBytes(Encode(*runner, empty).values, e.values) && + SameBytes(Encode(*runner, positive).values, a.values), + "repeated positive/empty encoding must be bit-identical"); + + // A session on the same weights: encoding between its prefill and decode + // must not change a single logit. + gufo::hip::QwenGpuExecutor session(model, kContext); + const auto baseline = RunSession(session, positive, {}); + const auto interleaved = RunSession(session, positive, [&] { + (void)Encode(*runner, empty); + (void)Encode(*runner, positive); + }); + Expect(baseline.size() == interleaved.size(), "session step count"); + for (std::size_t step = 0; step < baseline.size(); ++step) + Expect(SameBytes(baseline[step], interleaved[step]), + "hidden-state encoding changed a concurrent session's logits"); + Expect(SameBytes(Encode(*runner, positive).values, a.values), + "session activity changed hidden-state encoding"); + + // Past the first 1,024-token step the private executor is rebuilt larger + // and the capture spans several prefill chunks; neither may change results. + std::vector long_prompt; + while (long_prompt.size() < 2500) + long_prompt.insert(long_prompt.end(), positive.begin(), positive.end()); + const auto long_states = Encode(*runner, long_prompt); + Expect(SameBytes(Encode(*runner, positive).values, a.values) && + SameBytes(Encode(*runner, long_prompt).values, long_states.values), + "a larger private executor changed hidden-state encoding"); + + TextHiddenStates states; + Expect(runner->EncodeHiddenStates( + {.tokens = positive, .is_cancelled = [] { return true; }}, &states, + &error) == TextHiddenStateStatus::kCancelled && + states.values.empty(), + "cancelled encoding reports kCancelled without states"); + int polls = 0; + const auto status = runner->EncodeHiddenStates( + {.tokens = positive, .is_cancelled = [&polls] { return ++polls > 3; }}, + &states, &error); + Expect(status == TextHiddenStateStatus::kCancelled || + status == TextHiddenStateStatus::kOk, + "mid-pass cancellation either stops or completes"); + Expect(SameBytes(Encode(*runner, positive).values, a.values), + "encoding after cancellation must be unaffected"); + + Expect(runner->EncodeHiddenStates({}, &states, &error) == + TextHiddenStateStatus::kInvalidInput, + "an empty prompt is invalid input"); + const std::vector outside{ + static_cast(model->GetConfig().vocab_size)}; + Expect(runner->EncodeHiddenStates({.tokens = outside, .is_cancelled = {}}, + &states, &error) == + TextHiddenStateStatus::kInvalidInput, + "a token outside the vocabulary is invalid input"); + const std::vector too_long(kContext + 1, positive.front()); + Expect(runner->EncodeHiddenStates({.tokens = too_long, .is_cancelled = {}}, + &states, &error) == + TextHiddenStateStatus::kInvalidInput, + "a prompt above max_tokens is invalid input"); + + if (const char* path = std::getenv("GUFO_QWEN_HIDDEN_REFERENCE")) + CompareWithReference(*runner, LoadReference(path), width); + else + std::cout << "GUFO_QWEN_HIDDEN_REFERENCE unset: reference comparison " + "skipped\n"; + std::cout << "positive rows=" << a.rows << " empty rows=" << e.rows + << " width=" << width << '\n'; +} + +} // namespace + +int main(int argc, const char* const* argv) { + try { + const char* model = + argc > 1 ? argv[1] : std::getenv("GUFO_QWEN_HIDDEN_MODEL"); + if (model == nullptr) { + std::cout << "Set GUFO_QWEN_HIDDEN_MODEL for the Qwen hidden-state " + "check.\n"; + return 77; + } + Run(model); + std::cout << "Qwen hidden-state checks passed.\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/tools/README.md b/tools/README.md index 060b223c..9eba5346 100644 --- a/tools/README.md +++ b/tools/README.md @@ -80,5 +80,10 @@ It replaces the old standalone W8A8 kernel copies. dispatch (exact outputs, weights rotated beyond the MALL cache); build the `qwen27b_verify_sweep` target and run `sweep.py`, see its README. `qwen27b/dflash_reference.py` checks a GPU trace against -pinned upstream PyTorch operators using the same GGUF weights. See +pinned upstream PyTorch operators using the same GGUF weights. +`qwen27b/hidden_state_reference.py` records unquantized Hugging Face pre- and +post-norm last-layer states for `qwen_text_hidden_states_test`, which checks +`TextModelRunner::EncodeHiddenStates` (`GUFO_QWEN_HIDDEN_REFERENCE`); +`--device cuda --dtype bfloat16` loads the checkpoint onto the GPU instead of +host RAM (needs `accelerate`). See `docs/models/qwen3.8-27b/BENCHMARKS.md` and its quality report for commands and evidence. diff --git a/tools/qwen27b/hidden_state_reference.py b/tools/qwen27b/hidden_state_reference.py new file mode 100644 index 00000000..dba2ef29 --- /dev/null +++ b/tools/qwen27b/hidden_state_reference.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Write an independent reference for TextModelRunner::EncodeHiddenStates. + +Runs an unquantized Hugging Face Qwen3.5/3.8 checkpoint with transformers on +fixed, MOSS-Image-framed prompts and records per token: + +- the last decoder layer's output (the residual stream before the final norm), +- the same rows after the checkpoint's final norm, and +- the final norm's effective per-channel scale and epsilon. + +tests/models/qwen/hip/text_hidden_states_test.cpp (GUFO_QWEN_HIDDEN_REFERENCE) +compares Gufo's Qwen adapter with the first and, after applying the norm once +itself, with the second. Token IDs are stored, so both sides encode identical +inputs. No download is implicit: pass local directories. The default float32 +CPU run needs about four bytes of RAM per parameter; --device cuda (ROCm or +CUDA) loads the weights straight onto the GPU, normally with --dtype bfloat16, +without staging them in host RAM. Recorded rows are float32 either way. +""" +from __future__ import annotations + +import argparse +import math +import struct +from pathlib import Path + +import numpy as np +import torch +from transformers import AutoModel, AutoTokenizer + +MAGIC = b"GQHS" +VERSION = 1 +# MOSS-Image framing; enable_thinking=True leaves the think block open. +FRAME = "<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n\n" +PROMPTS = ("A red fox reading a newspaper on a park bench, watercolor.", "") + + +def text_backbone(model): + """The decoder owning embed_tokens, layers and the final norm.""" + for name, module in model.named_modules(): + if all(hasattr(module, attr) for attr in ("embed_tokens", "layers", "norm")): + return name, module + raise SystemExit("no decoder with embed_tokens, layers and norm found") + + +def norm_epsilon(norm) -> float: + for attr in ("variance_epsilon", "eps"): + value = getattr(norm, attr, None) + if value is not None: + return float(value) + raise SystemExit("cannot read the final norm epsilon") + + +@torch.no_grad() +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("checkpoint", type=Path, + help="Hugging Face model directory (for MOSS-Image: text_encoder)") + parser.add_argument("output", type=Path) + parser.add_argument("--tokenizer", type=Path, + help="tokenizer directory (default: the checkpoint)") + parser.add_argument("--dtype", choices=("float32", "bfloat16"), default="float32") + parser.add_argument("--device", default="cpu", + help="torch device, e.g. cuda (also ROCm) to load the weights " + "directly onto the GPU (default: cpu); needs accelerate") + args = parser.parse_args() + + tokenizer = AutoTokenizer.from_pretrained(args.tokenizer or args.checkpoint) + # A device map streams each tensor straight to its device instead of + # materializing the whole model in host RAM first. + placement = ({} if torch.device(args.device).type == "cpu" else + {"low_cpu_mem_usage": True, "device_map": {"": args.device}}) + model = AutoModel.from_pretrained(args.checkpoint, torch_dtype=getattr(torch, args.dtype), + **placement) + model.eval() + name, backbone = text_backbone(model) + norm = backbone.norm + epsilon = norm_epsilon(norm) + width = backbone.embed_tokens.embedding_dim + # For x = 1 an RMS norm returns scale / sqrt(1 + eps), whatever its weight + # parameterization (plain or zero-centered). + ones = torch.ones(1, 1, width, dtype=norm.weight.dtype, device=norm.weight.device) + scale = (norm(ones).float() * math.sqrt(1.0 + epsilon)).reshape(width).cpu() + + captured = [] + hook = backbone.layers[-1].register_forward_hook( + lambda _module, _inputs, output: captured.append( + output[0] if isinstance(output, tuple) else output)) + cases = [] + for prompt in PROMPTS: + ids = tokenizer(FRAME.format(prompt), add_special_tokens=False, + return_tensors="pt").input_ids + captured.clear() + backbone(input_ids=ids.to(backbone.embed_tokens.weight.device), use_cache=False) + if len(captured) != 1: + raise SystemExit(f"expected one last-layer output, got {len(captured)}") + pre = captured[0] + cases.append((ids.reshape(-1).numpy().astype("'} width={width} " + f"epsilon={epsilon} cases={[int(ids.size) for ids, _, _ in cases]}") + + +if __name__ == "__main__": + main()