mirror of
https://github.com/Niko1221/Strata.git
synced 2026-11-13 16:51:30 -10:00
Techannel
9c5bbc36d0
dequant_kernel's thread reads its 32-value group a byte at a time and writes 32 two-byte values one by one, 64 bytes from its neighbour's. On a Tesla P40 that is 10 GB/s for Q6_K, and the IQ3_S pack's dense weights cost every prompt chunk ~0.5 s of it, whatever the chunk's length (a fifth of a 560-token prompt). The wide kernel takes 8 values a thread: 16-bit loads, one 16-byte store, the same expression per value. dequant_wide_parity: 360 cases bitwise (FP16 / BF16 / FP32 out, a padded stride, row slices, sane and raw random blocks); --bench on the P40: Q6_K 10240 x 2560 7.25 -> 0.33 ms, Q4_K 5.2x, Q5_K 3.8x, IQ4_XS 3.6x. On by default on compute capability 6.1 only (where it was measured); STRATA_DEQUANT_WIDE=0 / =1 for any card.
37 lines
1.9 KiB
C++
37 lines
1.9 KiB
C++
// include/strata/kernels/dequant_bf16.hpp - plan v0.3 P5: native GGUF blocks -> BF16 on the device, for the
|
|
// tensor-core GEMMs of batched prompt processing.
|
|
//
|
|
// One thread per 32-element group, transcribed from the reference `dequantize_row_*` functions of the pinned
|
|
// llama.cpp (`ggml/src/ggml-quants.c`, MIT): Q4_0 (2), Q5_0 (6), Q8_0 (8), Q3_K (11), Q4_K (12), Q5_K (13), Q6_K (14),
|
|
// IQ4_NL (20), IQ4_XS (23) and Q2_0 (42). A row-major (n_rows, n_cols) tensor of blocks becomes a row-major BF16
|
|
// matrix; `n_cols` must be a multiple of the type's block size.
|
|
#pragma once
|
|
|
|
#include <cstdint>
|
|
|
|
namespace strata::kernels {
|
|
|
|
bool dequant_bf16_supported(int ggml_type) noexcept;
|
|
|
|
/// `rows` rows of `cols` values starting at block row `row0` (so a caller can dequantize a slice of a tensor).
|
|
void dequant_bf16(int ggml_type, const void* blocks, int64_t row0, int64_t rows, int64_t cols, uint16_t* out,
|
|
void* stream);
|
|
|
|
/// The same into FP16 bits (the prompt path's quantized-weight GEMMs: Q2_0 values are exact in FP16).
|
|
void dequant_f16(int ggml_type, const void* blocks, int64_t row0, int64_t rows, int64_t cols, uint16_t* out,
|
|
void* stream);
|
|
|
|
/// dequant_f16 with output row stride `ld` >= cols (a multiple of 8): S23's padded weight scratch. False (nothing
|
|
/// launched) for the i-quant formats and unsupported shapes.
|
|
bool dequant_f16_ld(int ggml_type, const void* blocks, int64_t row0, int64_t rows, int64_t cols, int64_t ld,
|
|
uint16_t* out, void* stream);
|
|
|
|
/// The same into FP32 (tests and small tensors).
|
|
void dequant_f32(int ggml_type, const void* blocks, int64_t row0, int64_t rows, int64_t cols, float* out, void* stream);
|
|
|
|
/// The wide kernels (a thread per 8 values, bitwise the same output; Q4_K, Q5_K, Q6_K, IQ4_XS): 0 off, 1 on, -1 the
|
|
/// default (STRATA_DEQUANT_WIDE, else on for compute capability 6.1). For dequant_wide_parity.
|
|
void dequant_set_wide(int mode);
|
|
|
|
} // namespace strata::kernels
|