mirror of
https://github.com/Niko1221/Strata.git
synced 2026-11-07 06:15:58 -10:00
Formats cost nothing unless used: Q6_K gate/up experts in the normal build; the K-quant MMQ prompt kernels on by default, on CUDA as their own library (strata_kq) opened only for a model with these experts and STRATA_MMQ_KQ=1
This commit is contained in:
Niko1221 committed
1 parent
0d3553fc8f
commit
465cd61dfb
9 files changed
+364
-77
No files matched your search
+53
-16
@@ -70,14 +70,17 @@ if(STRATA_HIP_GFX906)
|
||||
endif()
|
||||
option(STRATA_ENABLE_HIP "Build the HIP targets for wave32 AMD GPUs (gfx1100, gfx1201; gfx1012/gfx1030 unvalidated)" OFF)
|
||||
option(STRATA_PREFILL_MMQ "Build the optional GGML MMQ prefill path for HIP" OFF)
|
||||
# CUDA: MMQ prompt kernels for Q4_K / Q5_K / Q5_1 (Unsloth's UD-Q4_K_XL experts). Off by default: the engine loads
|
||||
# every kernel when it starts (CUDA_MODULE_LOADING=EAGER), so these took ~5 MB of VRAM - 1-3 expert slots - from every
|
||||
# model, and the UD-Q4_K_XL prompt is SSD-bound with or without them (docs/UNSLOTH_Q4.md)
|
||||
option(STRATA_MMQ_KQUANTS "Build the MMQ prompt kernels for Q4_K / Q5_K / Q5_1 / Q6_K: the CUDA experts of Unsloth's UD-Q4_K_XL, and on HIP the dense GGUF projections (STRATA_DENSE_MMQ)" OFF)
|
||||
# CUDA: Q6_K gate/up in the grouped expert kernels (Unsloth's UD-Q6_K_XL, docs/UNSLOTH_Q6.md). Off by default: every
|
||||
# grouped-kernel instance is loaded when the engine starts (CUDA_MODULE_LOADING=EAGER), so one more costs VRAM that
|
||||
# every other model would lose as expert slots (PR #860 review)
|
||||
option(STRATA_Q6K_EXPERTS "Build the Q6_K gate/up grouped expert kernels (UD-Q6_K_XL)" OFF)
|
||||
# MMQ prompt kernels for Q4_K / Q5_K / Q5_1 / Q6_K (Unsloth's UD-Q4/Q5/Q6_K_XL experts; on HIP also the dense GGUF
|
||||
# projections, STRATA_DENSE_MMQ). On by default since MODEL-UPDATE Phase 1: on CUDA they are their own library
|
||||
# (strata_kq, loaded only for a model with these formats: the engine loads every kernel of its own binary at start,
|
||||
# CUDA_MODULE_LOADING=EAGER, and these took ~3.8 MB of VRAM - 3 IQ3_XXS expert slots on a P100 - from every model); on
|
||||
# HIP a code object loads on first use, so they cost other models nothing there (R9700: the same slots). Used at run
|
||||
# time with STRATA_MMQ_KQ=1 (opt-in: it changes the prompt's rounding against the FP16 path).
|
||||
option(STRATA_MMQ_KQUANTS "Build the MMQ prompt kernels for Q4_K / Q5_K / Q5_1 / Q6_K: the experts of Unsloth's K-quant files, and on HIP the dense GGUF projections (STRATA_DENSE_MMQ)" ON)
|
||||
# Q6_K gate/up in the grouped expert kernels (Unsloth's UD-Q6_K_XL, and layer 2 of UD-Q5_K_XL; docs/QUANTS.md). On by
|
||||
# default since MODEL-UPDATE Phase 1: one more instance per grouped kernel in iq_kernels.cu, +19 KB of sm_60 code,
|
||||
# measured to leave every expert-cache slot count as it was (P100, R9700). -DSTRATA_Q6K_EXPERTS=OFF drops it.
|
||||
option(STRATA_Q6K_EXPERTS "Build the Q6_K gate/up grouped expert kernels (UD-Q6_K_XL, UD-Q5_K_XL)" ON)
|
||||
if(STRATA_Q6K_EXPERTS)
|
||||
add_compile_definitions(STRATA_Q6K_EXPERTS=1)
|
||||
endif()
|
||||
@@ -1365,14 +1368,18 @@ if(STRATA_NATIVE_EXPERTS)
|
||||
add_library(strata_mmq STATIC src/prefill/moe_mmq.cu src/prefill/moe_fused.cu src/prefill/moe_fused_iq.cu
|
||||
src/prefill/ggml_cuda_host.cu ${_strata_mmq}/quantize.cu)
|
||||
# q8_0: the drafter's matrices (E-9); STRATA_ORCA_Q4KS_MMQ adds q5_0 (#296, in _strata_mmq_types), and
|
||||
# STRATA_MMQ_KQUANTS q4_k q5_k q5_1, the experts of Unsloth's UD-Q4_K_XL (docs/UNSLOTH_Q4.md; the dispatch
|
||||
# follows eddoursul/Strata 7706c1b); q6_k is HIP only
|
||||
# STRATA_MMQ_KQUANTS q4_k q5_k q5_1 q6_k, the experts of Unsloth's UD-Q4/Q5/Q6_K_XL (docs/QUANTS.md; the dispatch
|
||||
# follows eddoursul/Strata 7706c1b). MODEL-UPDATE Phase 1: on CUDA they are a library of their own, strata_kq
|
||||
# (below), loaded only for a model with these experts: the engine loads every kernel of its own binary at start
|
||||
# (CUDA_MODULE_LOADING=EAGER), and these four took ~3.8 MB of VRAM (3 expert slots of IQ3_XXS on a P100) from every
|
||||
# model. gfx906 (HIP: code objects load on first use) keeps them in strata_mmq.
|
||||
set(_strata_mmq_cuda_types ${_strata_mmq_types})
|
||||
if(STRATA_MMQ_KQUANTS)
|
||||
list(APPEND _strata_mmq_cuda_types q4_k q5_k q5_1)
|
||||
if(STRATA_Q6K_EXPERTS OR STRATA_HIP_GFX906) # UD-Q6_K_XL gate/up (docs/UNSLOTH_Q6.md),
|
||||
list(APPEND _strata_mmq_cuda_types q6_k) # and gfx906: moe_mmq.cu calls it under __HIPCC__
|
||||
endif()
|
||||
set(_strata_kq_types q4_k q5_k q5_1 q6_k)
|
||||
set(_strata_kq_dso OFF)
|
||||
if(STRATA_MMQ_KQUANTS AND STRATA_HIP_GFX906)
|
||||
list(APPEND _strata_mmq_cuda_types ${_strata_kq_types})
|
||||
elseif(STRATA_MMQ_KQUANTS)
|
||||
set(_strata_kq_dso ON)
|
||||
endif()
|
||||
foreach(_t IN LISTS _strata_mmq_cuda_types)
|
||||
target_sources(strata_mmq PRIVATE ${_strata_mmq}/template-instances/mmq-instance-${_t}.cu)
|
||||
@@ -1391,6 +1398,35 @@ if(STRATA_NATIVE_EXPERTS)
|
||||
if(STRATA_MMQ_KQUANTS)
|
||||
target_compile_definitions(strata_mmq PRIVATE STRATA_MMQ_KQUANTS=1)
|
||||
endif()
|
||||
if(_strata_kq_dso)
|
||||
# strata_kq: the K-quant MMQ instances with their own copy of ggml_cuda_host.cu's symbols and the static CUDA
|
||||
# runtime, next to the engine (libstrata_kq.so / strata_kq.dll); moe_mmq.cu opens it when a model asks for one
|
||||
# of these formats (STRATA_MMQ_KQ=1). Without the file the engine runs them on the FP16 prompt path, as before.
|
||||
target_compile_definitions(strata_mmq PRIVATE STRATA_KQ_DSO=1)
|
||||
target_sources(strata_mmq PRIVATE src/prefill/mmq_kq_loader.cpp)
|
||||
if(NOT WIN32)
|
||||
target_link_libraries(strata_mmq PUBLIC ${CMAKE_DL_LIBS})
|
||||
endif()
|
||||
add_library(strata_kq SHARED src/prefill/mmq_kq_lib.cu src/prefill/ggml_cuda_host.cu)
|
||||
foreach(_t IN LISTS _strata_kq_types)
|
||||
target_sources(strata_kq PRIVATE ${_strata_mmq}/template-instances/mmq-instance-${_t}.cu)
|
||||
endforeach()
|
||||
target_include_directories(strata_kq PRIVATE ${_strata_gg}/include ${_strata_gg}/src ${_strata_mmq}
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/include)
|
||||
target_compile_options(strata_kq PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:--extended-lambda -use_fast_math>)
|
||||
if(MSVC)
|
||||
target_compile_options(strata_kq PRIVATE $<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=/bigobj>)
|
||||
endif()
|
||||
set_target_properties(strata_kq PROPERTIES CUDA_RUNTIME_LIBRARY Static POSITION_INDEPENDENT_CODE ON
|
||||
CXX_VISIBILITY_PRESET hidden CUDA_VISIBILITY_PRESET hidden)
|
||||
# ggml-base is static: position-independent so a shared library can hold it (the same code otherwise)
|
||||
set_target_properties(ggml-base PROPERTIES POSITION_INDEPENDENT_CODE ON)
|
||||
target_link_libraries(strata_kq PRIVATE ggml-base)
|
||||
if(TARGET CUDA::cuda_driver)
|
||||
target_link_libraries(strata_kq PRIVATE CUDA::cuda_driver)
|
||||
endif()
|
||||
add_dependencies(strata_mmq strata_kq)
|
||||
endif()
|
||||
if(STRATA_ORCA_Q4KS_MMQ)
|
||||
target_compile_definitions(strata_mmq PRIVATE STRATA_ORCA_Q4KS_MMQ=1)
|
||||
endif()
|
||||
@@ -1413,7 +1449,8 @@ if(STRATA_NATIVE_EXPERTS)
|
||||
add_executable(prefill_mmq_kquant_test tests/cuda/prefill_mmq_kquant_test.cpp)
|
||||
target_link_libraries(prefill_mmq_kquant_test PRIVATE strata_mmq ggml-base ${_strata_gpu_runtime_target})
|
||||
add_test(NAME prefill_mmq_kquant_test COMMAND prefill_mmq_kquant_test)
|
||||
set_tests_properties(prefill_mmq_kquant_test PROPERTIES TIMEOUT 120)
|
||||
# the K-quant MMQ is opt-in at run time (STRATA_MMQ_KQ=1), and on CUDA it is strata_kq next to the test
|
||||
set_tests_properties(prefill_mmq_kquant_test PROPERTIES TIMEOUT 120 ENVIRONMENT STRATA_MMQ_KQ=1)
|
||||
endif()
|
||||
elseif(STRATA_PREFILL_MMQ)
|
||||
# Compile only the pinned GGML MMQ and activation-quantizer sources used by prefill. This uses
|
||||
|
||||
@@ -1,56 +1,49 @@
|
||||
## What is in the file
|
||||
- **GPU (VRAM):** attention and DeltaNet mixers, the gated-residual weights, routers, shared experts, output head, the MTP
|
||||
draft layer, the KV cache (from 64K: only its most-read part, the rest streams from RAM), and an **expert cache** that fills the rest of VRAM with the most-used experts (it adapts to
|
||||
the conversation while you chat).
|
||||
- **RAM:** all 24,576 experts, pinned. The CPU computes the experts that are not on the GPU **in place**, at the same time
|
||||
as the GPU works on the cached ones (AVX-512 / AVX2 kernels, ggml's for the i-quants).
|
||||
- **SSD:** the 28.8 GB n-gram table, a few rows per token, read unbuffered past the OS cache (`--ple-io direct`, the
|
||||
default, made for SSDs; on a rotational disk `--ple-io ram` keeps the table in RAM, #605). A start-up probe warns when direct reads measure slow (under 15K rows/s) but never switches by itself (since 0.1.43.1; `STRATA_PLE_AUTO_MMAP=1` brings back the 0.1.42 switch to `--ple-io mmap`): the bar is calibrated for NVMe, a SATA SSD tops out at 10-14K rows/s with direct's 16 reads in flight, and on one (Linux, 77.8K-token prompt) mmap read 2.1K rows/s against direct's 14.4K (#2040), while on a Windows box with a slow mapped file mmap stalled prompts (#1984). The engine reads the table
|
||||
in the format the GGUF has it: IQ4_NL (the default table), Q4_0, Q5_0, Q5_1, Q8_0, FP8 (E4M3 with a scale) or BF16.
|
||||
Measured on 4,000 random rows against the checkpoint's own BF16 table, the mean per-row error is Q8_0 0.53%, FP8 2.64%,
|
||||
Q5_1 3.78%, Q5_0 4.25%, IQ4_NL 7.60%, Q4_1 7.80%, Q4_0 8.55% (a Q8_0 table is 54 GB, an IQ4_NL one 28.8 GB). On the Q2_0 model, 700 teacher-forced tokens, the mean KL against the BF16 table is 0.0117 (Q5_0), 0.0121 (Q5_1), 0.0125
|
||||
(Q8_0), 0.0128 (FP8), 0.0140 (Q4_0) and 0.0146 (IQ4_NL), with the perplexity within 1% of BF16's either way: any change to the
|
||||
table moves the 2-bit model by about 0.012, so the formats are hard to tell apart. Setup does not offer another table;
|
||||
it only changes which GGUF the engine is given.
|
||||
- **Speculation:** the model's own MTP layer drafts up to 3 tokens; one pass over all 48 layers checks them. 2.4-3.2
|
||||
tokens per pass on average. When the reply repeats the context (code edits, quoted text), **prompt lookup** (engine
|
||||
0.1.7) drafts up to 5 tokens from the earlier copy, but only where its measured acceptance and cost say it pays:
|
||||
code edits 6-11% faster, other text unchanged. The drafts are checked like the MTP's, so the output is the same.
|
||||
- **Prompts** are processed in chunks of up to 8,192 tokens (`--prefill auto`; 32,768 opt-in) with the experts
|
||||
streamed to the GPU over PCIe. Since engine 0.1.39b (#583) the streamed ring is sized in bytes for the pack (a
|
||||
native pack with bigger blobs gets fewer ring slots and keeps more cache slots) and `--prefill auto` picks the
|
||||
largest chunk that keeps that ring full. It only gives up ring slots where 0.1.39's rule held the chunk at 4,096
|
||||
tokens or less; from 6,144 on it keeps 0.1.39's ring and only looks for a bigger chunk beside it (shrinking the
|
||||
ring there measured slower: the Coder -12%), and a prompt that fits 0.1.39's chunk keeps 0.1.39's ring (one
|
||||
chunk: a smaller ring only slowed it). RTX 5070, 32K prompts against `STRATA_RING_BYTES=0`, 2-3 interleaved
|
||||
rounds: IQ3_XXS +18.5% at a 1,500-slot cache and +9% in another memory state, the Coder +3%, IQ3_S unchanged; 4K
|
||||
prompts unchanged (-0.7% to +1.1%, bit-identical). Q2_0 (a fixed `--prefill 2048`) is not affected. It changes
|
||||
the bits of a long prompt against 0.1.39 (the experts go through a different
|
||||
mix of cached and streamed groups). Measured quality, teacher-forced over the next 2,001 tokens of a long document
|
||||
against the FP16 prompt path (IQ3_XXS, `--prefill auto`): 8K prompt KL 0.042 (0.1.39: 0.054), top-1 agreement
|
||||
93.5% (92.2%); 32K prompt KL 0.020 (0.019), top-1 95.3% (95.0%) - the same band as before. `STRATA_RING_BYTES=0`
|
||||
restores 0.1.39's ring, loan and chunk choice.
|
||||
|
||||
Six shards, 169,165,382,688 bytes (169.2 GB) together ([sizes and SHA-256 from the HF tree API](https://huggingface.co/api/models/unsloth/Qwen3.8-Flash-Next-GGUF/tree/38bb39ee97821de2c9009abb7e93950eec396e66/UD-Q6_K_XL);
|
||||
shards 1, 2 and 6 were also checked bit for bit against their Hugging Face LFS hashes after download). Shard 1 holds
|
||||
only metadata; shard 2 only the output head (`output.weight`, `output_hc_*`, all Q8_0 but `output_hc_norm`, F32);
|
||||
shard 3 is exactly the PLE table, nothing else.
|
||||
The full story, with measurements, bottlenecks and what comes next: **[docs/paper/Strata-Paper.pdf](paper/Strata-Paper.pdf)**.
|
||||
|
||||
| File | Bytes | SHA-256 |
|
||||
| --- | --- | --- |
|
||||
| `Qwen3.8-Flash-Next-UD-Q6_K_XL-00001-of-00006.gguf` | 10,946,624 | `8bdc6bad55b6699f46f60708829932ff4018d7ac9bd309c388534fa31753bdb7` |
|
||||
| `Qwen3.8-Flash-Next-UD-Q6_K_XL-00002-of-00006.gguf` | 682,434,912 | `494ca4ed3dbf97bc28da88af3890b8877b9032f909812d00c0526a9ca5e91d2e` |
|
||||
| `Qwen3.8-Flash-Next-UD-Q6_K_XL-00003-of-00006.gguf` | 54,400,261,312 | `34efd79a80a1ce540a517a5d56171924b66ce1c38b04c904f17ad6d8ef17cf20` |
|
||||
| `Qwen3.8-Flash-Next-UD-Q6_K_XL-00004-of-00006.gguf` | 49,814,108,512 | `0ea4b599880a5a52fcd9c188ba1443f646c69c96571904ddbd6a79246a01a997` |
|
||||
| `Qwen3.8-Flash-Next-UD-Q6_K_XL-00005-of-00006.gguf` | 49,419,317,344 | `9948e81ae8368144b135a7403d8c0f2cf9a6c2f9f692313865855b2832e696c9` |
|
||||
| `Qwen3.8-Flash-Next-UD-Q6_K_XL-00006-of-00006.gguf` | 14,838,313,984 | `e4ae255234f42b18012f6d6eec5bf615b57479b7e0a0233c47436a321ffeec3f` |
|
||||
---
|
||||
|
||||
Formats, read from the shard headers with `tools/gguf_reader.py` (all six shards verified: shard 4 = blocks 0-20,
|
||||
520 tensors, shard 5 = blocks 20-41, 530 tensors, and per layer the types above):
|
||||
## Credits and licenses
|
||||
|
||||
- **Routed experts:** `ffn_gate_exps` and `ffn_up_exps` are Q6_K (seen in every layer of shard 6: blocks 41-47);
|
||||
`ffn_down_exps` is Q8_0 (per layer 891.3 MB at 640×2560×512, against Q6_K's 688.1 MB at 2560×640×512 for gate or
|
||||
up). Gate and up are Q6_K in 47 of 48 layers and Q8_0 in layer 2; down is Q8_0 in every layer (the pack's
|
||||
`native_experts.txt`: 47 rows of ggml types 14/8, and layer 2 is 8/8). Q6_K gate/up is native: the
|
||||
engine multiplies them directly on the GPU in every path check below, and the CPU has the ggml block layout (210
|
||||
bytes per 256 elements); the per-role layout in `native_experts.txt` v4 handles shards whose boundaries cut
|
||||
through a layer like UD-Q4_K_XL's layer 11 case does.
|
||||
- **Everything else:** attention and shared-expert projections, SSM, hyper-connections, the PLE key/value, the token
|
||||
embedding and the output head are Q8_0, or F32/BF16 where the engine reads floats — the same pattern
|
||||
[UD-Q4_K_XL](UNSLOTH_Q4.md) handles with `tools/iq_pack.py --compat-bf16`.
|
||||
- **The PLE table** (`per_layer_token_embd.weight`, [160, 320001536]): **Q8_0**, its own shard 3. A row is
|
||||
5 blocks of 32 elements at 34 bytes = 170 bytes, and the whole table is 54.4 GB, against the 28.8 GB IQ4_NL of
|
||||
UD-Q4_K_XL. Engine 0.1.38 reads Q8_0 rows natively: `PleTable` accepts the type, the row reader is already
|
||||
parameterized by bytes-per-row (its selftest runs 90, 110 and 170 byte rows), and no GPU kernel changes —
|
||||
`gather` outputs floats. (UD-Q4_K_XL's table is IQ4_NL; the FP8 table of [tools/ple_fp8_pack.py](../tools/ple_fp8_pack.py)
|
||||
needs the original checkpoint, which this file is not a substitute for. Converting this Q8_0 table to FP8 would
|
||||
trade table precision away for 6% smaller rows; Q8_0 read directly has none that conversion's error.)
|
||||
Strata itself: [MIT](../LICENSE). The model files are not part of it; their licenses apply to them (below).
|
||||
|
||||
Q6_K gate/up costs 210 bytes per 256 elements, against Q4_K's 144, so the routed experts' share of the file grows
|
||||
about 45% per element read; expected decode speed accordingly lower than UD-Q4_K_XL's on the same hardware. The
|
||||
table row gather grows 170/90 = 1.89x over IQ4_NL.
|
||||
|
||||
## The pack (by hand)
|
||||
|
||||
Same workflow as [UD-Q4_K_XL](UNSLOTH_Q4.md#the-pack-by-hand), from the repository root:
|
||||
|
||||
```sh
|
||||
.venv/bin/python tools/iq_pack.py \
|
||||
--gguf /path/to/Qwen3.8-Flash-Next-UD-Q6_K_XL-00001-of-00006.gguf \
|
||||
--out packs/ud-q6_k_xl --compat-bf16
|
||||
```
|
||||
|
||||
**`--compat-bf16` is required** for the same reason as there (the 195 small Q8_0 projections the engine reads as
|
||||
BF16). Do not add `--experts-bin`; the experts stay in the GGUF files. Built here end to end on this file
|
||||
(`packs/ud-q6_k_xl`, the packer's own tests cover the shard and conversion logic via
|
||||
`.venv/bin/python -m unittest discover -s tools -p test_iq_pack.py`): 1079 tensors indexed, 303 served natively, 460
|
||||
tensors converted (26
|
||||
- Model: [Qwen/Qwen3.8-Flash-Next](https://huggingface.co/Qwen/Qwen3.8-Flash-Next) by the Qwen team; quantizations:
|
||||
[ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF](https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF).
|
||||
The Coder: [ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-Coder-GGUF](https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-Coder-GGUF)
|
||||
(Apache-2.0 per its card); its support in Strata came from @pjgmobile's PR #54.
|
||||
Swift 1.5: [ukisai/Swift-1.5-Qwen3.8-Flash-Next-GSQ-RCO-GGUF](https://huggingface.co/ukisai/Swift-1.5-Qwen3.8-Flash-Next-GSQ-RCO-GGUF)
|
||||
by UkisAI. Their licenses apply to the weights.
|
||||
- [llama.cpp / ggml](https://github.com/ggml-org/llama.cpp) (MIT): the i-quant formats, the GPU dot products and
|
||||
dequantizers transcribed in `src/kernels/cuda/iq_kernels.cu`, the CPU backend linked for the i-quant experts, the
|
||||
`mtmd` library behind the image encoder (`tools/vision/`), and `gguf-py` use
|
||||
@@ -1 +1 @@
|
||||
{"version": "v1", "seg_tokens": 1000, "segments": [{"name": "code", "ids": [12237, 85286, 54802, 19913, 343, 18, 4108, 6396, 471, 874, 50285, 980, 11, 874, 50285, 1644, 2195, 88, 13, 271, 34447, 1521, 279, 1510, 13985, 1644, 63, 5187, 1828, 6129, 4021, 1724, 411, 35426, 25, 1141, 1510, 21563, 2581, 42750, 1954, 900, 63, 198, 8746, 682, 874, 4314, 364, 913, 220, 19, 17, 27718, 48, 17, 62, 15, 63, 681, 864, 369, 24733, 1056, 279, 6522, 13, 2844, 369, 4924, 264, 387, 15, 782, 21, 198, 64919, 471, 279, 3443, 411, 4560, 369, 26208, 364, 369, 8047, 314, 279, 5129, 5224, 286, 13, 271, 1919, 369, 279, 5604, 7873, 364, 1510, 3431, 53683, 20123, 14, 63, 318, 77646, 16137, 16, 20, 25, 328, 21563, 19913, 343, 18, 6396, 318, 76, 2117, 11, 198, 2083, 50285, 980, 23285, 61406, 1049, 68434, 279, 4108, 1132, 26, 15167, 795, 369, 2496, 1301, 13, 198, 12237, 198, 1445, 1259, 20409, 548, 1120, 31144, 271, 464, 795, 8588, 198, 464, 51372, 198, 464, 1968, 198, 1445, 18904, 1120, 5586, 271, 21563, 19913, 47540, 283, 220, 15, 87, 19, 21, 20, 20, 19, 22, 19, 22, 220, 653, 328, 21563, 19913, 1, 2545, 12692, 1068, 271, 2, 50285, 980, 1756, 2663, 13, 220, 19, 17, 283, 1167, 17, 62, 15, 11, 279, 34100, 48, 12, 7146, 46, 69764, 17830, 517, 10845, 11, 26334, 494, 50285, 1644, 2195, 88, 430, 314, 220, 15, 13, 16, 24, 13, 198, 21563, 2581, 24973, 25, 6249, 18078, 11, 592, 60, 283, 313, 198, 262, 220, 15, 25, 328, 37, 18, 17, 487, 220, 16, 25, 328, 37, 16, 21, 487, 220, 17, 25, 328, 48, 19, 62, 15, 487, 220, 18, 25, 328, 48, 19, 62, 16, 487, 220, 19, 25, 328, 48, 19, 62, 17, 487, 220, 20, 25, 328, 48, 19, 62, 18, 487, 220, 21, 25, 328, 48, 20, 62, 15, 487, 220, 22, 25, 328, 48, 20, 62, 16, 487, 198, 262, 220, 23, 25, 328, 48, 23, 62, 15, 487, 220, 24, 25, 328, 48, 23, 62, 16, 487, 220, 16, 15, 25, 328, 48, 17, 9802, 487, 220, 16, 16, 25, 328, 48, 18, 9802, 487, 220, 16, 17, 25, 328, 48, 19, 9802, 487, 220, 16, 18, 25, 328, 48, 20, 9802, 487, 220, 16, 19, 25, 328, 48, 21, 9802, 487, 198, 262, 220, 16, 20, 25, 328, 48, 23, 9802, 487, 220, 16, 21, 25, 328, 59865, 17, 62, 5955, 50, 487, 220, 16, 22, 25, 328, 59865, 17, 6645, 50, 487, 220, 16, 23, 25, 328, 59865, 18, 62, 5955, 50, 487, 220, 16, 24, 25, 328, 59865, 16, 1063, 487, 220, 17, 15, 25, 328, 59865, 19, 1548, 43, 487, 198, 262, 220, 17, 16, 25, 328, 59865, 18, 1063, 487, 220, 17, 17, 25, 328, 59865, 17, 1063, 487, 220, 17, 18, 25, 328, 59865, 19, 6645, 50, 487, 220, 17, 19, 25, 328, 40, 23, 487, 220, 17, 20, 25, 328, 40, 16, 21, 487, 220, 17, 21, 25, 328, 40, 18, 17, 487, 220, 17, 22, 25, 328, 40, 21, 19, 487, 198, 262, 220, 17, 23, 25, 328, 37, 21, 19, 487, 220, 17, 24, 25, 328, 59865, 16, 1203, 487, 220, 18, 15, 25, 328, 19278, 16, 21, 487, 220, 18, 16, 25, 328, 48, 19, 62, 15, 62, 19, 62, 19, 487, 220, 18, 17, 25, 328, 48, 19, 62, 15, 62, 19, 62, 23, 487, 220, 18, 18, 25, 328, 48, 19, 62, 15, 62, 23, 62, 23, 487, 198, 262, 220, 18, 19, 25, 328, 51, 48, 16, 62, 15, 487, 220, 18, 20, 25, 328, 51, 48, 17, 62, 15, 487, 220, 18, 21, 25, 328, 59865, 19, 1548, 43, 62, 19, 62, 19, 487, 220, 18, 22, 25, 328, 59865, 19, 1548, 43, 62, 19, 62, 23, 487, 220, 18, 23, 25, 328, 59865, 19, 1548, 43, 62, 23, 62, 23, 487, 198, 262, 220, 18, 24, 25, 328, 28463, 11361, 19, 487, 220, 19, 15, 25, 328, 35125, 11361, 19, 487, 220, 19, 16, 25, 328, 48, 19, 62, 15, 62, 23, 62, 23, 487, 220, 19, 17, 25, 328, 48, 17, 62, 15, 487, 198, 92, 271, 2, 54802, 19913, 10836, 869, 913, 13746, 198, 21563, 19913, 57818, 283, 313, 198, 262, 220, 15, 25, 3376, 84, 23, 487, 220, 16, 681, 220, 16, 25, 3376, 72, 23, 487, 220, 16, 681, 220, 17, 25, 3376, 84, 16, 21, 487, 220, 17, 681, 220, 18, 25, 3376, 72, 16, 21, 487, 220, 17, 681, 220, 19, 25, 3376, 84, 18, 17, 487, 220, 19, 681, 220, 20, 25, 3376, 72, 18, 17, 487, 220, 19, 681, 198, 262, 220, 21, 25, 3376, 69, 18, 17, 487, 220, 19, 681, 220, 22, 25, 3376, 2554, 487, 220, 16, 681, 220, 23, 25, 3376, 889, 487, 2168, 681, 220, 24, 25, 3376, 1595, 487, 2168, 681, 198, 262, 220, 16, 15, 25, 3376, 84, 21, 19, 487, 220, 23, 681, 220, 16, 16, 25, 3376, 72, 21, 19, 487, 220, 23, 681, 220, 16, 17, 25, 3376, 69, 21, 19, 487, 220, 23, 681, 198, 92, 271, 2, 50285, 980, 2424, 16547, 364, 279, 3107, 56373, 411, 1558, 5533, 25, 318, 4425, 5255, 11, 5639, 791, 2424, 8, 198, 2, 1061, 369, 1092, 3520, 264, 791, 2308, 3677, 4763, 1698, 1716, 470, 11, 321, 424, 369, 279, 4954, 279, 61369, 4191, 13, 198, 38637, 57038, 83072, 25, 6249, 16793, 11, 13992, 18078, 11, 514, 4891, 283, 313, 198, 262, 328, 37, 18, 17, 763, 318, 16, 11, 220, 19, 681, 328, 37, 16, 21, 763, 318, 16, 11, 220, 17, 681, 328, 19278, 16, 21, 763, 318, 16, 11, 220, 17, 681, 328, 37, 21, 19, 763, 318, 16, 11, 220, 23, 681, 198, 262, 328, 48, 19, 62, 15, 763, 318, 18, 17, 11, 220, 16, 23, 681, 328, 48, 19, 62, 16, 763, 318, 18, 17, 11, 220, 17, 15, 681, 328, 48, 20, 62, 15, 763, 318, 18, 17, 11, 220, 17, 17, 681, 328, 48]}, {"name": "docs", "ids": [550, 3437, 369, 303, 279, 999, 271, 40087, 71590, 11, 220, 16, 21, 24, 11, 16, 21, 2Line truncated
|
||||
{"version": "v1", "seg_tokens": 1000, "segments": [{"name": "code", "ids": [12237, 85286, 54802, 19913, 343, 18, 4108, 6396, 471, 874, 50285, 980, 11, 874, 50285, 1644, 2195, 88, 13, 271, 34447, 1521, 279, 1510, 13985, 1644, 63, 5187, 1828, 6129, 4021, 1724, 411, 35426, 25, 1141, 1510, 21563, 2581, 42750, 1954, 900, 63, 198, 8746, 682, 874, 4314, 364, 913, 220, 19, 17, 27718, 48, 17, 62, 15, 63, 681, 864, 369, 24733, 1056, 279, 6522, 13, 2844, 369, 4924, 264, 387, 15, 782, 21, 198, 64919, 471, 279, 3443, 411, 4560, 369, 26208, 364, 369, 8047, 314, 279, 5129, 5224, 286, 13, 271, 1919, 369, 279, 5604, 7873, 364, 1510, 3431, 53683, 20123, 14, 63, 318, 77646, 16137, 16, 20, 25, 328, 21563, 19913, 343, 18, 6396, 318, 76, 2117, 11, 198, 2083, 50285, 980, 23285, 61406, 1049, 68434, 279, 4108, 1132, 26, 15167, 795, 369, 2496, 1301, 13, 198, 12237, 198, 1445, 1259, 20409, 548, 1120, 31144, 271, 464, 795, 8588, 198, 464, 51372, 198, 464, 1968, 198, 1445, 18904, 1120, 5586, 271, 21563, 19913, 47540, 283, 220, 15, 87, 19, 21, 20, 20, 19, 22, 19, 22, 220, 653, 328, 21563, 19913, 1, 2545, 12692, 1068, 271, 2, 50285, 980, 1756, 2663, 13, 220, 19, 17, 283, 1167, 17, 62, 15, 11, 279, 34100, 48, 12, 7146, 46, 69764, 17830, 517, 10845, 11, 26334, 494, 50285, 1644, 2195, 88, 430, 314, 220, 15, 13, 16, 24, 13, 198, 21563, 2581, 24973, 25, 6249, 18078, 11, 592, 60, 283, 313, 198, 262, 220, 15, 25, 328, 37, 18, 17, 487, 220, 16, 25, 328, 37, 16, 21, 487, 220, 17, 25, 328, 48, 19, 62, 15, 487, 220, 18, 25, 328, 48, 19, 62, 16, 487, 220, 19, 25, 328, 48, 19, 62, 17, 487, 220, 20, 25, 328, 48, 19, 62, 18, 487, 220, 21, 25, 328, 48, 20, 62, 15, 487, 220, 22, 25, 328, 48, 20, 62, 16, 487, 198, 262, 220, 23, 25, 328, 48, 23, 62, 15, 487, 220, 24, 25, 328, 48, 23, 62, 16, 487, 220, 16, 15, 25, 328, 48, 17, 9802, 487, 220, 16, 16, 25, 328, 48, 18, 9802, 487, 220, 16, 17, 25, 328, 48, 19, 9802, 487, 220, 16, 18, 25, 328, 48, 20, 9802, 487, 220, 16, 19, 25, 328, 48, 21, 9802, 487, 198, 262, 220, 16, 20, 25, 328, 48, 23, 9802, 487, 220, 16, 21, 25, 328, 59865, 17, 62, 5955, 50, 487, 220, 16, 22, 25, 328, 59865, 17, 6645, 50, 487, 220, 16, 23, 25, 328, 59865, 18, 62, 5955, 50, 487, 220, 16, 24, 25, 328, 59865, 16, 1063, 487, 220, 17, 15, 25, 328, 59865, 19, 1548, 43, 487, 198, 262, 220, 17, 16, 25, 328, 59865, 18, 1063, 487, 220, 17, 17, 25, 328, 59865, 17, 1063, 487, 220, 17, 18, 25, 328, 59865, 19, 6645, 50, 487, 220, 17, 19, 25, 328, 40, 23, 487, 220, 17, 20, 25, 328, 40, 16, 21, 487, 220, 17, 21, 25, 328, 40, 18, 17, 487, 220, 17, 22, 25, 328, 40, 21, 19, 487, 198, 262, 220, 17, 23, 25, 328, 37, 21, 19, 487, 220, 17, 24, 25, 328, 59865, 16, 1203, 487, 220, 18, 15, 25, 328, 19278, 16, 21, 487, 220, 18, 16, 25, 328, 48, 19, 62, 15, 62, 19, 62, 19, 487, 220, 18, 17, 25, 328, 48, 19, 62, 15, 62, 19, 62, 23, 487, 220, 18, 18, 25, 328, 48, 19, 62, 15, 62, 23, 62, 23, 487, 198, 262, 220, 18, 19, 25, 328, 51, 48, 16, 62, 15, 487, 220, 18, 20, 25, 328, 51, 48, 17, 62, 15, 487, 220, 18, 21, 25, 328, 59865, 19, 1548, 43, 62, 19, 62, 19, 487, 220, 18, 22, 25, 328, 59865, 19, 1548, 43, 62, 19, 62, 23, 487, 220, 18, 23, 25, 328, 59865, 19, 1548, 43, 62, 23, 62, 23, 487, 198, 262, 220, 18, 24, 25, 328, 28463, 11361, 19, 487, 220, 19, 15, 25, 328, 35125, 11361, 19, 487, 220, 19, 16, 25, 328, 48, 19, 62, 15, 62, 23, 62, 23, 487, 220, 19, 17, 25, 328, 48, 17, 62, 15, 487, 198, 92, 271, 2, 54802, 19913, 10836, 869, 913, 13746, 198, 21563, 19913, 57818, 283, 313, 198, 262, 220, 15, 25, 3376, 84, 23, 487, 220, 16, 681, 220, 16, 25, 3376, 72, 23, 487, 220, 16, 681, 220, 17, 25, 3376, 84, 16, 21, 487, 220, 17, 681, 220, 18, 25, 3376, 72, 16, 21, 487, 220, 17, 681, 220, 19, 25, 3376, 84, 18, 17, 487, 220, 19, 681, 220, 20, 25, 3376, 72, 18, 17, 487, 220, 19, 681, 198, 262, 220, 21, 25, 3376, 69, 18, 17, 487, 220, 19, 681, 220, 22, 25, 3376, 2554, 487, 220, 16, 681, 220, 23, 25, 3376, 889, 487, 2168, 681, 220, 24, 25, 3376, 1595, 487, 2168, 681, 198, 262, 220, 16, 15, 25, 3376, 84, 21, 19, 487, 220, 23, 681, 220, 16, 16, 25, 3376, 72, 21, 19, 487, 220, 23, 681, 220, 16, 17, 25, 3376, 69, 21, 19, 487, 220, 23, 681, 198, 92, 271, 2, 50285, 980, 2424, 16547, 364, 279, 3107, 56373, 411, 1558, 5533, 25, 318, 4425, 5255, 11, 5639, 791, 2424, 8, 198, 2, 1061, 369, 1092, 3520, 264, 791, 2308, 3677, 4763, 1698, 1716, 470, 11, 321, 424, 369, 279, 4954, 279, 61369, 4191, 13, 198, 38637, 57038, 83072, 25, 6249, 16793, 11, 13992, 18078, 11, 514, 4891, 283, 313, 198, 262, 328, 37, 18, 17, 763, 318, 16, 11, 220, 19, 681, 328, 37, 16, 21, 763, 318, 16, 11, 220, 17, 681, 328, 19278, 16, 21, 763, 318, 16, 11, 220, 17, 681, 328, 37, 21, 19, 763, 318, 16, 11, 220, 23, 681, 198, 262, 328, 48, 19, 62, 15, 763, 318, 18, 17, 11, 220, 16, 23, 681, 328, 48, 19, 62, 16, 763, 318, 18, 17, 11, 220, 17, 15, 681, 328, 48, 20, 62, 15, 763, 318, 18, 17, 11, 220, 17, 17, 681, 328, 48]}, {"name": "docs", "ids": [12, 2972, 48213, 318, 18232, 1354, 188912, 6326, 321, 24167, 6738, 6311, 382, 11, 27Line truncated
|
||||
@@ -0,0 +1,30 @@
|
||||
// include/strata/prefill/mmq_kq_abi.hpp - the interface between the engine (src/prefill/moe_mmq.cu) and strata_kq, the
|
||||
// shared library with the K-quant MMQ prompt kernels (src/prefill/mmq_kq_lib.cu). Bump the version when a function's
|
||||
// meaning changes; the argument block (ggml's mmq_args) is checked by its size.
|
||||
#pragma once
|
||||
|
||||
#define STRATA_KQ_ABI_VERSION 1
|
||||
|
||||
#if defined(_WIN32)
|
||||
#define STRATA_KQ_LIB_NAME "strata_kq.dll"
|
||||
#else
|
||||
#define STRATA_KQ_LIB_NAME "libstrata_kq.so"
|
||||
#endif
|
||||
|
||||
extern "C" {
|
||||
typedef int (*strata_kq_abi_fn)();
|
||||
typedef int (*strata_kq_init_fn)(int device);
|
||||
typedef int (*strata_kq_run_fn)(int device, int ggml_type, const void* mmq_args, void* stream);
|
||||
}
|
||||
|
||||
// the engine's side (src/prefill/mmq_kq_loader.cpp)
|
||||
namespace strata::prefill::mmq::kq {
|
||||
/// STRATA_MMQ_KQ=1: the K-quant experts' prompt products through MMQ (opt-in: other rounding than the FP16 path).
|
||||
bool wanted();
|
||||
/// wanted() and the library is open and started on every GPU (the first call opens it and says what happened).
|
||||
bool available();
|
||||
/// One product on the current GPU: 0 = launched, else an error (-1: no library or not this type).
|
||||
int run(int ggml_type, const void* mmq_args, void* stream);
|
||||
/// sizeof(mmq_args) on the engine's side (moe_mmq.cu), checked against the library's.
|
||||
int abi_args_size();
|
||||
} // namespace strata::prefill::mmq::kq
|
||||
@@ -0,0 +1,60 @@
|
||||
// src/prefill/mmq_kq_lib.cu - strata_kq, the shared library with llama.cpp's MMQ prompt kernels for the K-quants
|
||||
// (Q4_K, Q5_K, Q5_1, Q6_K: the experts of Unsloth's UD-Q4/Q5/Q6_K_XL files). MODEL-UPDATE Phase 1: the engine loads
|
||||
// every kernel of its own binary when it starts (CUDA_MODULE_LOADING=EAGER, src/program/generate.cpp), so kernels in
|
||||
// the engine cost every model VRAM; these four instances were ~3.8 MB of sm_60 code (3 expert slots of IQ3_XXS on a
|
||||
// P100). Here they load only when moe_mmq.cu opens this library - for a model with these experts, before the expert
|
||||
// cache is sized - and never otherwise. The library has its own copy of ggml_cuda_host.cu's symbols and of the static
|
||||
// CUDA runtime; it works in the engine's context on the stream it is given.
|
||||
#include "common.cuh"
|
||||
#include "mmq.cuh"
|
||||
|
||||
#include "strata/prefill/mmq_kq_abi.hpp"
|
||||
|
||||
#include <mutex>
|
||||
|
||||
#if defined(_WIN32)
|
||||
#define STRATA_KQ_EXPORT extern "C" __declspec(dllexport)
|
||||
#else
|
||||
#define STRATA_KQ_EXPORT extern "C" __attribute__((visibility("default")))
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
std::mutex g_mu;
|
||||
ggml_backend_cuda_context* g_ctx[GGML_CUDA_MAX_DEVICES] = {};
|
||||
|
||||
ggml_backend_cuda_context* ctx_for(int dev) {
|
||||
std::lock_guard<std::mutex> lk(g_mu);
|
||||
if (g_ctx[dev] == nullptr) g_ctx[dev] = new ggml_backend_cuda_context(dev);
|
||||
return g_ctx[dev];
|
||||
}
|
||||
} // namespace
|
||||
|
||||
// the interface version and the argument block's size: both sides are compiled from the same mmq.cuh
|
||||
STRATA_KQ_EXPORT int strata_kq_abi() { return STRATA_KQ_ABI_VERSION * 100000 + (int) sizeof(mmq_args); }
|
||||
|
||||
// Starts this library's CUDA runtime in the engine's context on `device`: with CUDA_MODULE_LOADING=EAGER (the engine
|
||||
// sets it before its first CUDA call) that loads every kernel here now, while the VRAM for it is still free.
|
||||
// 0 = ready, else the cudaError_t.
|
||||
STRATA_KQ_EXPORT int strata_kq_init(int device) {
|
||||
cudaError_t e = cudaSetDevice(device);
|
||||
if (e == cudaSuccess) e = cudaFree(nullptr);
|
||||
return (int) e;
|
||||
}
|
||||
|
||||
// One MMQ product (moe_mmq.cu's Context::run for these types) on `device`, on the engine's `stream`. 0 = launched, -1 =
|
||||
// a type this library does not have, else the cudaError_t of the launch.
|
||||
STRATA_KQ_EXPORT int strata_kq_run(int device, int type, const void* args, void* stream) {
|
||||
if (device < 0 || device >= GGML_CUDA_MAX_DEVICES) return -1;
|
||||
if (cudaSetDevice(device) != cudaSuccess) return (int) cudaGetLastError();
|
||||
ggml_backend_cuda_context& ctx = *ctx_for(device);
|
||||
const mmq_args& a = *(const mmq_args*) args;
|
||||
const cudaStream_t s = (cudaStream_t) stream;
|
||||
switch ((ggml_type) type) {
|
||||
case GGML_TYPE_Q4_K: mul_mat_q_case<GGML_TYPE_Q4_K>(ctx, a, s); break;
|
||||
case GGML_TYPE_Q5_K: mul_mat_q_case<GGML_TYPE_Q5_K>(ctx, a, s); break;
|
||||
case GGML_TYPE_Q5_1: mul_mat_q_case<GGML_TYPE_Q5_1>(ctx, a, s); break;
|
||||
case GGML_TYPE_Q6_K: mul_mat_q_case<GGML_TYPE_Q6_K>(ctx, a, s); break;
|
||||
default: return -1;
|
||||
}
|
||||
return (int) cudaGetLastError();
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
// src/prefill/mmq_kq_loader.cpp - opens strata_kq (src/prefill/mmq_kq_lib.cu), the library with the K-quant MMQ prompt
|
||||
// kernels, from the engine's own folder - once, for the first model format that asks for it (moe_mmq.cu's supported()),
|
||||
// which the engine does before it sizes the expert cache (src/program/generate.cpp). Without the library, or with an
|
||||
// interface of another version, the K-quants keep the FP16 prompt path and the engine says so once.
|
||||
#include "strata/prefill/mmq_kq_abi.hpp"
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
|
||||
#if defined(_WIN32)
|
||||
#ifndef NOMINMAX
|
||||
#define NOMINMAX
|
||||
#endif
|
||||
#define WIN32_LEAN_AND_MEAN
|
||||
#include <windows.h>
|
||||
#else
|
||||
#include <dlfcn.h>
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
|
||||
namespace strata::prefill::mmq::kq {
|
||||
namespace {
|
||||
|
||||
struct Lib {
|
||||
bool ok = false;
|
||||
strata_kq_run_fn run = nullptr;
|
||||
};
|
||||
|
||||
std::string own_folder() {
|
||||
#if defined(_WIN32)
|
||||
char p[MAX_PATH] = {};
|
||||
const DWORD n = GetModuleFileNameA(nullptr, p, MAX_PATH);
|
||||
std::string s(p, n);
|
||||
const size_t k = s.find_last_of("\\/");
|
||||
return k == std::string::npos ? std::string() : s.substr(0, k + 1);
|
||||
#else
|
||||
char p[4096] = {};
|
||||
const ssize_t n = readlink("/proc/self/exe", p, sizeof(p) - 1);
|
||||
std::string s(p, n > 0 ? (size_t) n : 0);
|
||||
const size_t k = s.find_last_of('/');
|
||||
return k == std::string::npos ? std::string() : s.substr(0, k + 1);
|
||||
#endif
|
||||
}
|
||||
|
||||
void* sym(void* h, const char* name) {
|
||||
#if defined(_WIN32)
|
||||
return reinterpret_cast<void*>(GetProcAddress((HMODULE) h, name));
|
||||
#else
|
||||
return dlsym(h, name);
|
||||
#endif
|
||||
}
|
||||
|
||||
Lib open_lib() {
|
||||
Lib L;
|
||||
const std::string path = own_folder() + STRATA_KQ_LIB_NAME;
|
||||
#if defined(_WIN32)
|
||||
void* h = (void*) LoadLibraryA(path.c_str());
|
||||
#else
|
||||
void* h = dlopen(path.c_str(), RTLD_NOW | RTLD_LOCAL);
|
||||
#endif
|
||||
if (h == nullptr) {
|
||||
std::fprintf(stderr, "strata: K-quant prompt kernels (STRATA_MMQ_KQ=1): %s not found next to the engine - "
|
||||
"these experts take the FP16 prompt path\n", STRATA_KQ_LIB_NAME);
|
||||
return L;
|
||||
}
|
||||
const auto abi = (strata_kq_abi_fn) sym(h, "strata_kq_abi");
|
||||
const auto init = (strata_kq_init_fn) sym(h, "strata_kq_init");
|
||||
L.run = (strata_kq_run_fn) sym(h, "strata_kq_run");
|
||||
if (abi == nullptr || init == nullptr || L.run == nullptr || abi() / 100000 != STRATA_KQ_ABI_VERSION ||
|
||||
abi() % 100000 != abi_args_size()) {
|
||||
std::fprintf(stderr, "strata: K-quant prompt kernels: %s is from another engine version - these experts take "
|
||||
"the FP16 prompt path\n", path.c_str());
|
||||
L.run = nullptr;
|
||||
return L;
|
||||
}
|
||||
// every GPU this engine sees: each context loads the library's kernels now, while the VRAM is still free
|
||||
int n = 0, cur = 0;
|
||||
cudaGetDevice(&cur);
|
||||
if (cudaGetDeviceCount(&n) != cudaSuccess) n = 0;
|
||||
for (int d = 0; d < n; ++d) {
|
||||
const int e = init(d);
|
||||
if (e != 0) {
|
||||
std::fprintf(stderr, "strata: K-quant prompt kernels: %s did not start on GPU %d (CUDA error %d) - these "
|
||||
"experts take the FP16 prompt path\n", path.c_str(), d, e);
|
||||
L.run = nullptr;
|
||||
cudaSetDevice(cur);
|
||||
return L;
|
||||
}
|
||||
}
|
||||
cudaSetDevice(cur);
|
||||
std::fprintf(stderr, "strata: K-quant prompt kernels from %s (STRATA_MMQ_KQ=1)\n", path.c_str());
|
||||
L.ok = true;
|
||||
return L;
|
||||
}
|
||||
|
||||
const Lib& lib() {
|
||||
static const Lib L = open_lib();
|
||||
return L;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
bool wanted() {
|
||||
static const bool on = [] {
|
||||
const char* v = std::getenv("STRATA_MMQ_KQ");
|
||||
return v != nullptr && v[0] == '1';
|
||||
}();
|
||||
return on;
|
||||
}
|
||||
|
||||
bool available() { return wanted() && lib().ok; }
|
||||
|
||||
int run(int type, const void* args, void* stream) {
|
||||
int dev = 0;
|
||||
cudaGetDevice(&dev);
|
||||
return lib().ok ? lib().run(dev, type, args, stream) : -1;
|
||||
}
|
||||
|
||||
} // namespace strata::prefill::mmq::kq
|
||||
+37
-9
@@ -7,11 +7,33 @@
|
||||
#include "mmq.cuh"
|
||||
#include "quantize.cuh"
|
||||
|
||||
#include "strata/prefill/mmq_kq_abi.hpp"
|
||||
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace strata::prefill::mmq {
|
||||
#if defined(STRATA_KQ_DSO)
|
||||
namespace kq {
|
||||
int abi_args_size() { return (int) sizeof(mmq_args); }
|
||||
} // namespace kq
|
||||
#endif
|
||||
namespace {
|
||||
#if defined(STRATA_MMQ_KQUANTS)
|
||||
// MODEL-UPDATE Phase 1: the K-quant products through MMQ are opt-in at run time (STRATA_MMQ_KQ=1); on CUDA their
|
||||
// kernels are the strata_kq library (mmq_kq_loader.cpp), opened only then
|
||||
bool kq_ok() {
|
||||
#if defined(STRATA_KQ_DSO)
|
||||
return kq::available();
|
||||
#else
|
||||
static const bool on = [] {
|
||||
const char* v = std::getenv("STRATA_MMQ_KQ");
|
||||
return v != nullptr && v[0] == '1';
|
||||
}();
|
||||
return on;
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
||||
void ck(cudaError_t e, const char* what) {
|
||||
if (e != cudaSuccess) {
|
||||
@@ -127,13 +149,12 @@ bool supported(int t) {
|
||||
case GGML_TYPE_IQ2_XXS: case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ2_S:
|
||||
case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ3_S: case GGML_TYPE_IQ4_NL: case GGML_TYPE_IQ4_XS:
|
||||
case GGML_TYPE_Q8_0: // the draft layer's dense matrices (E-9)
|
||||
#ifdef STRATA_MMQ_KQUANTS
|
||||
case GGML_TYPE_Q4_K: case GGML_TYPE_Q5_K: case GGML_TYPE_Q5_1: // Unsloth's UD-Q4_K_XL experts (CUDA)
|
||||
#if defined(__HIPCC__) || defined(STRATA_Q6K_EXPERTS)
|
||||
case GGML_TYPE_Q6_K: // HIP: the dense GGUF projections (STRATA_DENSE_MMQ); CUDA: only the opt-in -DSTRATA_Q6K_EXPERTS=ON build
|
||||
#endif
|
||||
#endif
|
||||
return true;
|
||||
#ifdef STRATA_MMQ_KQUANTS
|
||||
// Unsloth's UD-Q4/Q5/Q6_K_XL experts; on HIP also the dense GGUF projections (STRATA_DENSE_MMQ)
|
||||
case GGML_TYPE_Q4_K: case GGML_TYPE_Q5_K: case GGML_TYPE_Q5_1: case GGML_TYPE_Q6_K:
|
||||
return kq_ok();
|
||||
#endif
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
@@ -213,12 +234,19 @@ void Context::run(const Product& p, void* stream) {
|
||||
case GGML_TYPE_IQ4_NL: mul_mat_q_case<GGML_TYPE_IQ4_NL>(ctx, a, s); break;
|
||||
case GGML_TYPE_IQ4_XS: mul_mat_q_case<GGML_TYPE_IQ4_XS>(ctx, a, s); break;
|
||||
case GGML_TYPE_Q8_0: mul_mat_q_case<GGML_TYPE_Q8_0>(ctx, a, s); break;
|
||||
#ifdef STRATA_MMQ_KQUANTS
|
||||
#if defined(STRATA_MMQ_KQUANTS) && defined(STRATA_KQ_DSO)
|
||||
case GGML_TYPE_Q4_K: case GGML_TYPE_Q5_K: case GGML_TYPE_Q6_K: case GGML_TYPE_Q5_1: {
|
||||
const int e = kq::run((int) t, &a, s);
|
||||
if (e != 0) {
|
||||
std::fprintf(stderr, "prefill mmq: the K-quant library failed on type %d (error %d)\n", (int) t, e);
|
||||
std::exit(1);
|
||||
}
|
||||
break;
|
||||
}
|
||||
#elif defined(STRATA_MMQ_KQUANTS)
|
||||
case GGML_TYPE_Q4_K: mul_mat_q_case<GGML_TYPE_Q4_K>(ctx, a, s); break;
|
||||
case GGML_TYPE_Q5_K: mul_mat_q_case<GGML_TYPE_Q5_K>(ctx, a, s); break;
|
||||
#if defined(__HIPCC__) || defined(STRATA_Q6K_EXPERTS)
|
||||
case GGML_TYPE_Q6_K: mul_mat_q_case<GGML_TYPE_Q6_K>(ctx, a, s); break;
|
||||
#endif
|
||||
case GGML_TYPE_Q5_1: mul_mat_q_case<GGML_TYPE_Q5_1>(ctx, a, s); break;
|
||||
#endif
|
||||
default:
|
||||
|
||||
@@ -62,6 +62,9 @@
|
||||
#include "strata/core/verify.hpp"
|
||||
#include "strata/core/mtp.hpp"
|
||||
#include "strata/prefill/prefill.hpp"
|
||||
#if defined(STRATA_PREFILL_MMQ)
|
||||
#include "strata/prefill/moe_mmq.hpp"
|
||||
#endif
|
||||
#include "strata/core/native_dense.hpp"
|
||||
#include "strata/program/batch_read.hpp"
|
||||
#include "strata/program/logits_selection.hpp"
|
||||
@@ -2759,6 +2762,12 @@ int main(int argc, char** argv) {
|
||||
f.gu_type, f.d_type);
|
||||
return 1;
|
||||
}
|
||||
#if defined(STRATA_PREFILL_MMQ)
|
||||
// MODEL-UPDATE Phase 1: the K-quants' prompt kernels (STRATA_MMQ_KQ=1) are a library of their own on CUDA
|
||||
// (strata_kq): asking for the formats opens it now, before the expert cache takes the free VRAM
|
||||
(void) strata::prefill::mmq::supported(f.gu_type);
|
||||
(void) strata::prefill::mmq::supported(f.d_type);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
const bool native_pack = strata::kernels::cpu::expert_layout().native;
|
||||
|
||||
@@ -149,6 +149,12 @@ find_package(Threads REQUIRED)
|
||||
target_link_libraries(strata_kernels_cpu PUBLIC strata_includes Threads::Threads)
|
||||
|
||||
option(STRATA_NATIVE_EXPERTS "Native (IQ) expert support through ggml-cpu" ON)
|
||||
# Q6_K gate/up in the grouped expert kernels (UD-Q6_K_XL, layer 2 of UD-Q5_K_XL): on by default, as in the CUDA / HIP
|
||||
# build (MODEL-UPDATE Phase 1)
|
||||
option(STRATA_Q6K_EXPERTS "Build the Q6_K gate/up grouped expert kernels (UD-Q6_K_XL, UD-Q5_K_XL)" ON)
|
||||
if(STRATA_Q6K_EXPERTS)
|
||||
add_compile_definitions(STRATA_Q6K_EXPERTS=1)
|
||||
endif()
|
||||
set(STRATA_GGML_DIR "" CACHE PATH "llama.cpp checkout to take ggml from (empty: fetch the pinned commit)")
|
||||
if(STRATA_NATIVE_EXPERTS)
|
||||
set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
|
||||
|
||||
Reference in new issue
Block a user