From b7da5cc70680103baea35ff3f869ad135bcaf951 Mon Sep 17 00:00:00 2001 From: zhifu gao Date: Fri, 19 Jun 2026 12:06:42 +0800 Subject: [PATCH] Add llama.cpp / GGUF runtime for FunASR models (#2988) * Add llama.cpp/GGUF runtime for Fun-ASR-Nano, SenseVoice and Paraformer Run FunASR models entirely on the llama.cpp / ggml stack: CPU / edge, single binary, no Python at runtime, quantized GGUF weights (like whisper.cpp for Whisper). Adds runtime/llama.cpp/ with three models sharing a common ggml SAN-M encoder / FSMN / attention and kaldi fbank front end: - fun-asr-nano : SAN-M encoder + adaptor + Qwen3-0.6B LLM (audio embeds injected via llama_decode embedding input, llava/mtmd style); integrated WAV->text CLI. - sensevoice : SAN-M encoder + CTC; WAV->CTC ids. - paraformer : SAN-M encoder + CIF predictor + SAN-M decoder (non-autoregressive). Each ships a GGUF export script and a README (architecture, quickstart, accuracy, gotchas). Validated against the PyTorch reference (encoder cosine ~1.0; SenseVoice CTC ids identical; Paraformer text identical; Fun-ASR-Nano CER within 0.02% under identical conditions). Adds a new directory only; no existing code is modified. * docs: add comprehensive design document (architecture, GGUF format, validation, gotchas) * docs: link README to DESIGN.md * fix: address code review (WAV ch/bits validation, chunk guard, null/leak checks) Fixes gemini-code-assist findings: validate channels/bits in the WAV reader (reject non-16-bit/zero-channel instead of dividing by zero), guard chunk size >=1, null-check fopen/llama_context, and fclose/ggml_free on all paths. --------- Co-authored-by: LauraGPT --- runtime/llama.cpp/DESIGN.md | 337 ++++++++++++++++++ runtime/llama.cpp/README.md | 54 +++ runtime/llama.cpp/fun-asr-nano/README.md | 144 ++++++++ .../fun-asr-nano/export_encoder_gguf.py | 72 ++++ .../fun-asr-nano/funasr-cli/CMakeLists.txt | 5 + .../fun-asr-nano/funasr-cli/funasr-cli.cpp | 258 ++++++++++++++ .../fun-asr-nano/funasr-embd/CMakeLists.txt | 5 + .../fun-asr-nano/funasr-embd/funasr-embd.cpp | 141 ++++++++ .../funasr-encoder/CMakeLists.txt | 5 + .../funasr-encoder/funasr-encoder.cpp | 275 ++++++++++++++ runtime/llama.cpp/paraformer/README.md | 103 ++++++ .../llama.cpp/paraformer/detok_paraformer.py | 8 + .../paraformer/export_paraformer_gguf.py | 55 +++ .../funasr-paraformer/CMakeLists.txt | 5 + .../funasr-paraformer/funasr-paraformer.cpp | 213 +++++++++++ runtime/llama.cpp/sensevoice/README.md | 102 ++++++ runtime/llama.cpp/sensevoice/detok.py | 10 + .../sensevoice/export_sensevoice_gguf.py | 62 ++++ .../funasr-sensevoice/CMakeLists.txt | 5 + .../funasr-sensevoice/funasr-sensevoice.cpp | 179 ++++++++++ 20 files changed, 2038 insertions(+) create mode 100644 runtime/llama.cpp/DESIGN.md create mode 100644 runtime/llama.cpp/README.md create mode 100644 runtime/llama.cpp/fun-asr-nano/README.md create mode 100644 runtime/llama.cpp/fun-asr-nano/export_encoder_gguf.py create mode 100644 runtime/llama.cpp/fun-asr-nano/funasr-cli/CMakeLists.txt create mode 100644 runtime/llama.cpp/fun-asr-nano/funasr-cli/funasr-cli.cpp create mode 100644 runtime/llama.cpp/fun-asr-nano/funasr-embd/CMakeLists.txt create mode 100644 runtime/llama.cpp/fun-asr-nano/funasr-embd/funasr-embd.cpp create mode 100644 runtime/llama.cpp/fun-asr-nano/funasr-encoder/CMakeLists.txt create mode 100644 runtime/llama.cpp/fun-asr-nano/funasr-encoder/funasr-encoder.cpp create mode 100644 runtime/llama.cpp/paraformer/README.md create mode 100644 runtime/llama.cpp/paraformer/detok_paraformer.py create mode 100644 runtime/llama.cpp/paraformer/export_paraformer_gguf.py create mode 100644 runtime/llama.cpp/paraformer/funasr-paraformer/CMakeLists.txt create mode 100644 runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp create mode 100644 runtime/llama.cpp/sensevoice/README.md create mode 100644 runtime/llama.cpp/sensevoice/detok.py create mode 100644 runtime/llama.cpp/sensevoice/export_sensevoice_gguf.py create mode 100644 runtime/llama.cpp/sensevoice/funasr-sensevoice/CMakeLists.txt create mode 100644 runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp diff --git a/runtime/llama.cpp/DESIGN.md b/runtime/llama.cpp/DESIGN.md new file mode 100644 index 000000000..c427bedd4 --- /dev/null +++ b/runtime/llama.cpp/DESIGN.md @@ -0,0 +1,337 @@ +# FunASR on llama.cpp / GGUF — Design Document + +This document describes the design of the `runtime/llama.cpp` directory: a C++ / +ggml runtime that runs FunASR models (Fun-ASR-Nano, SenseVoiceSmall, Paraformer) +without PyTorch, on CPU and edge devices, with quantized GGUF weights. It is the +counterpart of [whisper.cpp](https://github.com/ggml-org/whisper.cpp) for FunASR. + +It is written to be read without the source: it explains *why* the runtime exists, +*how* each model maps onto ggml, the GGUF weight format, the numerical-fidelity and +validation methodology, the non-obvious gotchas discovered during the port, and the +roadmap. + +> This is the **shared design document** for the FunASR-on-llama.cpp effort and is +> kept identical across the FunASR family repos (modelscope/FunASR, Fun-ASR, +> SenseVoice). The three models share one ggml SAN-M encoder / FSMN / fbank +> foundation, so the design is documented once here in full; a single-model repo +> ships only the relevant model directory (§2) but the shared design still applies. + +--- + +## 1. Motivation + +FunASR's reference inference runs on PyTorch (and vLLM for the LLM-based models) on +GPU. That is the right tool for a server that batches many requests and wants to +saturate a GPU. It is the wrong tool when there is **no GPU and no Python**: a +laptop, a phone, a Raspberry Pi, an embedded C/C++ application, an offline desktop +app. There, you want a single self-contained binary, a few hundred MB of quantized +weights, and CPU SIMD. + +[llama.cpp](https://github.com/ggml-org/llama.cpp) / ggml is the de-facto runtime +for that world (Ollama, LM Studio, whisper.cpp all build on it). Porting FunASR to +ggml + GGUF makes FunASR run anywhere llama.cpp runs, dramatically widening the +deployment surface. + +| | PyTorch / vLLM (existing) | this runtime (llama.cpp) | +|---|---|---| +| target | GPU server, high QPS | CPU / edge / embedded | +| deps | Python + CUDA + PyTorch | none (C/C++ single binary) | +| weights | HF fp16/bf16 safetensors | GGUF, 2–8 bit quantization | +| key tech | PagedAttention, continuous batching | quantization, mmap, CPU SIMD | +| best for | online service, batch eval | offline, on-device, embedded | + +These are complementary, not competing: cloud serving stays on vLLM; this runtime +covers the on-device / offline case. + +--- + +## 2. System overview + +Three models are supported. They share more than they differ — all three use the +same **SAN-M encoder**, the same **FSMN memory block**, the same **kaldi-compatible +fbank front end**, and the same ggml building blocks. + +``` + ┌─────────────────────── shared C++ / ggml ───────────────────────┐ + audio.wav (16k mono) ──► kaldi 80-mel fbank + LFR(7/6) ──► SAN-M encoder (50 layers, ggml) + └──────────────────────────────────────────────────────────────────┘ + │ encoder_out [T, 512] + ┌───────────────────────────────────┼───────────────────────────────────┐ + Fun-ASR-Nano SenseVoiceSmall Paraformer + adaptor → audio embeds + 4 query tokens CIF predictor (host) + → inject into Qwen3-0.6B CTC head → greedy CTC → SAN-M decoder (cross-attn) + (llama_decode embd path) → SentencePiece → argmax → tokens.json + → text → text → text +``` + +| model | head / decoder | autoregressive? | output units | +|---|---|---|---| +| Fun-ASR-Nano | adaptor + Qwen3-0.6B LLM | yes (LLM) | Qwen3 BPE | +| SenseVoiceSmall | CTC | no | spectok BPE (25055) | +| Paraformer | CIF + SAN-M decoder | no (parallel) | char/BPE (8404) | + +Directory layout: +``` +runtime/llama.cpp/ + README.md overview + DESIGN.md this document + fun-asr-nano/ funasr-cli, funasr-encoder, funasr-embd, export_encoder_gguf.py + sensevoice/ funasr-sensevoice, export_sensevoice_gguf.py, detok.py + paraformer/ funasr-paraformer, export_paraformer_gguf.py, detok_paraformer.py +``` +Each model dir holds the llama.cpp example sources (drop-in under `examples/`), a +GGUF export script, and a model-specific README. + +--- + +## 3. Audio front end (kaldi fbank in C++) + +All models use FunASR's `WavFrontend`: kaldi-compatible 80-bin log-mel fbank with a +hamming window (25 ms / 10 ms), pre-emphasis 0.97, DC removal, 512-pt FFT, then +**Low-Frame-Rate (LFR)** stacking of 7 frames with stride 6 → a 560-dim feature +per output frame. + +The C++ implementation (`compute_fbank`) reproduces this exactly: +1. upscale the waveform by 32768 (FunASR feeds int16-range samples to kaldi), +2. per frame: remove DC offset, pre-emphasis, hamming window, zero-pad to 512, +3. radix-2 FFT, power spectrum, 80 triangular mel filters (kaldi mel: `1127·ln(1+f/700)`, + low 20 Hz, high 8000 Hz), log floor `FLT_EPSILON`, +4. LFR: left-pad 3 copies of frame 0, stack 7 frames stride 6 → 560-dim. + +**Validation:** vs torchaudio kaldi.fbank (dither=0), cosine **1.000000**, +max_abs_diff 1.75e-3. + +**Gotcha — dither.** FunASR's frontend uses `dither=1.0` by default, which adds +random noise per sample, so the fbank (and everything downstream) is *non-deterministic* +in the reference. The C++ front end uses dither=0 (deterministic). The model is +robust to this; it accounts for the small (<1%) cosine gap seen when comparing +against a dithered reference. + +--- + +## 4. The SAN-M encoder in ggml + +The SenseVoice/Paraformer encoder is a 50-layer (Paraformer) or 50+20-layer +(SenseVoice, with extra `tp_encoders`) **SAN-M** stack. Each layer is pre-norm: + +``` +x → LN → SAN-M self-attention → +residual → LN → FFN(relu) → +residual +``` + +SAN-M self-attention = standard multi-head attention **plus** an FSMN memory branch +that runs in parallel on the value projection and is added to the attention output: + +``` +q,k,v = split(linear_q_k_v(x)) # one fused projection +fsmn = FSMN(v) # depthwise conv over time + residual +attn = softmax(qkᵀ/√d)·v → linear_out +out = attn + fsmn +``` + +### 4.1 FSMN as an exact f32 shift-accumulate (design decision) + +FSMN is a per-channel (depthwise) 1-D convolution over time with a symmetric +kernel (size 11). ggml has `ggml_conv_1d_dw`, but it (a) requires the kernel in +F16 and (b) is flagged as "very likely wrong for some cases" upstream. Both are +unacceptable for a faithful port. + +Instead FSMN is implemented as an **exact f32 shift-accumulate**: the kernel is +exported as `[K, D]`, the value tensor is zero-padded by `(K-1)/2` on each side +along time, and the output is `Σ_j kernel[:,j] ⊙ pad(v)[:, t+j]` plus the residual. +This is 11 element-wise multiply-adds — exact in f32, no F16 rounding, no dependence +on the questionable conv kernel. It dropped the full-encoder max_abs_diff vs PyTorch +from 2.93 to **0.0052**. + +### 4.2 Position encoding & input scaling + +Input is pre-scaled by `√(d_model)=√512` then a sinusoidal position encoding is +added, with **depth = the input feature dim (560)** and **positions starting at 1** +(not 0) — both quirks of the FunASR encoder that must be matched exactly. + +### 4.3 LayerNorm + +eps = 1e-5 everywhere. + +**Validation:** first layer cosine 1.0 (max_abs_diff 1.8e-4); full encoder cosine +**1.000000**, max_abs_diff 5.2e-3 (f32). + +--- + +## 5. Per-model design + +### 5.1 Fun-ASR-Nano (encoder + adaptor + LLM) + +Pipeline: `fbank → encoder → adaptor → audio embeds [T', 1024] → inject into +Qwen3-0.6B → text`. + +- **LLM half is native.** Qwen3 is supported by llama.cpp, so the extracted + Qwen3-0.6B converts to GGUF with the stock `convert_hf_to_gguf.py` and runs + unchanged. +- **Embedding injection.** The audio embeddings are fed into the LLM through + `llama_decode`'s embedding-input path — exactly how llava/mtmd inject vision + embeddings. The integrated CLI builds the prompt as a *mixed* sequence: + `[prefix tokens | audio embeds | suffix tokens]`, where prefix/suffix are fed as + token ids (llama.cpp embeds them internally; `llama_tokenize(parse_special=true)` + reproduces the exact 18-token prefix) and the audio slot is fed as embeddings. +- **Low-frame-rate truncation (critical).** The adaptor emits `T'` frames, but the + model only uses the first `fake_token_len` of them as audio tokens, where + `fake_token_len` derives from the fbank length by a 3-stage `÷2` formula + (≈ T'/8). Feeding all `T'` frames is out-of-distribution and makes the LLM loop. +- **Chunking.** Decoding a long (e.g. 60 s) clip as one segment is OOD and triggers + greedy repetition; the CLI's `--chunk 15` splits into windows with a fresh KV per + window, dropping micro-CER from ~29% to ~9.5%. +- **Numerics.** The adaptor output has large magnitude (std ≈ 28, |max| ≈ 1187), so + fp16 can overflow; the runtime uses f32/f16 weights with f32 activations. + +### 5.2 SenseVoiceSmall (encoder + CTC) + +Pipeline: `fbank → prepend 4 query tokens → encoder → CTC head → greedy CTC → +SentencePiece`. + +- **Query tokens.** Four learned embeddings are prepended: `[language(auto), event, + emotion, textnorm]` (indices `[0,1,2,15]` for auto/woitn). They are 560-dim and + prepended *before* the encoder's `√512` scaling and position encoding. +- **CTC decode.** `argmax → collapse consecutive → drop blank(0)` → ids → SentencePiece. +- **Gotcha — no CMVN at inference.** SenseVoice's `inference()` feeds the **raw** + log-mel fbank to the encoder; it does **not** apply `am.mvn` CMVN (that code path + is unused at inference). Applying CMVN makes the model predict `<|nospeech|>`. + +**Validation:** CTC token ids **identical** to PyTorch (108/108 on a clip); text +matches `AutoModel` exactly. + +### 5.3 Paraformer (encoder + CIF + decoder, non-autoregressive) + +Pipeline: `fbank → CMVN → encoder → CIF predictor → acoustic embeds [N, 512] → +SAN-M decoder (cross-attn to encoder) → argmax → tokens.json`. + +- **CMVN IS applied** here (unlike SenseVoice): `(fbank + shift)·scale`, per-dim + (560), from `am.mvn`. +- **CIF predictor (runs on host).** Continuous Integrate-and-Fire: a 1-D conv + (k=3) + residual + relu + linear → sigmoid → per-frame weight α; then a sequential + integrate-and-fire loop emits one acoustic embedding each time the running α-sum + crosses 1.0. This both decides the token count and produces the decoder input. It + is inherently sequential, so it runs in plain C++ (cheap: ~0.5 G MACs); the + encoder and decoder run in ggml. +- **SAN-M decoder (ggml).** 16 layers, each: `FFN → FSMN self-attention → + cross-attention to the encoder output`. The self-attention is **FSMN-only** (no + QK attention); cross-attention has q from the decoder slots and k,v from a fused + `linear_k_v` of the encoder output. A 17th `decoders3` layer is FFN-only. The + decoder FFN has an internal LayerNorm and the second linear has no bias. The + layer ordering (FFN *before* the attention inside the residual) is unusual and is + matched exactly. + +**Validation:** decoded text **identical** to `AutoModel`; CIF token count exact +(105/105). Encoder cosine 0.997 (residual is the reference's random dither). + +**Gotcha — `am.mvn` has three bracketed blocks.** `[Splice idx]`, `[AddShift=shift]`, +`[Rescale=scale]`. The shift/scale are the two 560-length vectors; naively taking +the first two blocks grabs `[0]` as the shift and mis-scales everything, which makes +CIF emit ~4× too few tokens. Parse by length. + +--- + +## 6. GGUF conversion & weight layout + +Each model has an `export_*_gguf.py` that packs weights + architecture metadata into +a single GGUF. + +- Tensor names are kept verbatim from the checkpoint (e.g. + `encoder.encoders.3.norm1.weight`); the C++ looks them up by name. +- **FSMN kernels** are transposed from `(D,1,K)` to `[K,D]` at export so the C++ + shift-accumulate can take a contiguous per-tap `[D]` vector. +- **CMVN** (`am.mvn`) is parsed to `cmvn.shift` / `cmvn.scale` tensors (Paraformer + uses them; SenseVoice ships them but the runtime ignores them). +- **Quantization.** `--wtype f16` stores the 2-D matmul weights as F16 (norms, + biases, FSMN kernels stay f32), halving the encoder GGUF (e.g. 935 → 469 MB) with + cosine 0.999999. The Qwen3 LLM uses the standard llama.cpp quantizer (Q8_0 / Q4_K_M). + +| file | model | dtype | size | +|---|---|---|---| +| funasr-encoder.gguf | Nano | f32 / f16 | 935 / 469 MB | +| qwen3-0.6b-q8_0.gguf | Nano LLM | Q8_0 | 805 MB | +| sensevoice-small.gguf | SenseVoice | f32 | 936 MB | +| paraformer.gguf | Paraformer | f32 | 863 MB | + +--- + +## 7. Numerical fidelity & validation methodology + +The port is validated **stage by stage** against the PyTorch reference, using golden +dumps (fbank, encoder output, adaptor/CIF output, logits/ids) compared by cosine +similarity and max-abs-diff, then end-to-end by transcription text / CER. + +Summary of results (benchmark clip / set): + +| stage | metric | +|---|---| +| kaldi fbank vs torchaudio | cosine 1.000000 | +| SAN-M encoder (full) vs PyTorch | cosine 1.000000, max_abs_diff 5e-3 (f32) | +| SenseVoice CTC ids | identical (108/108) | +| Paraformer text / token count | identical / 105 = 105 | +| Fun-ASR-Nano end-to-end CER (same conditions) | C++ 11.68% vs PyTorch 11.70% (Δ0.02%) | + +**Why not bit-exact tokens everywhere?** Greedy decoding is chaotic: a ~5e-3 +difference (from ggml-CPU vs torch-GPU matmul summation order) can flip a token on +a borderline frame, and over a long sequence the paths diverge — *this also happens +between PyTorch's own GPU and CPU*. What is faithful and what we verify is (a) the +per-tensor numerics (cosine 1.0) and (b) the **aggregate CER**, which matches the +reference under identical conditions. + +**fp16 caution.** The Fun-ASR-Nano adaptor output magnitude (std ≈ 28) can overflow +fp16; the audio path is kept in f32 (weights may be f16, activations f32). + +--- + +## 8. Performance + +CPU, 8 threads, a 44 s clip: +- Encoder (50 layers): ~1.2 s. Paraformer decoder: ~0.5 s. +- Fun-ASR-Nano end-to-end (with LLM): ~7 s. +- Fully-quantized footprint (f16 encoder + Q8 LLM) ≈ 1.3 GB. + +These are first-correctness numbers; quantizing the encoder and threading/batching +the front end are open optimizations. + +--- + +## 9. Design decisions & trade-offs + +- **ggml for encoder/decoder, host C++ for CIF.** The neural matmul-heavy parts run + in ggml (SIMD, future GPU backends); CIF is a sequential scalar loop with data- + dependent control flow, so it is clearer and not slower in plain C++. +- **Exact f32 FSMN instead of `ggml_conv_1d_dw`.** Correctness and f32 precision + over reusing a flagged, F16-only op (§4.1). +- **Prompt as tokens, not a Python embedding table.** The integrated CLI tokenizes + the prompt with `llama_tokenize` and lets llama.cpp embed it, so no embedding + matrix needs to be shipped or matched (Fun-ASR-Nano). +- **f32 by default, f16/Q8 opt-in.** f32 is the faithful default; quantization is a + size/latency lever the user opts into. (Interestingly, Q8 on the LLM slightly + *helps* greedy stability by regularizing away from repetition loops.) +- **Per-model self-contained example dirs.** Mirrors llama.cpp's `examples/` layout + so each builds as a drop-in target; the shared code is duplicated rather than + factored to keep each example independently buildable. + +--- + +## 10. Limitations & roadmap + +- **WAV input** assumes 16 kHz mono PCM16; arbitrary formats / resampling are TODO. +- **VAD.** Long audio needs segmentation; today Fun-ASR-Nano uses fixed `--chunk` + windows. A real FSMN-VAD front end would close the last ~1.3% CER gap to the + production VAD-segmented number and is the highest-value next step. +- **Single packaged GGUF** (encoder + adaptor + LLM in one file) and a one-command + converter. +- **Encoder/decoder quantization** (Q8 via gguf-py quants), streaming, timestamps + (Paraformer CIF peaks give alignment; SenseVoice/Nano via CTC). +- **Upstream.** The example sources are drop-in for llama.cpp; upstreaming the + runtime to ggml-org/llama.cpp (as whisper.cpp-style tools) is a separate track. + +--- + +## 11. Reproducing the validation + +Each model dir's README has the build + convert + run quickstart. The export +scripts read a standard FunASR checkpoint (`model.pt` + `config.yaml` + `am.mvn` / +tokenizer). To reproduce a stage comparison, dump the corresponding PyTorch tensor +(`model.encode`, `model.calc_predictor`, …) and compare with cosine / max-abs-diff; +the numbers in §7 should reproduce within dither noise. diff --git a/runtime/llama.cpp/README.md b/runtime/llama.cpp/README.md new file mode 100644 index 000000000..f3bf03d65 --- /dev/null +++ b/runtime/llama.cpp/README.md @@ -0,0 +1,54 @@ +# FunASR on llama.cpp / GGUF + +Run FunASR models on the [llama.cpp](https://github.com/ggml-org/llama.cpp) / ggml +stack — **CPU, edge, a single binary, no Python at runtime, quantized weights**. +This is to FunASR what [whisper.cpp](https://github.com/ggml-org/whisper.cpp) is to +Whisper: it lets the models run where there is no GPU and no Python (laptops, +phones, edge boxes, embedded C/C++ apps), complementing the PyTorch / ONNX / vLLM +paths used for GPU serving. + +## Models + +| model | architecture | runtime | status | +|---|---|---|---| +| [Fun-ASR-Nano](fun-asr-nano/) | SenseVoice SAN-M encoder + adaptor + Qwen3-0.6B LLM | `llama-funasr-cli` | validated vs PyTorch | +| [SenseVoiceSmall](sensevoice/) | SAN-M encoder + CTC | `llama-funasr-sensevoice` | CTC ids identical to PyTorch | +| [Paraformer](paraformer/) | SAN-M encoder + CIF predictor + SAN-M decoder (non-autoregressive) | `llama-funasr-paraformer` | text identical to PyTorch | + +All three share the same ggml SAN-M encoder / FSMN / attention primitives and the +same kaldi-compatible fbank front end (80-mel, LFR 7/6), so the C++ is consistent +across models. + +## How it works + +Each model's neural path is implemented as a ggml graph; the audio front end (kaldi +fbank) is plain C++. Weights are converted to GGUF (f32 or f16) with the per-model +`export_*_gguf.py` script. For Fun-ASR-Nano the LLM half is a standard Qwen3 GGUF +and the audio embeddings are injected into it via `llama_decode`'s embedding input +(the llava/mtmd mechanism). See each model's README for the architecture diagram, +build/convert/run quickstart, validation numbers, and gotchas. + +## Build (shared) +```bash +git clone https://github.com/ggml-org/llama.cpp && cd llama.cpp +cp -r /path/to/runtime/llama.cpp// examples/ +echo 'add_subdirectory()' >> examples/CMakeLists.txt +cmake -B build -DGGML_NATIVE=ON -DLLAMA_CURL=OFF +cmake --build build -j --target +``` + +## Validation + +Each model was validated against the FunASR PyTorch reference (encoder cosine ≈ 1.0; +SenseVoice CTC token ids identical; Paraformer text identical; Fun-ASR-Nano aggregate +CER matches PyTorch within 0.02% under identical conditions). See per-model READMEs. + +## Status / notes +- WAV input currently assumes 16 kHz mono PCM16. +- For long audio, Fun-ASR-Nano supports `--chunk` windowing; a proper VAD front end + is on the roadmap. +- This adds a new `runtime/llama.cpp/` directory only; no existing code is modified. + +## Further reading + +See [DESIGN.md](DESIGN.md) for the full system design — architecture, the shared SAN-M encoder, GGUF weight format, numerical-fidelity and validation methodology, design trade-offs, and gotchas. diff --git a/runtime/llama.cpp/fun-asr-nano/README.md b/runtime/llama.cpp/fun-asr-nano/README.md new file mode 100644 index 000000000..64eefd675 --- /dev/null +++ b/runtime/llama.cpp/fun-asr-nano/README.md @@ -0,0 +1,144 @@ +# Fun-ASR-Nano on llama.cpp / GGUF + +Run **Fun-ASR-Nano** entirely on the [llama.cpp](https://github.com/ggml-org/llama.cpp) +/ ggml stack — **CPU, edge, a single binary, no Python at runtime**. This is to +Fun-ASR what [whisper.cpp](https://github.com/ggml-org/whisper.cpp) is to Whisper. + +## Why this exists + +Fun-ASR-Nano normally runs on PyTorch / vLLM (GPU). That is great for a server +serving many requests, but it cannot run where there is no GPU and no Python. +This runtime ports the model to **ggml + GGUF**, so Fun-ASR-Nano can run: + +- on a laptop / phone / Raspberry Pi / edge box, offline, CPU-only; +- embedded directly into a C/C++ application (one static binary); +- with quantized weights (Q8 / Q4), shrinking the model to ~1.3 GB total. + +| | vLLM (existing) | this runtime (llama.cpp) | +|---|---|---| +| target | GPU server, high QPS | CPU / edge / embedded | +| deps | Python + CUDA + PyTorch | none (C/C++ single binary) | +| weights | HF fp16/bf16 | GGUF, quantized | +| best for | online service, batch | offline, on-device | + +## Architecture + +Fun-ASR-Nano = **SenseVoice SAN-M encoder (70 layers) + adaptor + Qwen3-0.6B LLM**. +The whole pipeline runs in C++: + +``` + audio.wav (16k mono) + │ kaldi 80-mel fbank + LFR (C++) + ▼ + features [T, 560] + │ SAN-M encoder + adaptor (ggml) ── funasr-encoder.gguf + ▼ + audio embeds [T', 1024] + │ keep first fake_token_len frames (low-frame-rate) + ▼ + [ prefix tokens | audio embeds | suffix tokens ] + │ Qwen3-0.6B, embeds injected via llama_decode (llava/mtmd style) ── qwen3-0.6b.gguf + ▼ + transcription +``` + +The audio embeddings are fed into the LLM through `llama_decode`'s embedding-input +path — exactly how llava/mtmd inject vision embeddings. + +## Quickstart + +**1. Build** (drop the examples into a llama.cpp checkout): +```bash +git clone https://github.com/ggml-org/llama.cpp && cd llama.cpp +cp -r /path/to/runtime/llama.cpp/funasr-cli examples/ +echo 'add_subdirectory(funasr-cli)' >> examples/CMakeLists.txt +cmake -B build -DGGML_NATIVE=ON -DLLAMA_CURL=OFF +cmake --build build -j --target llama-funasr-cli +``` + +**2. Convert weights to GGUF** (one-time; needs the checkpoint, e.g. +`FunAudioLLM/Fun-ASR-Nano-2512`): +```bash +# LLM half — Qwen3-0.6B is natively supported by llama.cpp +python llama.cpp/convert_hf_to_gguf.py /Qwen3-0.6B-vllm \ + --outfile qwen3-0.6b-f32.gguf --outtype f32 +build/bin/llama-quantize qwen3-0.6b-f32.gguf qwen3-0.6b-q8_0.gguf Q8_0 # smaller, recommended + +# audio half — SenseVoice encoder + adaptor +python runtime/llama.cpp/export_encoder_gguf.py \ + --model_pt /model.pt --out funasr-encoder.gguf # f32, 935 MB +python runtime/llama.cpp/export_encoder_gguf.py \ + --model_pt /model.pt --out funasr-encoder-f16.gguf --wtype f16 # 469 MB +``` + +**3. Transcribe:** +```bash +build/bin/llama-funasr-cli \ + --enc funasr-encoder.gguf -m qwen3-0.6b-q8_0.gguf \ + -a audio.wav --chunk 15 +``` +Expected output (one of the benchmark clips): +``` +我想问我在滨海新区有房我一直没有照顾孩子但是我想要抚养权...你觉得这是正常的想法吗 +[done] 7.40s ; chunk=15s +``` + +## Models & sizes + +| file | dtype | size | +|---|---|---| +| funasr-encoder.gguf | f32 | 935 MB | +| funasr-encoder-f16.gguf | f16 (matmul weights) | 469 MB | +| qwen3-0.6b-f32.gguf | f32 | 3.0 GB | +| qwen3-0.6b-q8_0.gguf | Q8_0 | 805 MB | +| qwen3-0.6b-q4km.gguf | Q4_K_M | 484 MB | + +Fully-quantized config (f16 encoder + Q8 LLM) ≈ **1.3 GB**, edge-friendly. + +## Accuracy & validation + +Validated against the PyTorch reference on the 184-file benchmark: + +- **Encoder + adaptor (ggml) vs PyTorch:** cosine **1.000000**, max_abs_diff **5e-3** (f32). +- **kaldi fbank (C++) vs torchaudio:** cosine **1.000000**. +- **End-to-end CER, identical conditions (f32 LLM, 15 s chunking):** + C++ macro 17.41% / micro 11.68% vs PyTorch macro 17.42% / micro 11.70% + → aggregate CER matches to **0.02%**; the port is faithful. +- Best practical config (Q8 LLM + 15 s chunking): **micro-CER 9.51%** (production + VAD-segmented reference is ~8.2%; the gap is fixed-window vs VAD, not the port). + +## Tips & gotchas + +- **Use `--chunk 15`** for long audio. Decoding a whole 60 s clip in one segment is + out-of-distribution and makes greedy decoding loop; 15 s windows fix it + (micro-CER 29% → 9.5%). +- **Low-frame-rate truncation** is required: only the first `fake_token_len` + adaptor frames are real audio tokens. The CLI does this automatically; feeding + all frames makes the LLM repeat. +- **Use bf16/fp32, avoid fp16 for the audio path** — the adaptor output has large + magnitude (std ≈ 28, |max| ≈ 1187); fp16 can overflow. The GGUFs here are f32/f16 + weights with f32 activations, which is safe. +- **WAV input** currently assumes 16 kHz mono PCM16. Resample first if needed. +- Q8 quantization slightly *helps* greedy stability (quant noise regularizes away + from repetition loops), so Q8 is a good default. + +## Implementation notes + +- FSMN depthwise memory is an exact f32 shift-accumulate (avoids the F16-only, + upstream-flagged `ggml_conv_1d_dw`). +- LayerNorm eps = 1e-5; sinusoidal position encoding depth = input feature dim (560), + positions start at 1; encoder input pre-scaled by sqrt(512). +- Prompt is fed as tokens via `llama_tokenize(parse_special=true)` (prefix = 18 + tokens, matching the HF tokenizer), so no Python embedding table is needed. + +## Files +``` +funasr-cli/ integrated binary: WAV → transcription +funasr-encoder/ encoder+adaptor only (ggml) — validation/debugging +funasr-embd/ LLM decode from precomputed embeds — validation/debugging +export_encoder_gguf.py export the audio encoder + adaptor to GGUF +``` + +## Roadmap +- True FSMN-VAD segmentation (replace fixed windows; closes the last ~1.3% CER). +- Arbitrary WAV formats / resampling; encoder Q8 quantization; single packaged GGUF. diff --git a/runtime/llama.cpp/fun-asr-nano/export_encoder_gguf.py b/runtime/llama.cpp/fun-asr-nano/export_encoder_gguf.py new file mode 100644 index 000000000..2804200f1 --- /dev/null +++ b/runtime/llama.cpp/fun-asr-nano/export_encoder_gguf.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Export Fun-ASR-Nano audio encoder + adaptor weights to a GGUF file. + +Packs all `audio_encoder.*` and `audio_adaptor.*` tensors (bf16 -> f32) plus +architecture metadata into funasr-encoder.gguf, for the ggml C++ forward pass. +Tensor names are kept verbatim (e.g. audio_encoder.encoders.3.norm1.weight) so +the C++ side can look them up directly. +""" +import argparse, os +import numpy as np +import torch +import gguf + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--model_pt", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--wtype", default="f32", choices=["f32", "f16"], + help="dtype for 2D Linear (matmul) weights; norms/bias/fsmn stay f32") + args = ap.parse_args() + + sd = torch.load(args.model_pt, map_location="cpu") + sd = sd.get("state_dict", sd) + + w = gguf.GGUFWriter(args.out, "funasr-sensevoice-encoder") + + # --- architecture metadata (from config.yaml) --- + w.add_uint32("funasr.enc.input_size", 560) # lfr_m(7) * n_mels(80) + w.add_uint32("funasr.enc.output_size", 512) + w.add_uint32("funasr.enc.attention_heads", 4) + w.add_uint32("funasr.enc.linear_units", 2048) + w.add_uint32("funasr.enc.num_blocks", 50) # encoders0(1) + encoders(49) + w.add_uint32("funasr.enc.tp_blocks", 20) + w.add_uint32("funasr.enc.kernel_size", 11) + w.add_uint32("funasr.enc.sanm_shfit", 0) + w.add_uint32("funasr.adp.llm_dim", 1024) + w.add_uint32("funasr.adp.encoder_dim", 512) + w.add_uint32("funasr.adp.ffn_dim", 2048) + w.add_uint32("funasr.adp.n_layer", 2) + w.add_uint32("funasr.adp.attention_heads", 8) + w.add_uint32("funasr.adp.downsample_rate", 1) + w.add_uint32("funasr.frontend.n_mels", 80) + w.add_uint32("funasr.frontend.lfr_m", 7) + w.add_uint32("funasr.frontend.lfr_n", 6) + + n = 0 + for k, v in sd.items(): + if not (k.startswith("audio_encoder.") or k.startswith("audio_adaptor.")): + continue + arr = v.detach().to(torch.float32).contiguous().numpy() + # FSMN depthwise kernel: store as (K, D) so the C++ side can slice a + # contiguous per-tap [D] vector and do an exact f32 shift-accumulate + # (avoids the F16-only ggml_conv_1d_dw path). + if k.endswith("fsmn_block.weight"): # (D, 1, K) -> (K, D) + arr = np.ascontiguousarray(arr[:, 0, :].T) + # matmul (Linear) weights -> optional f16; norms/biases/fsmn stay f32 + elif args.wtype == "f16" and arr.ndim == 2 and "norm" not in k: + arr = arr.astype(np.float16) + w.add_tensor(k, arr) + n += 1 + print(f"writing {n} tensors to {args.out}") + + w.write_header_to_file() + w.write_kv_data_to_file() + w.write_tensors_to_file() + w.close() + print(f"done: {args.out} ({os.path.getsize(args.out)/1e6:.1f} MB)") + + +if __name__ == "__main__": + main() diff --git a/runtime/llama.cpp/fun-asr-nano/funasr-cli/CMakeLists.txt b/runtime/llama.cpp/fun-asr-nano/funasr-cli/CMakeLists.txt new file mode 100644 index 000000000..6c2d09d33 --- /dev/null +++ b/runtime/llama.cpp/fun-asr-nano/funasr-cli/CMakeLists.txt @@ -0,0 +1,5 @@ +set(TARGET llama-funasr-cli) +add_executable(${TARGET} funasr-cli.cpp) +install(TARGETS ${TARGET} RUNTIME) +target_link_libraries(${TARGET} PRIVATE llama ggml ${CMAKE_THREAD_LIBS_INIT}) +target_compile_features(${TARGET} PRIVATE cxx_std_17) diff --git a/runtime/llama.cpp/fun-asr-nano/funasr-cli/funasr-cli.cpp b/runtime/llama.cpp/fun-asr-nano/funasr-cli/funasr-cli.cpp new file mode 100644 index 000000000..f63fdd339 --- /dev/null +++ b/runtime/llama.cpp/fun-asr-nano/funasr-cli/funasr-cli.cpp @@ -0,0 +1,258 @@ +// funasr-cli: end-to-end Fun-ASR-Nano in C++ on the llama.cpp / ggml stack. +// +// wav(16k mono) -> kaldi fbank -> SAN-M encoder + adaptor (ggml) -> +// low-frame-rate truncation -> [prefix tokens | audio embeds | suffix tokens] +// -> Qwen3 LLM (llama.cpp) -> transcription. +// +// This is the whisper.cpp-style single-binary path: no Python at runtime. +// +// funasr-cli --enc funasr-encoder.gguf -m qwen3-0.6b.gguf -a audio.wav + +#include "ggml.h" +#include "ggml-cpu.h" +#include "ggml-alloc.h" +#include "ggml-backend.h" +#include "gguf.h" +#include "llama.h" + +#include +#include +#include +#include +#include +#include +#include + +// ======================= WAV (16 kHz mono PCM16) ======================= +static bool read_wav16(const char * path, std::vector & out) { + FILE * f = fopen(path, "rb"); if (!f) return false; + char riff[4]; fread(riff, 1, 4, f); + if (strncmp(riff, "RIFF", 4)) { fclose(f); return false; } + fseek(f, 4, SEEK_CUR); char wave[4]; fread(wave, 1, 4, f); + if (strncmp(wave, "WAVE", 4)) { fclose(f); return false; } + uint16_t ch = 1, bits = 16; uint32_t sr = 16000; + while (!feof(f)) { + char id[4]; if (fread(id, 1, 4, f) != 4) break; + uint32_t sz; if (fread(&sz, 4, 1, f) != 1) break; + if (!strncmp(id, "fmt ", 4)) { + uint16_t fmt; fread(&fmt, 2, 1, f); fread(&ch, 2, 1, f); + fread(&sr, 4, 1, f); fseek(f, 6, SEEK_CUR); fread(&bits, 2, 1, f); + if (sz > 16) fseek(f, sz - 16, SEEK_CUR); + } else if (!strncmp(id, "data", 4)) { + if (ch < 1 || bits != 16) { fclose(f); fprintf(stderr, "only 16-bit PCM WAV supported (ch=%u bits=%u)\n", ch, bits); return false; } + int n = sz / (bits / 8); + std::vector pcm(n); fread(pcm.data(), 2, n, f); + out.resize(n / ch); + for (int i = 0; i < n / ch; i++) out[i] = pcm[i * ch] / 32768.0f; // ch0 + fclose(f); + if (sr != 16000) fprintf(stderr, "warning: sample rate %u != 16000\n", sr); + return true; + } else fseek(f, sz, SEEK_CUR); + } + fclose(f); return false; +} + +// ======================= kaldi fbank + LFR ======================= +static const int FS=16000, WINLEN=400, SHIFT=160, NFFT=512, NMEL=80, LFR_M=7, LFR_N=6; +static const float PREEMPH=0.97f, LOWF=20.0f, HIGHF=8000.0f; +static inline float mel(float f){ return 1127.0f*logf(1.0f+f/700.0f); } +static void fft(std::vector&re,std::vector&im,int n){ + for(int i=1,j=0;i>1;for(;j&b;b>>=1)j^=b;j^=b;if(i compute_fbank(std::vector wav, int & T_out) { + for (auto & v : wav) v *= 32768.0f; + std::vector win(WINLEN); + for (int i=0;i> fb(NMEL, std::vector(NBIN,0.0f)); + for(int m=0;mL&&mf> feat(T, std::vector(NMEL)); + std::vector re(NFFT),im(NFFT),fr(WINLEN); + const float fl=1.1920929e-07f; + for(int t=0;t0;i--)fr[i]-=PREEMPH*fr[i-1];fr[0]-=PREEMPH*fr[0]; + for(int i=0;i0)e+=fb[m][k]*(re[k]*re[k]+im[k]*im[k]); + feat[t][m]=logf(e>fl?e:fl);}} + // LFR + const int pad=(LFR_M-1)/2; int T_lfr=(T+LFR_N-1)/LFR_N; + std::vector> pd; pd.reserve(T+pad+LFR_M); + for(int i=0;i out((size_t)T_lfr*D); + for(int i=0;i t; + ggml_tensor* g(const std::string&n){auto it=t.find(n);if(it==t.end()){fprintf(stderr,"missing %s\n",n.c_str());exit(1);}return it->second;} }; +static const float LN_EPS=1e-5f; +static bool load_enc(const char*p, enc_model&m){ + gguf_init_params gp={false,&m.ctx_w}; gguf_context*g=gguf_init_from_file(p,gp); if(!g)return false; + auto rd=[&](const char*k,int d){int i=gguf_find_key(g,k);return i<0?d:(int)gguf_get_val_u32(g,i);}; + m.c.d_model=rd("funasr.enc.output_size",512); m.c.n_head=rd("funasr.enc.attention_heads",4); + m.c.num_blocks=rd("funasr.enc.num_blocks",50); m.c.tp_blocks=rd("funasr.enc.tp_blocks",20); + m.c.kernel=rd("funasr.enc.kernel_size",11); m.c.adp_llm=rd("funasr.adp.llm_dim",1024); + m.c.adp_layers=rd("funasr.adp.n_layer",2); m.c.adp_head=rd("funasr.adp.attention_heads",8); + int n=gguf_get_n_tensors(g); for(int i=0;inb[1]; + ggml_tensor*q=ggml_cont(c,ggml_view_2d(c,qkv,D,T,nb1,0)); + ggml_tensor*k=ggml_cont(c,ggml_view_2d(c,qkv,D,T,nb1,(size_t)D*sizeof(float))); + ggml_tensor*v=ggml_cont(c,ggml_view_2d(c,qkv,D,T,nb1,(size_t)2*D*sizeof(float))); + const int pad=(K-1)/2; ggml_tensor*fk=m.g(p+"fsmn_block.weight"); + ggml_tensor*vp=ggml_pad_ext(c,v,0,0,pad,pad,0,0,0,0); ggml_tensor*fsmn=v; + for(int j=0;jnb[1],(size_t)j*vp->nb[1]); + auto wj=ggml_view_1d(c,fk,D,(size_t)j*fk->nb[1]); fsmn=ggml_add(c,fsmn,ggml_mul(c,ggml_cont(c,sl),wj));} + q=ggml_permute(c,ggml_reshape_3d(c,q,dk,H,T),0,2,1,3); k=ggml_permute(c,ggml_reshape_3d(c,k,dk,H,T),0,2,1,3); + ggml_tensor*vh=ggml_cont(c,ggml_permute(c,ggml_reshape_3d(c,v,dk,H,T),1,2,0,3)); + ggml_tensor*kq=ggml_soft_max(c,ggml_scale(c,ggml_mul_mat(c,k,q),1.0f/sqrtf((float)dk))); + ggml_tensor*o=ggml_cont_2d(c,ggml_permute(c,ggml_mul_mat(c,vh,kq),0,2,1,3),D,T); + return ggml_add(c,lin(c,m.g(p+"linear_out.weight"),m.g(p+"linear_out.bias"),o),fsmn); +} +static ggml_tensor* sanm_layer(ggml_context*c,enc_model&m,const std::string&p,ggml_tensor*x,int T,bool res){ + auto r=x; auto h=lnorm(c,x,m.g(p+"norm1.weight"),m.g(p+"norm1.bias")); + auto sa=sanm_attn(c,m,p+"self_attn.",h,T); x=res?ggml_add(c,r,sa):sa; r=x; + h=lnorm(c,x,m.g(p+"norm2.weight"),m.g(p+"norm2.bias")); + h=lin(c,m.g(p+"feed_forward.w_1.weight"),m.g(p+"feed_forward.w_1.bias"),h); h=ggml_relu(c,h); + h=lin(c,m.g(p+"feed_forward.w_2.weight"),m.g(p+"feed_forward.w_2.bias"),h); return ggml_add(c,r,h); +} +static ggml_tensor* adp_layer(ggml_context*c,enc_model&m,const std::string&p,ggml_tensor*x,int T){ + const int D=m.c.adp_llm,H=m.c.adp_head,dk=D/H; auto r=x; + auto h=lnorm(c,x,m.g(p+"norm1.weight"),m.g(p+"norm1.bias")); + auto q=ggml_permute(c,ggml_reshape_3d(c,lin(c,m.g(p+"self_attn.linear_q.weight"),m.g(p+"self_attn.linear_q.bias"),h),dk,H,T),0,2,1,3); + auto k=ggml_permute(c,ggml_reshape_3d(c,lin(c,m.g(p+"self_attn.linear_k.weight"),m.g(p+"self_attn.linear_k.bias"),h),dk,H,T),0,2,1,3); + auto vh=ggml_cont(c,ggml_permute(c,ggml_reshape_3d(c,lin(c,m.g(p+"self_attn.linear_v.weight"),m.g(p+"self_attn.linear_v.bias"),h),dk,H,T),1,2,0,3)); + auto kq=ggml_soft_max(c,ggml_scale(c,ggml_mul_mat(c,k,q),1.0f/sqrtf((float)dk))); + auto o=ggml_cont_2d(c,ggml_permute(c,ggml_mul_mat(c,vh,kq),0,2,1,3),D,T); + x=ggml_add(c,r,lin(c,m.g(p+"self_attn.linear_out.weight"),m.g(p+"self_attn.linear_out.bias"),o)); r=x; + h=lnorm(c,x,m.g(p+"norm2.weight"),m.g(p+"norm2.bias")); + h=lin(c,m.g(p+"feed_forward.w_1.weight"),m.g(p+"feed_forward.w_1.bias"),h); h=ggml_relu(c,h); + h=lin(c,m.g(p+"feed_forward.w_2.weight"),m.g(p+"feed_forward.w_2.bias"),h); return ggml_add(c,r,h); +} +static void add_posenc(std::vector&x,int T,int depth){ + double inc=log(10000.0)/(depth/2.0-1.0); + for(int t=0;t adaptor out [T x adp_llm] row-major +static std::vector run_encoder(enc_model&m,std::vector fbank,int T,int F,int&Dout){ + float sc=sqrtf((float)m.c.d_model); for(auto&v:fbank)v*=sc; add_posenc(fbank,T,F); + ggml_backend_t be=ggml_backend_cpu_init(); + ggml_init_params cp={(size_t)1024*1024*1024,nullptr,true}; ggml_context*c=ggml_init(cp); + ggml_tensor*inp=ggml_new_tensor_2d(c,GGML_TYPE_F32,F,T); ggml_set_input(inp); + ggml_tensor*x=sanm_layer(c,m,"audio_encoder.encoders0.0.",inp,T,false); + for(int i=0;ine[0]; std::vector out((size_t)Dout*T); ggml_backend_tensor_get(x,out.data(),0,ggml_nbytes(x)); + ggml_gallocr_free(ga); ggml_free(c); ggml_backend_free(be); return out; +} + +// ======================= LLM (llama.cpp) ======================= +static int decode_batch(llama_context*ctx,int n,llama_token*tok,float*embd,int n_embd,int&n_past,bool last_logits){ + std::vector pos(n); std::vector nsid(n,1); + std::vector s0(1,0); std::vector sid(n); std::vector lg(n,0); + for(int i=0;i wav; + if(!read_wav16(wav_path.c_str(),wav)){fprintf(stderr,"failed to read wav\n");return 1;} + int64_t t0=ggml_time_us(); + + enc_model em; if(!load_enc(enc_path.c_str(),em))return 1; + ggml_backend_load_all(); + llama_model_params mp=llama_model_default_params(); mp.n_gpu_layers=0; + llama_model*model=llama_model_load_from_file(llm_path.c_str(),mp); if(!model)return 1; + const llama_vocab*vocab=llama_model_get_vocab(model); + llama_context_params cp=llama_context_default_params(); + cp.n_ctx=2048; cp.n_batch=2048; cp.n_ubatch=2048; + llama_context*ctx=llama_init_from_model(model,cp); + if(!ctx){fprintf(stderr,"failed to create llama context\n");llama_model_free(model);return 1;} + auto sp=llama_sampler_chain_default_params(); llama_sampler*smpl=llama_sampler_chain_init(sp); + if(rep!=1.0f) llama_sampler_chain_add(smpl,llama_sampler_init_penalties(256,rep,0.0f,0.0f)); + llama_sampler_chain_add(smpl,llama_sampler_init_greedy()); + + const char*prefix="<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n语音转写："; + const char*suffix="<|im_end|>\n<|im_start|>assistant\n"; + auto tokenize=[&](const char*s){int n=-llama_tokenize(vocab,s,strlen(s),nullptr,0,false,true); + std::vector v(n); llama_tokenize(vocab,s,strlen(s),v.data(),n,false,true); return v;}; + auto pre=tokenize(prefix); auto suf=tokenize(suffix); + + // split into chunks (chunk_sec<=0 -> whole file) + int chunk_n = chunk_sec > 0 ? std::max(1, (int)(chunk_sec*16000)) : (int)wav.size(); + std::string full; + for (size_t off = 0; off < wav.size(); off += chunk_n) { + int len = std::min((size_t)chunk_n, wav.size()-off); + if (len < WINLEN) break; // too short for one frame + std::vector seg(wav.begin()+off, wav.begin()+off+len); + int T=0; auto fbank=compute_fbank(seg,T); + int D=0; auto adp=run_encoder(em,fbank,T,560,D); + int ol=1+(T-3+2)/2; ol=1+(ol-3+2)/2; int n_aud=(ol-1)/2+1; + + llama_memory_clear(llama_get_memory(ctx), true); // fresh context per chunk + int n_past=0; + decode_batch(ctx,pre.size(),pre.data(),nullptr,0,n_past,false); + decode_batch(ctx,n_aud,nullptr,adp.data(),D,n_past,false); + decode_batch(ctx,suf.size(),suf.data(),nullptr,0,n_past,true); + llama_token tk=llama_sampler_sample(smpl,ctx,-1); + for(int i=0;i0) full.append(buf,k); + decode_batch(ctx,1,&tk,nullptr,0,n_past,true); + tk=llama_sampler_sample(smpl,ctx,-1); + } + } + printf("%s\n", full.c_str()); + int64_t t2=ggml_time_us(); + fprintf(stderr,"[done] %.2fs ; chunk=%.0fs\n",(t2-t0)/1e6, chunk_sec); + llama_sampler_free(smpl); llama_free(ctx); llama_model_free(model); + if(em.ctx_w) ggml_free(em.ctx_w); + return 0; +} diff --git a/runtime/llama.cpp/fun-asr-nano/funasr-embd/CMakeLists.txt b/runtime/llama.cpp/fun-asr-nano/funasr-embd/CMakeLists.txt new file mode 100644 index 000000000..2d682d00d --- /dev/null +++ b/runtime/llama.cpp/fun-asr-nano/funasr-embd/CMakeLists.txt @@ -0,0 +1,5 @@ +set(TARGET llama-funasr-embd) +add_executable(${TARGET} funasr-embd.cpp) +install(TARGETS ${TARGET} RUNTIME) +target_link_libraries(${TARGET} PRIVATE llama ${CMAKE_THREAD_LIBS_INIT}) +target_compile_features(${TARGET} PRIVATE cxx_std_17) diff --git a/runtime/llama.cpp/fun-asr-nano/funasr-embd/funasr-embd.cpp b/runtime/llama.cpp/fun-asr-nano/funasr-embd/funasr-embd.cpp new file mode 100644 index 000000000..851b94a6c --- /dev/null +++ b/runtime/llama.cpp/fun-asr-nano/funasr-embd/funasr-embd.cpp @@ -0,0 +1,141 @@ +// funasr-embd: decode Fun-ASR-Nano audio embeddings through a Qwen3 GGUF. +// +// Reads an inputs_embeds matrix (produced by the FunASR audio encoder+adaptor, +// concatenated with the text prompt embeddings) and feeds it directly to the +// LLM via llama_decode's embedding input path -- the same mechanism llava/mtmd +// use to inject vision embeddings. This bridges FunASR's audio frontend to the +// llama.cpp / GGUF ecosystem. +// +// embeds.bin format: int32 n_tokens, int32 n_embd, then n_tokens*n_embd float32 +// (row-major). n_embd must equal the model's input embedding dim. + +#include "llama.h" + +#include +#include +#include +#include +#include + +static void print_usage(char ** argv) { + printf("\nusage: %s -m model.gguf -e embeds.bin [-n n_predict] [-ngl n_gpu_layers]\n\n", argv[0]); +} + +// read embeds.bin -> (n_tokens, n_embd, data) +static bool read_embeds(const std::string & path, int & n_tokens, int & n_embd, std::vector & data) { + FILE * f = fopen(path.c_str(), "rb"); + if (!f) { fprintf(stderr, "error: cannot open %s\n", path.c_str()); return false; } + int32_t hdr[2]; + if (fread(hdr, sizeof(int32_t), 2, f) != 2) { fclose(f); return false; } + n_tokens = hdr[0]; + n_embd = hdr[1]; + if (n_tokens <= 0 || n_embd <= 0) { fclose(f); return false; } + data.resize((size_t) n_tokens * n_embd); + size_t got = fread(data.data(), sizeof(float), data.size(), f); + fclose(f); + if (got != data.size()) { + fprintf(stderr, "error: short read (%zu/%zu floats)\n", got, data.size()); + return false; + } + return true; +} + +int main(int argc, char ** argv) { + std::string model_path, embeds_path; + int n_predict = 512; + int ngl = 0; // CPU by default; the whole point is CPU/edge + + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "-m") && i + 1 < argc) model_path = argv[++i]; + else if (!strcmp(argv[i], "-e") && i + 1 < argc) embeds_path = argv[++i]; + else if (!strcmp(argv[i], "-n") && i + 1 < argc) n_predict = std::stoi(argv[++i]); + else if (!strcmp(argv[i], "-ngl") && i + 1 < argc) ngl = std::stoi(argv[++i]); + else { print_usage(argv); return 1; } + } + if (model_path.empty() || embeds_path.empty()) { print_usage(argv); return 1; } + + int n_tokens = 0, n_embd_in = 0; + std::vector embd; + if (!read_embeds(embeds_path, n_tokens, n_embd_in, embd)) return 1; + fprintf(stderr, "loaded embeds: n_tokens=%d n_embd=%d\n", n_tokens, n_embd_in); + + ggml_backend_load_all(); + + llama_model_params mparams = llama_model_default_params(); + mparams.n_gpu_layers = ngl; + llama_model * model = llama_model_load_from_file(model_path.c_str(), mparams); + if (!model) { fprintf(stderr, "error: unable to load model\n"); return 1; } + + const llama_vocab * vocab = llama_model_get_vocab(model); + const int n_embd_model = llama_model_n_embd_inp(model); + if (n_embd_in != n_embd_model) { + fprintf(stderr, "error: embd dim %d != model input embd dim %d\n", n_embd_in, n_embd_model); + llama_model_free(model); + return 1; + } + + llama_context_params cparams = llama_context_default_params(); + cparams.n_ctx = n_tokens + n_predict + 8; + cparams.n_batch = n_tokens + 8; // process the whole embd prompt in one ubatch + cparams.n_ubatch = n_tokens + 8; + cparams.no_perf = false; + llama_context * ctx = llama_init_from_model(model, cparams); + if (!ctx) { fprintf(stderr, "error: failed to create context\n"); llama_model_free(model); return 1; } + + auto sparams = llama_sampler_chain_default_params(); + llama_sampler * smpl = llama_sampler_chain_init(sparams); + llama_sampler_chain_add(smpl, llama_sampler_init_greedy()); + + // --- decode the embedding prompt (causal, single sequence, positions 0..n-1) --- + std::vector pos(n_tokens); + std::vector n_seq_id(n_tokens, 1); + std::vector seq_id_0(1, 0); + std::vector seq_id(n_tokens); + std::vector logits(n_tokens, 0); + for (int i = 0; i < n_tokens; i++) { pos[i] = i; seq_id[i] = seq_id_0.data(); } + logits[n_tokens - 1] = 1; // only need logits for the last position + + llama_batch batch = { + /*n_tokens =*/ n_tokens, + /*token =*/ nullptr, + /*embd =*/ embd.data(), + /*pos =*/ pos.data(), + /*n_seq_id =*/ n_seq_id.data(), + /*seq_id =*/ seq_id.data(), + /*logits =*/ logits.data(), + }; + + const int64_t t_start = ggml_time_us(); + if (llama_decode(ctx, batch) != 0) { + fprintf(stderr, "error: llama_decode failed on embd prompt\n"); + return 1; + } + + // --- generation loop --- + std::string out; + int n_decode = 0; + llama_token tok = llama_sampler_sample(smpl, ctx, -1); + for (int n_pos = n_tokens; n_pos < n_tokens + n_predict; ) { + if (llama_vocab_is_eog(vocab, tok)) break; + char buf[256]; + int n = llama_token_to_piece(vocab, tok, buf, sizeof(buf), 0, true); + if (n > 0) { out.append(buf, n); } + printf("%.*s", n > 0 ? n : 0, buf); + fflush(stdout); + + llama_batch tb = llama_batch_get_one(&tok, 1); + if (llama_decode(ctx, tb) != 0) { fprintf(stderr, "error: decode failed\n"); return 1; } + n_pos += 1; + n_decode += 1; + tok = llama_sampler_sample(smpl, ctx, -1); + } + printf("\n"); + const int64_t t_end = ggml_time_us(); + fprintf(stderr, "\n[funasr-embd] generated %d tokens in %.2f s (%.1f tok/s)\n", + n_decode, (t_end - t_start) / 1e6, n_decode / ((t_end - t_start) / 1e6)); + + llama_sampler_free(smpl); + llama_free(ctx); + llama_model_free(model); + return 0; +} diff --git a/runtime/llama.cpp/fun-asr-nano/funasr-encoder/CMakeLists.txt b/runtime/llama.cpp/fun-asr-nano/funasr-encoder/CMakeLists.txt new file mode 100644 index 000000000..d30f2ef05 --- /dev/null +++ b/runtime/llama.cpp/fun-asr-nano/funasr-encoder/CMakeLists.txt @@ -0,0 +1,5 @@ +set(TARGET llama-funasr-encoder) +add_executable(${TARGET} funasr-encoder.cpp) +install(TARGETS ${TARGET} RUNTIME) +target_link_libraries(${TARGET} PRIVATE ggml ${CMAKE_THREAD_LIBS_INIT}) +target_compile_features(${TARGET} PRIVATE cxx_std_17) diff --git a/runtime/llama.cpp/fun-asr-nano/funasr-encoder/funasr-encoder.cpp b/runtime/llama.cpp/fun-asr-nano/funasr-encoder/funasr-encoder.cpp new file mode 100644 index 000000000..d176cbc98 --- /dev/null +++ b/runtime/llama.cpp/fun-asr-nano/funasr-encoder/funasr-encoder.cpp @@ -0,0 +1,275 @@ +// funasr-encoder: ggml C++ forward pass for the Fun-ASR-Nano audio encoder +// (SenseVoice SAN-M, 50+20 layers) + Transformer adaptor. +// +// Input : fbank.bin (T x 560 f32, the encoder input features) +// Output: out.bin (T' x 1024 f32, audio embeddings for the LLM) +// Weights: funasr-encoder.gguf (exported by export_encoder_gguf.py) +// +// Validated layer-by-layer against PyTorch golden dumps. fbank is currently +// produced in Python; porting the fbank frontend to C++ is the remaining piece. + +#include "ggml.h" +#include "ggml-cpu.h" +#include "ggml-alloc.h" +#include "ggml-backend.h" +#include "gguf.h" + +#include +#include +#include +#include +#include +#include +#include + +struct cfg { + int input_size = 560, d_model = 512, n_head = 4, ffn = 2048; + int num_blocks = 50, tp_blocks = 20, kernel = 11; + int adp_llm = 1024, adp_ffn = 2048, adp_layers = 2, adp_head = 8; +}; + +static const float LN_EPS = 1e-5f; + +struct funasr_model { + cfg c; + struct ggml_context * ctx_w = nullptr; // weights (CPU malloc, data set) + std::map t; + struct ggml_tensor * get(const std::string & n) { + auto it = t.find(n); + if (it == t.end()) { fprintf(stderr, "missing tensor: %s\n", n.c_str()); exit(1); } + return it->second; + } +}; + +static bool load_model(const char * path, funasr_model & m) { + struct gguf_init_params p = { /*no_alloc=*/false, /*ctx=*/&m.ctx_w }; + struct gguf_context * gguf = gguf_init_from_file(path, p); + if (!gguf) { fprintf(stderr, "failed to load gguf %s\n", path); return false; } + auto rd = [&](const char * k, int def) { + int i = gguf_find_key(gguf, k); return i < 0 ? def : (int) gguf_get_val_u32(gguf, i); + }; + m.c.input_size = rd("funasr.enc.input_size", 560); + m.c.d_model = rd("funasr.enc.output_size", 512); + m.c.n_head = rd("funasr.enc.attention_heads", 4); + m.c.ffn = rd("funasr.enc.linear_units", 2048); + m.c.num_blocks = rd("funasr.enc.num_blocks", 50); + m.c.tp_blocks = rd("funasr.enc.tp_blocks", 20); + m.c.kernel = rd("funasr.enc.kernel_size", 11); + m.c.adp_llm = rd("funasr.adp.llm_dim", 1024); + m.c.adp_ffn = rd("funasr.adp.ffn_dim", 2048); + m.c.adp_layers = rd("funasr.adp.n_layer", 2); + m.c.adp_head = rd("funasr.adp.attention_heads", 8); + int n = gguf_get_n_tensors(gguf); + for (int i = 0; i < n; i++) { + const char * name = gguf_get_tensor_name(gguf, i); + m.t[name] = ggml_get_tensor(m.ctx_w, name); + } + fprintf(stderr, "loaded %d tensors; cfg: d_model=%d heads=%d blocks=%d tp=%d kernel=%d adp_llm=%d\n", + n, m.c.d_model, m.c.n_head, m.c.num_blocks, m.c.tp_blocks, m.c.kernel, m.c.adp_llm); + gguf_free(gguf); + return true; +} + +// helpers --------------------------------------------------------------- +static struct ggml_tensor * linear(ggml_context * ctx, ggml_tensor * w, ggml_tensor * b, ggml_tensor * x) { + struct ggml_tensor * y = ggml_mul_mat(ctx, w, x); // [out, T] + if (b) y = ggml_add(ctx, y, b); + return y; +} +static struct ggml_tensor * layernorm(ggml_context * ctx, ggml_tensor * x, ggml_tensor * g, ggml_tensor * b) { + x = ggml_norm(ctx, x, LN_EPS); + x = ggml_mul(ctx, x, g); + x = ggml_add(ctx, x, b); + return x; +} + +// SAN-M self-attention + FSMN. x:[D_in,T] -> [d_model,T] +static struct ggml_tensor * sanm_attn(ggml_context * ctx, funasr_model & m, const std::string & pfx, + ggml_tensor * x, int T) { + const int D = m.c.d_model, H = m.c.n_head, dk = D / H, K = m.c.kernel; + struct ggml_tensor * qkv = linear(ctx, m.get(pfx + "linear_q_k_v.weight"), + m.get(pfx + "linear_q_k_v.bias"), x); // [3D, T] + size_t nb1 = qkv->nb[1]; + struct ggml_tensor * q = ggml_cont(ctx, ggml_view_2d(ctx, qkv, D, T, nb1, 0)); + struct ggml_tensor * k = ggml_cont(ctx, ggml_view_2d(ctx, qkv, D, T, nb1, (size_t) D * sizeof(float))); + struct ggml_tensor * v = ggml_cont(ctx, ggml_view_2d(ctx, qkv, D, T, nb1, (size_t) 2 * D * sizeof(float))); + + // FSMN: depthwise conv1d along time (per-channel kernel K, "same" padding), + // plus residual v. Implemented as an exact f32 shift-accumulate to avoid the + // F16-only ggml_conv_1d_dw path. fsmn kernel stored as [D, K] (ne0=D, ne1=K). + const int pad = (K - 1) / 2; + struct ggml_tensor * fk = m.get(pfx + "fsmn_block.weight"); // [D, K] + struct ggml_tensor * vpad = ggml_pad_ext(ctx, v, 0, 0, pad, pad, 0, 0, 0, 0); // [D, T+2*pad] + struct ggml_tensor * fsmn = v; // residual + for (int j = 0; j < K; j++) { + struct ggml_tensor * sl = ggml_view_2d(ctx, vpad, D, T, vpad->nb[1], (size_t) j * vpad->nb[1]); + struct ggml_tensor * wj = ggml_view_1d(ctx, fk, D, (size_t) j * fk->nb[1]); + fsmn = ggml_add(ctx, fsmn, ggml_mul(ctx, ggml_cont(ctx, sl), wj)); + } + + // multi-head attention + q = ggml_reshape_3d(ctx, q, dk, H, T); + k = ggml_reshape_3d(ctx, k, dk, H, T); + struct ggml_tensor * vh = ggml_reshape_3d(ctx, v, dk, H, T); + q = ggml_permute(ctx, q, 0, 2, 1, 3); // [dk, T, H] + k = ggml_permute(ctx, k, 0, 2, 1, 3); // [dk, T, H] + vh = ggml_cont(ctx, ggml_permute(ctx, vh, 1, 2, 0, 3)); // [T, dk, H] + struct ggml_tensor * kq = ggml_mul_mat(ctx, k, q); // [T, T, H] + kq = ggml_scale(ctx, kq, 1.0f / sqrtf((float) dk)); + kq = ggml_soft_max(ctx, kq); + struct ggml_tensor * kqv = ggml_mul_mat(ctx, vh, kq); // [dk, T, H] + kqv = ggml_permute(ctx, kqv, 0, 2, 1, 3); // [dk, H, T] + kqv = ggml_cont_2d(ctx, kqv, D, T); // [D, T] + struct ggml_tensor * att = linear(ctx, m.get(pfx + "linear_out.weight"), + m.get(pfx + "linear_out.bias"), kqv); + return ggml_add(ctx, att, fsmn); +} + +// one SAN-M encoder layer. in_size may differ from d_model (first layer) +static struct ggml_tensor * sanm_layer(ggml_context * ctx, funasr_model & m, const std::string & pfx, + ggml_tensor * x, int T, bool residual_attn) { + struct ggml_tensor * res = x; + struct ggml_tensor * h = layernorm(ctx, x, m.get(pfx + "norm1.weight"), m.get(pfx + "norm1.bias")); + struct ggml_tensor * sa = sanm_attn(ctx, m, pfx + "self_attn.", h, T); + x = residual_attn ? ggml_add(ctx, res, sa) : sa; + res = x; + h = layernorm(ctx, x, m.get(pfx + "norm2.weight"), m.get(pfx + "norm2.bias")); + h = linear(ctx, m.get(pfx + "feed_forward.w_1.weight"), m.get(pfx + "feed_forward.w_1.bias"), h); + h = ggml_relu(ctx, h); + h = linear(ctx, m.get(pfx + "feed_forward.w_2.weight"), m.get(pfx + "feed_forward.w_2.bias"), h); + return ggml_add(ctx, res, h); +} + +// standard transformer layer (adaptor). d=adp_llm +static struct ggml_tensor * adp_layer(ggml_context * ctx, funasr_model & m, const std::string & pfx, + ggml_tensor * x, int T) { + const int D = m.c.adp_llm, H = m.c.adp_head, dk = D / H; + struct ggml_tensor * res = x; + struct ggml_tensor * h = layernorm(ctx, x, m.get(pfx + "norm1.weight"), m.get(pfx + "norm1.bias")); + struct ggml_tensor * q = linear(ctx, m.get(pfx + "self_attn.linear_q.weight"), m.get(pfx + "self_attn.linear_q.bias"), h); + struct ggml_tensor * k = linear(ctx, m.get(pfx + "self_attn.linear_k.weight"), m.get(pfx + "self_attn.linear_k.bias"), h); + struct ggml_tensor * v = linear(ctx, m.get(pfx + "self_attn.linear_v.weight"), m.get(pfx + "self_attn.linear_v.bias"), h); + q = ggml_permute(ctx, ggml_reshape_3d(ctx, q, dk, H, T), 0, 2, 1, 3); + k = ggml_permute(ctx, ggml_reshape_3d(ctx, k, dk, H, T), 0, 2, 1, 3); + struct ggml_tensor * vh = ggml_cont(ctx, ggml_permute(ctx, ggml_reshape_3d(ctx, v, dk, H, T), 1, 2, 0, 3)); + struct ggml_tensor * kq = ggml_soft_max(ctx, ggml_scale(ctx, ggml_mul_mat(ctx, k, q), 1.0f / sqrtf((float) dk))); + struct ggml_tensor * kqv = ggml_cont_2d(ctx, ggml_permute(ctx, ggml_mul_mat(ctx, vh, kq), 0, 2, 1, 3), D, T); + struct ggml_tensor * att = linear(ctx, m.get(pfx + "self_attn.linear_out.weight"), m.get(pfx + "self_attn.linear_out.bias"), kqv); + x = ggml_add(ctx, res, att); + res = x; + h = layernorm(ctx, x, m.get(pfx + "norm2.weight"), m.get(pfx + "norm2.bias")); + h = linear(ctx, m.get(pfx + "feed_forward.w_1.weight"), m.get(pfx + "feed_forward.w_1.bias"), h); + h = ggml_relu(ctx, h); + h = linear(ctx, m.get(pfx + "feed_forward.w_2.weight"), m.get(pfx + "feed_forward.w_2.bias"), h); + return ggml_add(ctx, res, h); +} + +// sinusoidal position encoding, depth = input feature dim, positions 1..T +static void add_posenc(std::vector & x, int T, int depth) { + double inc = log(10000.0) / (depth / 2.0 - 1.0); + for (int t = 0; t < T; t++) { + double pos = t + 1; // positions start at 1 + for (int i = 0; i < depth / 2; i++) { + double its = exp(i * -inc); + double st = pos * its; + x[(size_t) t * depth + i] += (float) sin(st); + x[(size_t) t * depth + depth / 2 + i] += (float) cos(st); + } + } +} + +int main(int argc, char ** argv) { + std::string gguf_path, fbank_path, out_path = "out.bin"; + int limit = -1; // -L: run only first N (encoders0+encoders) layers, dump running x + bool run_adaptor = true; + for (int i = 1; i < argc; i++) { + if (!strcmp(argv[i], "-m") && i+1 < argc) gguf_path = argv[++i]; + else if (!strcmp(argv[i], "-f") && i+1 < argc) fbank_path = argv[++i]; + else if (!strcmp(argv[i], "-o") && i+1 < argc) out_path = argv[++i]; + else if (!strcmp(argv[i], "-L") && i+1 < argc) { limit = atoi(argv[++i]); run_adaptor = false; } + else { fprintf(stderr, "usage: %s -m enc.gguf -f fbank.bin [-o out.bin] [-L nlayers]\n", argv[0]); return 1; } + } + + funasr_model m; + if (!load_model(gguf_path.c_str(), m)) return 1; + + // read fbank.bin (T x F) + FILE * f = fopen(fbank_path.c_str(), "rb"); + if (!f) { fprintf(stderr, "cannot open %s\n", fbank_path.c_str()); return 1; } + int32_t T, F; if (fread(&T, 4, 1, f) != 1 || fread(&F, 4, 1, f) != 1) { fclose(f); return 1; } + std::vector fbank((size_t) T * F); + if (fread(fbank.data(), sizeof(float), fbank.size(), f) != fbank.size()) { fclose(f); return 1; } + fclose(f); + fprintf(stderr, "fbank: T=%d F=%d\n", T, F); + + // pre-scale (*sqrt(d_model)) and add position encoding on the host + float scale = sqrtf((float) m.c.d_model); + for (auto & v : fbank) v *= scale; + add_posenc(fbank, T, F); + + // backend + compute context + ggml_backend_t backend = ggml_backend_cpu_init(); + size_t ctx_size = (size_t) 1024*1024*1024; // graph metadata + struct ggml_init_params cp = { ctx_size, nullptr, true }; + struct ggml_context * ctx = ggml_init(cp); + + struct ggml_tensor * inp = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, F, T); + ggml_set_name(inp, "inp"); + ggml_set_input(inp); + + struct ggml_tensor * x = inp; + int done = 0; + bool stop = false; + // encoders0 (1 layer, in_size=input_size != d_model -> no attn residual) + x = sanm_layer(ctx, m, "audio_encoder.encoders0.0.", x, T, /*residual_attn=*/false); + done++; + if (limit >= 0 && done >= limit) stop = true; + // encoders (num_blocks-1 layers) + for (int i = 0; i < m.c.num_blocks - 1 && !stop; i++) { + x = sanm_layer(ctx, m, "audio_encoder.encoders." + std::to_string(i) + ".", x, T, true); + done++; + if (limit >= 0 && done >= limit) stop = true; + } + if (!stop) { + x = layernorm(ctx, x, m.get("audio_encoder.after_norm.weight"), m.get("audio_encoder.after_norm.bias")); + for (int i = 0; i < m.c.tp_blocks; i++) + x = sanm_layer(ctx, m, "audio_encoder.tp_encoders." + std::to_string(i) + ".", x, T, true); + x = layernorm(ctx, x, m.get("audio_encoder.tp_norm.weight"), m.get("audio_encoder.tp_norm.bias")); + // adaptor: downsample_rate=1 -> linear1(relu)linear2 then blocks + if (run_adaptor) { + x = linear(ctx, m.get("audio_adaptor.linear1.weight"), m.get("audio_adaptor.linear1.bias"), x); + x = ggml_relu(ctx, x); + x = linear(ctx, m.get("audio_adaptor.linear2.weight"), m.get("audio_adaptor.linear2.bias"), x); + for (int i = 0; i < m.c.adp_layers; i++) + x = adp_layer(ctx, m, "audio_adaptor.blocks." + std::to_string(i) + ".", x, T); + } + } + ggml_set_output(x); + + struct ggml_cgraph * gf = ggml_new_graph_custom(ctx, 32768, false); + ggml_build_forward_expand(gf, x); + + ggml_gallocr_t galloc = ggml_gallocr_new(ggml_backend_cpu_buffer_type()); + ggml_gallocr_alloc_graph(galloc, gf); + ggml_backend_tensor_set(inp, fbank.data(), 0, ggml_nbytes(inp)); + + ggml_backend_cpu_set_n_threads(backend, 8); + int64_t t0 = ggml_time_us(); + if (ggml_backend_graph_compute(backend, gf) != GGML_STATUS_SUCCESS) { + fprintf(stderr, "compute failed\n"); return 1; + } + int64_t t1 = ggml_time_us(); + + int D = (int) x->ne[0]; + std::vector out((size_t) D * T); + ggml_backend_tensor_get(x, out.data(), 0, ggml_nbytes(x)); + FILE * fo = fopen(out_path.c_str(), "wb"); + if (!fo) { fprintf(stderr, "failed to open output file %s\n", out_path.c_str()); return 1; } + fwrite(&T, 4, 1, fo); fwrite(&D, 4, 1, fo); fwrite(out.data(), sizeof(float), out.size(), fo); + fclose(fo); + fprintf(stderr, "done: wrote %s [%d x %d] in %.2f s (layers run=%d, adaptor=%d)\n", + out_path.c_str(), T, D, (t1 - t0)/1e6, done, run_adaptor && !stop); + ggml_gallocr_free(galloc); ggml_free(ctx); ggml_backend_free(backend); + if (m.ctx_w) ggml_free(m.ctx_w); + return 0; +} diff --git a/runtime/llama.cpp/paraformer/README.md b/runtime/llama.cpp/paraformer/README.md new file mode 100644 index 000000000..21b40c74b --- /dev/null +++ b/runtime/llama.cpp/paraformer/README.md @@ -0,0 +1,103 @@ +# Paraformer on llama.cpp / GGUF + +Run **Paraformer** (the non-autoregressive ASR model) on the +[llama.cpp](https://github.com/ggml-org/llama.cpp) / ggml stack — **CPU, edge, +a single binary, no Python at runtime**. Like whisper.cpp, but for Paraformer. + +## Why this exists + +Paraformer normally runs on PyTorch / ONNX. This runtime ports it to **ggml + +GGUF** so it runs CPU-only, offline, embedded in a C/C++ app, with quantized +weights — laptops, phones, edge boxes, no GPU and no Python. (For high-QPS GPU +serving, the PyTorch path is still the way.) + +## Architecture + +Paraformer is **non-autoregressive**: it predicts all output tokens in one pass. + +``` + audio.wav (16k mono) + │ kaldi 80-mel fbank + LFR + CMVN (C++) + ▼ + features [T, 560] + │ SANM encoder (50 layers: LN + fused QKV + FSMN + FFN) (ggml) + ▼ + encoder_out [T, 512] + │ CIF predictor: conv1d → sigmoid → α; integrate-and-fire (host) + ▼ + acoustic embeds [N_tok, 512] (N_tok = number of output tokens) + │ SANM decoder (16 layers: FFN → FSMN self-attn → cross-attn to encoder) (ggml) + ▼ + logits [N_tok, vocab] → argmax → token ids → text +``` + +CIF (Continuous Integrate-and-Fire) walks the encoder output accumulating a +predicted "weight" α per frame; each time the running sum crosses 1.0 it emits one +acoustic token. This both decides the token count and produces the acoustic +embeddings the decoder consumes. The SANM encoder/FSMN/attention primitives are +shared with the Fun-ASR-Nano and SenseVoice runtimes. + +## Quickstart + +**1. Build:** +```bash +git clone https://github.com/ggml-org/llama.cpp && cd llama.cpp +cp -r /path/to/runtime/llama.cpp/funasr-paraformer examples/ +echo 'add_subdirectory(funasr-paraformer)' >> examples/CMakeLists.txt +cmake -B build -DGGML_NATIVE=ON -DLLAMA_CURL=OFF +cmake --build build -j --target llama-funasr-paraformer +``` + +**2. Convert weights** (needs the checkpoint, e.g. `funasr/paraformer-zh`): +```bash +python runtime/llama.cpp/export_paraformer_gguf.py \ + --model_pt /model.pt --mvn /am.mvn \ + --out paraformer.gguf # f32, ~863 MB +python runtime/llama.cpp/export_paraformer_gguf.py --wtype f16 \ + --model_pt /model.pt --mvn /am.mvn \ + --out paraformer-f16.gguf # half size +``` + +**3. Transcribe:** +```bash +build/bin/llama-funasr-paraformer -m paraformer.gguf -a audio.wav > ids.txt +python runtime/llama.cpp/detok.py /tokens.json ids.txt +``` +Expected output: +``` +我想问我在滨海新区有房我一直没有照顾孩子...你觉得这是正常的想法吗 +[paraformer] T=742 N_tok=105 enc 1.24s dec 0.48s +``` + +## Accuracy & validation + +- Decoded text is **character-for-character identical** to the FunASR `AutoModel` + output on a benchmark clip; the CIF token count matches exactly (105/105). +- Stage-by-stage vs PyTorch: encoder cosine 0.997, acoustic embeds cosine 0.993 + (the small residual is the reference frontend's random `dither=1.0`; the C++ + front end is deterministic, dither=0). +- Encode ≈ 1.2 s + decode ≈ 0.5 s on CPU for a 44 s clip. + +## Tips & gotchas + +- **CMVN IS applied** (unlike SenseVoice): `(fbank + shift) * scale`, per-dim (560), + from `am.mvn`. Parsing note: `am.mvn` has three `[...]` blocks — `[Splice idx]`, + `[AddShift=shift]`, `[Rescale=scale]`; use the two 560-length vectors. Getting + this wrong makes the CIF predictor emit ~4× too few tokens. +- **CIF/predictor runs on host** (it's a sequential integrate-and-fire loop); + the encoder and decoder run in ggml. +- The decoder self-attention is **FSMN-only** (no QK attention); cross-attention + attends to the encoder output. The decoder FFN has an internal LayerNorm and the + second linear has no bias. +- WAV input assumes 16 kHz mono PCM16. + +## Files +``` +funasr-paraformer/ ggml runtime: WAV → token ids +export_paraformer_gguf.py export encoder + predictor + decoder + CMVN to GGUF +detok.py token-id → text (tokens.json) +``` + +## Roadmap +- Built-in detok; timestamps (CIF peaks give alignment); arbitrary WAV / resampling; + encoder/decoder quantization. diff --git a/runtime/llama.cpp/paraformer/detok_paraformer.py b/runtime/llama.cpp/paraformer/detok_paraformer.py new file mode 100644 index 000000000..0e56aaab9 --- /dev/null +++ b/runtime/llama.cpp/paraformer/detok_paraformer.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 +"""Detokenize Paraformer token ids -> text via tokens.json. +Usage: python detok_paraformer.py """ +import sys, json +toks = json.load(open(sys.argv[1])) +ids = [int(x) for x in open(sys.argv[2]).read().split() if int(x) not in (1, 2)] # drop sos/eos +out = "".join(toks[i] for i in ids if 0 <= i < len(toks)) +print(out.replace("@@", "").replace("▁", " ").strip()) diff --git a/runtime/llama.cpp/paraformer/export_paraformer_gguf.py b/runtime/llama.cpp/paraformer/export_paraformer_gguf.py new file mode 100644 index 000000000..476f554d6 --- /dev/null +++ b/runtime/llama.cpp/paraformer/export_paraformer_gguf.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Export Paraformer (SANM encoder + CIF predictor + SANM decoder) to GGUF. +Encoder reuses the shared SAN-M forward. Predictor (CIF) runs on host in C++. +""" +import argparse, os, re +import numpy as np, torch, gguf + + +def parse_mvn(path): + # am.mvn (kaldi nnet) has 3 bracketed blocks: [Splice idx], [AddShift=shift], + # [Rescale=scale]. Take the two 560-length vectors (shift then scale). + # apply: out = (x + shift) * scale + blocks = [np.array([float(x) for x in b.split()], np.float32) + for b in re.findall(r"\[([^\]]*)\]", open(path).read())] + vecs = [b for b in blocks if b.size > 1] + return vecs[0], vecs[1] + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--model_pt", required=True); ap.add_argument("--mvn", required=True) + ap.add_argument("--out", required=True); ap.add_argument("--wtype", default="f32", choices=["f32","f16"]) + a = ap.parse_args() + sd = torch.load(a.model_pt, map_location="cpu"); sd = sd.get("state_dict", sd) + w = gguf.GGUFWriter(a.out, "paraformer") + w.add_uint32("pf.enc.output_size", 512); w.add_uint32("pf.enc.attention_heads", 4) + w.add_uint32("pf.enc.num_blocks", 50); w.add_uint32("pf.enc.kernel_size", 11) + w.add_uint32("pf.dec.num_blocks", 16); w.add_uint32("pf.dec.att_layer_num", 16) + w.add_uint32("pf.dec.decoders3", 1); w.add_uint32("pf.dec.attention_heads", 4) + w.add_uint32("pf.dec.kernel_size", 11); w.add_uint32("pf.vocab_size", 8404) + w.add_float32("pf.predictor.tail_threshold", 0.45) + w.add_float32("pf.predictor.threshold", 1.0) + shift, scale = parse_mvn(a.mvn) + w.add_tensor("cmvn.shift", shift); w.add_tensor("cmvn.scale", scale) + n = 0 + for k, v in sd.items(): + if not (k.startswith("encoder.") or k.startswith("decoder.") or k.startswith("predictor.")): + continue + if k == "decoder.embed.0.weight": # token embedding, unused at NAR inference + continue + arr = v.detach().to(torch.float32).contiguous().numpy() + if k.endswith("fsmn_block.weight") and arr.ndim == 3: # (D,1,K)->(K,D) + arr = np.ascontiguousarray(arr[:, 0, :].T) + elif args_f16(a) and arr.ndim == 2 and "norm" not in k and "cif_output" not in k: + arr = arr.astype(np.float16) + w.add_tensor(k, arr); n += 1 + print(f"writing {n} tensors (+cmvn) to {a.out}") + w.write_header_to_file(); w.write_kv_data_to_file(); w.write_tensors_to_file(); w.close() + print(f"done: {a.out} ({os.path.getsize(a.out)/1e6:.1f} MB)") + + +def args_f16(a): return a.wtype == "f16" + +if __name__ == "__main__": + main() diff --git a/runtime/llama.cpp/paraformer/funasr-paraformer/CMakeLists.txt b/runtime/llama.cpp/paraformer/funasr-paraformer/CMakeLists.txt new file mode 100644 index 000000000..7fb44e1f6 --- /dev/null +++ b/runtime/llama.cpp/paraformer/funasr-paraformer/CMakeLists.txt @@ -0,0 +1,5 @@ +set(TARGET llama-funasr-paraformer) +add_executable(${TARGET} funasr-paraformer.cpp) +install(TARGETS ${TARGET} RUNTIME) +target_link_libraries(${TARGET} PRIVATE ggml ${CMAKE_THREAD_LIBS_INIT}) +target_compile_features(${TARGET} PRIVATE cxx_std_17) diff --git a/runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp b/runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp new file mode 100644 index 000000000..0b8876e49 --- /dev/null +++ b/runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp @@ -0,0 +1,213 @@ +// funasr-paraformer: Paraformer (non-autoregressive ASR) on ggml. +// WAV → kaldi fbank → CMVN → SANM encoder (ggml) → CIF predictor (host) → +// SANM decoder w/ cross-attn (ggml) → argmax → token ids (stdout). +// Encoder/FSMN/attention primitives are shared with the Fun-ASR-Nano runtime. + +#include "ggml.h" +#include "ggml-cpu.h" +#include "ggml-alloc.h" +#include "ggml-backend.h" +#include "gguf.h" + +#include +#include +#include +#include +#include +#include +#include + +static const float LN_EPS = 1e-5f; + +// ===== WAV + kaldi fbank (shared) ===== +static bool read_wav16(const char*p,std::vector&o){FILE*f=fopen(p,"rb");if(!f)return false;char r[4];fread(r,1,4,f); + if(strncmp(r,"RIFF",4)){fclose(f);return false;}fseek(f,4,SEEK_CUR);char wv[4];fread(wv,1,4,f);if(strncmp(wv,"WAVE",4)){fclose(f);return false;} + uint16_t ch=1,bits=16;uint32_t sr=16000;while(!feof(f)){char id[4];if(fread(id,1,4,f)!=4)break;uint32_t sz;if(fread(&sz,4,1,f)!=1)break; + if(!strncmp(id,"fmt ",4)){uint16_t fm;fread(&fm,2,1,f);fread(&ch,2,1,f);fread(&sr,4,1,f);fseek(f,6,SEEK_CUR);fread(&bits,2,1,f);if(sz>16)fseek(f,sz-16,SEEK_CUR);} + else if(!strncmp(id,"data",4)){if(ch<1||bits!=16){fclose(f);fprintf(stderr,"only 16-bit PCM WAV (ch=%u bits=%u)\n",ch,bits);return false;}int n=sz/(bits/8);std::vectorpc(n);fread(pc.data(),2,n,f);o.resize(n/ch);for(int i=0;i&re,std::vector&im,int n){for(int i=1,j=0;i>1;for(;j&b;b>>=1)j^=b;j^=b;if(i compute_fbank(std::vector wav,int&T_out){ + for(auto&v:wav)v*=32768.0f;std::vectorwin(WINLEN);for(int i=0;i>fb(NMEL,std::vector(NB,0.0f)); + for(int m=0;mL&&mf>feat(T,std::vector(NMEL)); + std::vectorre(NFFT),im(NFFT),fr(WINLEN);const float fl=1.1920929e-07f; + for(int t=0;t0;i--)fr[i]-=PREEMPH*fr[i-1];fr[0]-=PREEMPH*fr[0]; + for(int i=0;i0)e+=fb[m][k]*(re[k]*re[k]+im[k]*im[k]);feat[t][m]=logf(e>fl?e:fl);}} + const int pad=(LFR_M-1)/2;int Tl=(T+LFR_N-1)/LFR_N;std::vector>pd;pd.reserve(T+pad+LFR_M); + for(int i=0;iout((size_t)Tl*D);for(int i=0;it; + ggml_tensor*g(const std::string&n){auto it=t.find(n);if(it==t.end()){fprintf(stderr,"missing %s\n",n.c_str());exit(1);}return it->second;} + bool has(const std::string&n){return t.count(n);} }; + +static ggml_tensor* lin(ggml_context*c,ggml_tensor*w,ggml_tensor*b,ggml_tensor*x){auto y=ggml_mul_mat(c,w,x);return b?ggml_add(c,y,b):y;} +static ggml_tensor* lnorm(ggml_context*c,ggml_tensor*x,ggml_tensor*g,ggml_tensor*b){return ggml_add(c,ggml_mul(c,ggml_norm(c,x,LN_EPS),g),b);} +// FSMN depthwise (kernel stored [K,D]): out = sum_j w[:,j]*pad(v)[:, t+j] (+ residual v) +static ggml_tensor* fsmn(ggml_context*c,ggml_tensor*v,ggml_tensor*fk,int D,int T,int K){ + const int pad=(K-1)/2;ggml_tensor*vp=ggml_pad_ext(c,v,0,0,pad,pad,0,0,0,0);ggml_tensor*acc=v; + for(int j=0;jnb[1],(size_t)j*vp->nb[1]);auto wj=ggml_view_1d(c,fk,D,(size_t)j*fk->nb[1]); + acc=ggml_add(c,acc,ggml_mul(c,ggml_cont(c,sl),wj));}return acc;} +// SAN-M encoder self-attn (fused qkv + fsmn) +static ggml_tensor* enc_attn(ggml_context*c,model&m,const std::string&p,ggml_tensor*x,int T){ + const int D=m.c.d_model,H=m.c.enc_head,dk=D/H,K=m.c.enc_kernel; + ggml_tensor*qkv=lin(c,m.g(p+"linear_q_k_v.weight"),m.g(p+"linear_q_k_v.bias"),x);size_t nb1=qkv->nb[1]; + ggml_tensor*q=ggml_cont(c,ggml_view_2d(c,qkv,D,T,nb1,0)); + ggml_tensor*k=ggml_cont(c,ggml_view_2d(c,qkv,D,T,nb1,(size_t)D*sizeof(float))); + ggml_tensor*v=ggml_cont(c,ggml_view_2d(c,qkv,D,T,nb1,(size_t)2*D*sizeof(float))); + ggml_tensor*fm=fsmn(c,v,m.g(p+"fsmn_block.weight"),D,T,K); + q=ggml_permute(c,ggml_reshape_3d(c,q,dk,H,T),0,2,1,3);k=ggml_permute(c,ggml_reshape_3d(c,k,dk,H,T),0,2,1,3); + ggml_tensor*vh=ggml_cont(c,ggml_permute(c,ggml_reshape_3d(c,v,dk,H,T),1,2,0,3)); + ggml_tensor*kq=ggml_soft_max(c,ggml_scale(c,ggml_mul_mat(c,k,q),1.0f/sqrtf((float)dk))); + ggml_tensor*o=ggml_cont_2d(c,ggml_permute(c,ggml_mul_mat(c,vh,kq),0,2,1,3),D,T); + return ggml_add(c,lin(c,m.g(p+"linear_out.weight"),m.g(p+"linear_out.bias"),o),fm);} +static ggml_tensor* enc_layer(ggml_context*c,model&m,const std::string&p,ggml_tensor*x,int T,bool res){ + auto r=x;auto h=lnorm(c,x,m.g(p+"norm1.weight"),m.g(p+"norm1.bias"));auto sa=enc_attn(c,m,p+"self_attn.",h,T); + x=res?ggml_add(c,r,sa):sa;r=x;h=lnorm(c,x,m.g(p+"norm2.weight"),m.g(p+"norm2.bias")); + h=lin(c,m.g(p+"feed_forward.w_1.weight"),m.g(p+"feed_forward.w_1.bias"),h);h=ggml_relu(c,h); + h=lin(c,m.g(p+"feed_forward.w_2.weight"),m.g(p+"feed_forward.w_2.bias"),h);return ggml_add(c,r,h);} +// decoder FFN-SANM: w_2(LayerNorm(relu(w_1(x)))) (w_2 has no bias) +static ggml_tensor* dec_ffn(ggml_context*c,model&m,const std::string&p,ggml_tensor*x){ + auto h=lin(c,m.g(p+"w_1.weight"),m.g(p+"w_1.bias"),x);h=ggml_relu(c,h); + h=lnorm(c,h,m.g(p+"norm.weight"),m.g(p+"norm.bias"));return ggml_mul_mat(c,m.g(p+"w_2.weight"),h);} +// cross attn: q=linear_q(tgt)[D,N], kv=linear_k_v(mem)[2D,T] +static ggml_tensor* cross_attn(ggml_context*c,model&m,const std::string&p,ggml_tensor*tgt,ggml_tensor*mem,int N,int T){ + const int D=m.c.d_model,H=m.c.dec_head,dk=D/H; + ggml_tensor*q=lin(c,m.g(p+"linear_q.weight"),m.g(p+"linear_q.bias"),tgt); // [D,N] + ggml_tensor*kv=lin(c,m.g(p+"linear_k_v.weight"),m.g(p+"linear_k_v.bias"),mem);// [2D,T] + size_t nb1=kv->nb[1]; + ggml_tensor*k=ggml_cont(c,ggml_view_2d(c,kv,D,T,nb1,0)); + ggml_tensor*v=ggml_cont(c,ggml_view_2d(c,kv,D,T,nb1,(size_t)D*sizeof(float))); + q=ggml_permute(c,ggml_reshape_3d(c,q,dk,H,N),0,2,1,3); // [dk,N,H] + k=ggml_permute(c,ggml_reshape_3d(c,k,dk,H,T),0,2,1,3); // [dk,T,H] + ggml_tensor*vh=ggml_cont(c,ggml_permute(c,ggml_reshape_3d(c,v,dk,H,T),1,2,0,3)); // [T,dk,H] + ggml_tensor*kq=ggml_soft_max(c,ggml_scale(c,ggml_mul_mat(c,k,q),1.0f/sqrtf((float)dk))); // [T,N,H] + ggml_tensor*o=ggml_cont_2d(c,ggml_permute(c,ggml_mul_mat(c,vh,kq),0,2,1,3),D,N); + return lin(c,m.g(p+"linear_out.weight"),m.g(p+"linear_out.bias"),o);} +static ggml_tensor* dec_layer(ggml_context*c,model&m,const std::string&p,ggml_tensor*tgt,ggml_tensor*mem,int N,int T){ + const int D=m.c.d_model,K=m.c.dec_kernel; + auto residual=tgt;auto h=lnorm(c,tgt,m.g(p+"norm1.weight"),m.g(p+"norm1.bias")); + h=dec_ffn(c,m,p+"feed_forward.",h); // FFN first + auto y=lnorm(c,h,m.g(p+"norm2.weight"),m.g(p+"norm2.bias")); + auto sa=fsmn(c,y,m.g(p+"self_attn.fsmn_block.weight"),D,N,K); // FSMN self-attn (+residual y inside) + auto x=ggml_add(c,residual,sa); + residual=x;auto z=lnorm(c,x,m.g(p+"norm3.weight"),m.g(p+"norm3.bias")); + auto ca=cross_attn(c,m,p+"src_attn.",z,mem,N,T); + return ggml_add(c,residual,ca);} +static ggml_tensor* dec3_layer(ggml_context*c,model&m,const std::string&p,ggml_tensor*tgt){ + auto h=lnorm(c,tgt,m.g(p+"norm1.weight"),m.g(p+"norm1.bias"));return dec_ffn(c,m,p+"feed_forward.",h);} +static void add_posenc(std::vector&x,int T,int depth){double inc=log(10000.0)/(depth/2.0-1.0); + for(int t=0;t run_graph(ggml_context*c,ggml_tensor*out,ggml_tensor*in1,const float*d1,ggml_tensor*in2,const float*d2){ + ggml_backend_t be=ggml_backend_cpu_init();ggml_cgraph*gf=ggml_new_graph_custom(c,32768,false);ggml_build_forward_expand(gf,out); + ggml_gallocr_t ga=ggml_gallocr_new(ggml_backend_cpu_buffer_type());ggml_gallocr_alloc_graph(ga,gf); + ggml_backend_tensor_set(in1,d1,0,ggml_nbytes(in1));if(in2)ggml_backend_tensor_set(in2,d2,0,ggml_nbytes(in2)); + ggml_backend_cpu_set_n_threads(be,8);ggml_backend_graph_compute(be,gf); + int D=out->ne[0],N=out->ne[1];std::vectorr((size_t)D*N);ggml_backend_tensor_get(out,r.data(),0,ggml_nbytes(out)); + ggml_gallocr_free(ga);ggml_backend_free(be);return r;} + +int main(int argc,char**argv){ + std::string gguf_path,wav_path; + for(int i=1;iwav;if(!read_wav16(wav_path.c_str(),wav)){fprintf(stderr,"read wav failed\n");return 1;} + int64_t t0=ggml_time_us();int T=0;auto fb=compute_fbank(wav,T); + // CMVN (Paraformer applies it): (x+shift)*scale + float*shift=(float*)m.g("cmvn.shift")->data,*scale=(float*)m.g("cmvn.scale")->data; + for(int t=0;tenc; + {ggml_init_params cp={(size_t)1024*1024*1024,nullptr,true};ggml_context*c=ggml_init(cp); + ggml_tensor*x=ggml_new_tensor_2d(c,GGML_TYPE_F32,F,T);ggml_set_input(x); + ggml_tensor*h=enc_layer(c,m,"encoder.encoders0.0.",x,T,false); + for(int i=0;i stored row-major as T rows? ggml out ne0=D,ne1=T + // run_graph returns row-major [N=ne1 rows, D=ne0]; here rows=T, cols=D + int64_t t1=ggml_time_us(); + // enc[t*D + d] + + // ===== CIF predictor (host) ===== + float*cw=(float*)m.g("predictor.cif_conv1d.weight")->data; // [512,512,3] = [o][i][j] + float*cb=(float*)m.g("predictor.cif_conv1d.bias")->data; // [512] + float*ow=(float*)m.g("predictor.cif_output.weight")->data; // [1,512] + float ob=((float*)m.g("predictor.cif_output.bias")->data)[0]; + // conv1d (k=3,pad1) + residual + relu, then cif_output -> sigmoid -> alpha + std::vector alphas(T); + std::vector outp((size_t)T*D); + for(int t=0;t=T)continue; const float*ev=&enc[(size_t)tt*D]; + const float*wo=&cw[(size_t)o*D*3]; for(int i=0;i0?s:0; // sigmoid, smooth=1 noise=0 -> relu(s)=s + } + // tail: append zero hidden frame + alpha=tail_threshold + std::vector hid=enc; hid.resize((size_t)(T+1)*D,0.0f); + std::vector al=alphas; al.push_back(m.c.tail); + int L=T+1; + // integrate-and-fire + std::vector acoustic; acoustic.reserve(64*D); + float integrate=0; std::vector frame(D,0.0f); + for(int t=0;t=m.c.thresh; float cur=fire?dc:alpha; float rem=alpha-cur; + for(int d=0;d logits; + {ggml_init_params cp={(size_t)2048*1024*1024,nullptr,true};ggml_context*c=ggml_init(cp); + ggml_tensor*tgt=ggml_new_tensor_2d(c,GGML_TYPE_F32,D,N);ggml_set_input(tgt); + ggml_tensor*mem=ggml_new_tensor_2d(c,GGML_TYPE_F32,D,T);ggml_set_input(mem); + ggml_tensor*x=tgt; + for(int i=0;i ids + for(int n=0;nbest){best=col[v];am=v;}printf("%d ",am);} + printf("\n"); + fprintf(stderr,"[paraformer] T=%d N_tok=%d enc %.2fs dec %.2fs\n",T,N,(t1-t0)/1e6,(t2-t1)/1e6); + if(m.ctx_w) ggml_free(m.ctx_w); + return 0; +} diff --git a/runtime/llama.cpp/sensevoice/README.md b/runtime/llama.cpp/sensevoice/README.md new file mode 100644 index 000000000..907a7323f --- /dev/null +++ b/runtime/llama.cpp/sensevoice/README.md @@ -0,0 +1,102 @@ +# SenseVoiceSmall on llama.cpp / GGUF + +Run **SenseVoiceSmall** on the [llama.cpp](https://github.com/ggml-org/llama.cpp) +/ ggml stack — **CPU, edge, a single binary, no Python at runtime**. Like +[whisper.cpp](https://github.com/ggml-org/whisper.cpp), but for SenseVoice. + +## Why this exists + +SenseVoiceSmall normally runs on PyTorch / ONNX / libtorch. This runtime ports it +to **ggml + GGUF** so it can run CPU-only, offline, embedded in a C/C++ app, with +quantized weights. Use it on laptops / phones / edge boxes where there is no GPU +and no Python. (For high-QPS GPU serving, the PyTorch/vLLM path is still the way.) + +## Architecture + +SenseVoiceSmall = **SAN-M encoder (70 layers) + CTC head** — no LLM, no autoregression. +The whole pipeline runs in C++: + +``` + audio.wav (16k mono) + │ kaldi 80-mel fbank + LFR (C++) + ▼ + features [T, 560] + │ prepend 4 query tokens [lang, event, emotion, itn] + ▼ + [4 + T, 560] + │ SAN-M encoder (ggml) ── sensevoice-small.gguf + ▼ + encoder out [4+T, 512] + │ CTC head (Linear 512→25055) → greedy CTC (argmax, dedup, drop blank) + ▼ + token ids + │ SentencePiece detok (detok.py) + ▼ + <|zh|><|NEUTRAL|><|Speech|><|woitn|> transcription... +``` + +The SAN-M encoder is the same architecture as Fun-ASR-Nano's, so the ggml forward +is shared between the two runtimes. + +## Quickstart + +**1. Build:** +```bash +git clone https://github.com/ggml-org/llama.cpp && cd llama.cpp +cp -r /path/to/runtime/llama.cpp/funasr-sensevoice examples/ +echo 'add_subdirectory(funasr-sensevoice)' >> examples/CMakeLists.txt +cmake -B build -DGGML_NATIVE=ON -DLLAMA_CURL=OFF +cmake --build build -j --target llama-funasr-sensevoice +``` + +**2. Convert weights** (needs the checkpoint, e.g. `FunAudioLLM/SenseVoiceSmall`): +```bash +python runtime/llama.cpp/export_sensevoice_gguf.py \ + --model_pt /model.pt --mvn /am.mvn \ + --out sensevoice-small.gguf # f32, ~936 MB +python runtime/llama.cpp/export_sensevoice_gguf.py --wtype f16 \ + --model_pt /model.pt --mvn /am.mvn \ + --out sensevoice-small-f16.gguf # half size +``` + +**3. Transcribe:** +```bash +build/bin/llama-funasr-sensevoice -m sensevoice-small.gguf -a audio.wav > ids.txt +python runtime/llama.cpp/detok.py /chn_jpn_yue_eng_ko_spectok.bpe.model ids.txt +``` +Expected output: +``` +<|zh|><|NEUTRAL|><|Speech|><|woitn|>我想问我在滨海新区有房我一直没有照顾孩子...你觉得这是正常的想法吗 +[sensevoice] N=746 (q4+T742) encode 1.32s +``` +The leading `<|...|>` tags are the predicted language / emotion / event / ITN. + +## Accuracy & validation + +- **CTC token ids (C++) vs PyTorch:** **identical** (108/108 on a benchmark clip). +- **Detokenized text:** matches the FunASR `AutoModel` output **exactly**. +- Encoder validated against PyTorch (shared with Fun-ASR-Nano runtime): cosine 1.0. +- Encode time ≈ **1.3 s** on CPU for a 44 s clip. + +## Tips & gotchas + +- **No CMVN at inference.** SenseVoice `inference()` feeds the **raw** log-mel fbank + to the encoder; it does **not** apply `am.mvn`. Applying CMVN makes the model + predict `<|nospeech|>`. (The export script reads `am.mvn` for completeness but the + runtime does not use it.) +- **Query tokens (4)** are prepended from `embed.weight`, default indices + `[language=auto(0), event=1, emotion=2, textnorm=woitn(15)]`. Change them for a + fixed language or to enable ITN (`withitn=14`). +- **WAV input** assumes 16 kHz mono PCM16. +- LayerNorm eps = 1e-5; FSMN = exact f32 shift-accumulate; fbank matches torchaudio. + +## Files +``` +funasr-sensevoice/ ggml runtime: WAV → CTC token ids +export_sensevoice_gguf.py export encoder + CTC head + query embeddings to GGUF +detok.py SentencePiece id → text (bpe model ships with the checkpoint) +``` + +## Roadmap +- Built-in SentencePiece detok (drop the Python step); arbitrary WAV formats; + encoder Q8 quantization; timestamps. diff --git a/runtime/llama.cpp/sensevoice/detok.py b/runtime/llama.cpp/sensevoice/detok.py new file mode 100644 index 000000000..48b11c953 --- /dev/null +++ b/runtime/llama.cpp/sensevoice/detok.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +"""Detokenize SenseVoice CTC token ids -> text via the SentencePiece bpe model. +Usage: python detok.py (ids.txt = space-separated ints) +""" +import sys +import sentencepiece as spm + +sp = spm.SentencePieceProcessor(model_file=sys.argv[1]) +ids = [int(x) for x in open(sys.argv[2]).read().split()] +print(sp.decode(ids)) diff --git a/runtime/llama.cpp/sensevoice/export_sensevoice_gguf.py b/runtime/llama.cpp/sensevoice/export_sensevoice_gguf.py new file mode 100644 index 000000000..997cca5f9 --- /dev/null +++ b/runtime/llama.cpp/sensevoice/export_sensevoice_gguf.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Export SenseVoiceSmall (encoder + CTC head + query embeddings + CMVN) to GGUF +for the ggml C++ runtime. The encoder is the same SAN-M architecture as +Fun-ASR-Nano, so the C++ forward is shared. +""" +import argparse, os, re +import numpy as np, torch, gguf + + +def parse_mvn(path): + """am.mvn (kaldi nnet): two ` 0 [ ... ]` blocks -> shift, scale. + apply: out = (in + shift) * scale, per-dim (560).""" + txt = open(path).read() + blocks = re.findall(r"\[([^\]]*)\]", txt) + shift = np.array([float(x) for x in blocks[0].split()], dtype=np.float32) + scale = np.array([float(x) for x in blocks[1].split()], dtype=np.float32) + return shift, scale + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--model_pt", required=True) + ap.add_argument("--mvn", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--wtype", default="f32", choices=["f32", "f16"]) + args = ap.parse_args() + + sd = torch.load(args.model_pt, map_location="cpu"); sd = sd.get("state_dict", sd) + w = gguf.GGUFWriter(args.out, "sensevoice-small") + w.add_uint32("sv.input_size", 560) + w.add_uint32("sv.output_size", 512) + w.add_uint32("sv.attention_heads", 4) + w.add_uint32("sv.num_blocks", 50) + w.add_uint32("sv.tp_blocks", 20) + w.add_uint32("sv.kernel_size", 11) + w.add_uint32("sv.vocab_size", 25055) + w.add_uint32("sv.blank_id", 0) + # query token embed indices used at inference: [lid(auto=0), 1, 2, textnorm(woitn=15)] + w.add_array("sv.query_tokens", [0, 1, 2, 15]) + + shift, scale = parse_mvn(args.mvn) + w.add_tensor("cmvn.shift", shift) # (560,) + w.add_tensor("cmvn.scale", scale) # (560,) + + n = 0 + for k, v in sd.items(): + if not (k.startswith("encoder.") or k.startswith("ctc.") or k == "embed.weight"): + continue + arr = v.detach().to(torch.float32).contiguous().numpy() + if k.endswith("fsmn_block.weight"): # (D,1,K) -> (K,D) + arr = np.ascontiguousarray(arr[:, 0, :].T) + elif args.wtype == "f16" and arr.ndim == 2 and "norm" not in k: + arr = arr.astype(np.float16) + w.add_tensor(k, arr) + n += 1 + print(f"writing {n} tensors (+cmvn) to {args.out}") + w.write_header_to_file(); w.write_kv_data_to_file(); w.write_tensors_to_file(); w.close() + print(f"done: {args.out} ({os.path.getsize(args.out)/1e6:.1f} MB)") + + +if __name__ == "__main__": + main() diff --git a/runtime/llama.cpp/sensevoice/funasr-sensevoice/CMakeLists.txt b/runtime/llama.cpp/sensevoice/funasr-sensevoice/CMakeLists.txt new file mode 100644 index 000000000..41ab1e443 --- /dev/null +++ b/runtime/llama.cpp/sensevoice/funasr-sensevoice/CMakeLists.txt @@ -0,0 +1,5 @@ +set(TARGET llama-funasr-sensevoice) +add_executable(${TARGET} funasr-sensevoice.cpp) +install(TARGETS ${TARGET} RUNTIME) +target_link_libraries(${TARGET} PRIVATE ggml ${CMAKE_THREAD_LIBS_INIT}) +target_compile_features(${TARGET} PRIVATE cxx_std_17) diff --git a/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp b/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp new file mode 100644 index 000000000..2f9f20046 --- /dev/null +++ b/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp @@ -0,0 +1,179 @@ +// funasr-sensevoice: SenseVoiceSmall (SAN-M encoder + CTC) on ggml. +// fbank.bin (T x 560) -> CMVN -> prepend 4 query tokens -> SAN-M encoder -> +// CTC head -> greedy CTC decode -> token ids (stdout). +// The encoder is the same SAN-M arch as Fun-ASR-Nano (shared forward). +// Detokenize the printed ids with the SentencePiece bpe model (Python side for now). + +#include "ggml.h" +#include "ggml-cpu.h" +#include "ggml-alloc.h" +#include "ggml-backend.h" +#include "gguf.h" + +#include +#include +#include +#include +#include +#include +#include + +static const float LN_EPS = 1e-5f; + +// ---- WAV (16k mono PCM16) + kaldi fbank (shared with funasr-cli) ---- +static bool read_wav16(const char*path,std::vector&out){ + FILE*f=fopen(path,"rb"); if(!f)return false; char r[4]; fread(r,1,4,f); + if(strncmp(r,"RIFF",4)){fclose(f);return false;} fseek(f,4,SEEK_CUR); char wv[4]; fread(wv,1,4,f); + if(strncmp(wv,"WAVE",4)){fclose(f);return false;} uint16_t ch=1,bits=16; uint32_t sr=16000; + while(!feof(f)){char id[4]; if(fread(id,1,4,f)!=4)break; uint32_t sz; if(fread(&sz,4,1,f)!=1)break; + if(!strncmp(id,"fmt ",4)){uint16_t fmt; fread(&fmt,2,1,f); fread(&ch,2,1,f); fread(&sr,4,1,f); fseek(f,6,SEEK_CUR); fread(&bits,2,1,f); if(sz>16)fseek(f,sz-16,SEEK_CUR);} + else if(!strncmp(id,"data",4)){if(ch<1||bits!=16){fclose(f);fprintf(stderr,"only 16-bit PCM WAV (ch=%u bits=%u)\n",ch,bits);return false;} int n=sz/(bits/8); std::vector pcm(n); fread(pcm.data(),2,n,f); out.resize(n/ch); + for(int i=0;i&re,std::vector&im,int n){ + for(int i=1,j=0;i>1;for(;j&b;b>>=1)j^=b;j^=b;if(i compute_fbank(std::vector wav,int&T_out){ + for(auto&v:wav)v*=32768.0f; std::vector win(WINLEN); + for(int i=0;i> fb(NMEL,std::vector(NBIN,0.0f)); + for(int m=0;mL&&mf> feat(T,std::vector(NMEL)); + std::vector re(NFFT),im(NFFT),fr(WINLEN); const float fl=1.1920929e-07f; + for(int t=0;t0;i--)fr[i]-=PREEMPH*fr[i-1]; fr[0]-=PREEMPH*fr[0]; + for(int i=0;i0)e+=fb[m][k]*(re[k]*re[k]+im[k]*im[k]); feat[t][m]=logf(e>fl?e:fl);}} + const int pad=(LFR_M-1)/2; int Tl=(T+LFR_N-1)/LFR_N; std::vector> pd; pd.reserve(T+pad+LFR_M); + for(int i=0;i out((size_t)Tl*D); + for(int i=0;i t; + ggml_tensor* g(const std::string&n){auto it=t.find(n);if(it==t.end()){fprintf(stderr,"missing %s\n",n.c_str());exit(1);}return it->second;} }; + +static ggml_tensor* lin(ggml_context*c,ggml_tensor*w,ggml_tensor*b,ggml_tensor*x){auto y=ggml_mul_mat(c,w,x);return b?ggml_add(c,y,b):y;} +static ggml_tensor* lnorm(ggml_context*c,ggml_tensor*x,ggml_tensor*g,ggml_tensor*b){return ggml_add(c,ggml_mul(c,ggml_norm(c,x,LN_EPS),g),b);} +static ggml_tensor* sanm_attn(ggml_context*c,model&m,const std::string&p,ggml_tensor*x,int T){ + const int D=m.c.d_model,H=m.c.n_head,dk=D/H,K=m.c.kernel; + ggml_tensor*qkv=lin(c,m.g(p+"linear_q_k_v.weight"),m.g(p+"linear_q_k_v.bias"),x); size_t nb1=qkv->nb[1]; + ggml_tensor*q=ggml_cont(c,ggml_view_2d(c,qkv,D,T,nb1,0)); + ggml_tensor*k=ggml_cont(c,ggml_view_2d(c,qkv,D,T,nb1,(size_t)D*sizeof(float))); + ggml_tensor*v=ggml_cont(c,ggml_view_2d(c,qkv,D,T,nb1,(size_t)2*D*sizeof(float))); + const int pad=(K-1)/2; ggml_tensor*fk=m.g(p+"fsmn_block.weight"); + ggml_tensor*vp=ggml_pad_ext(c,v,0,0,pad,pad,0,0,0,0); ggml_tensor*fsmn=v; + for(int j=0;jnb[1],(size_t)j*vp->nb[1]); + auto wj=ggml_view_1d(c,fk,D,(size_t)j*fk->nb[1]); fsmn=ggml_add(c,fsmn,ggml_mul(c,ggml_cont(c,sl),wj));} + q=ggml_permute(c,ggml_reshape_3d(c,q,dk,H,T),0,2,1,3); k=ggml_permute(c,ggml_reshape_3d(c,k,dk,H,T),0,2,1,3); + ggml_tensor*vh=ggml_cont(c,ggml_permute(c,ggml_reshape_3d(c,v,dk,H,T),1,2,0,3)); + ggml_tensor*kq=ggml_soft_max(c,ggml_scale(c,ggml_mul_mat(c,k,q),1.0f/sqrtf((float)dk))); + ggml_tensor*o=ggml_cont_2d(c,ggml_permute(c,ggml_mul_mat(c,vh,kq),0,2,1,3),D,T); + return ggml_add(c,lin(c,m.g(p+"linear_out.weight"),m.g(p+"linear_out.bias"),o),fsmn); +} +static ggml_tensor* sanm_layer(ggml_context*c,model&m,const std::string&p,ggml_tensor*x,int T,bool res){ + auto r=x; auto h=lnorm(c,x,m.g(p+"norm1.weight"),m.g(p+"norm1.bias")); + auto sa=sanm_attn(c,m,p+"self_attn.",h,T); x=res?ggml_add(c,r,sa):sa; r=x; + h=lnorm(c,x,m.g(p+"norm2.weight"),m.g(p+"norm2.bias")); + h=lin(c,m.g(p+"feed_forward.w_1.weight"),m.g(p+"feed_forward.w_1.bias"),h); h=ggml_relu(c,h); + h=lin(c,m.g(p+"feed_forward.w_2.weight"),m.g(p+"feed_forward.w_2.bias"),h); return ggml_add(c,r,h); +} +static void add_posenc(std::vector&x,int T,int depth){ + double inc=log(10000.0)/(depth/2.0-1.0); + for(int t=0;t qtok(nq); for(int i=0;i fb; + if(!wav_path.empty()){ + std::vector wav; if(!read_wav16(wav_path.c_str(),wav)){fprintf(stderr,"read wav failed\n");return 1;} + int t=0; fb=compute_fbank(wav,t); T=t; + } else { + FILE*f=fopen(fbank_path.c_str(),"rb"); if(!f){fprintf(stderr,"open fbank\n");return 1;} + if(fread(&T,4,1,f)!=1||fread(&Fc,4,1,f)!=1){fclose(f);return 1;} + fb.resize((size_t)T*Fc); if((int)fread(fb.data(),4,fb.size(),f)!=(int)fb.size()){fclose(f);return 1;} fclose(f); + } + + // NOTE: SenseVoiceSmall inference() feeds the RAW log-mel fbank to the encoder; + // it does NOT apply am.mvn CMVN (that path is unused at inference). Applying it + // makes the encoder predict <|nospeech|>. So no CMVN here. + + // prepend nq query tokens (rows of embed.weight) -> input [nq+T, F] + float*emb=(float*)m.g("embed.weight")->data; // [16, 560] row-major + int N=nq+T; std::vector inp((size_t)N*F); + for(int i=0;i lg((size_t)V*N); ggml_backend_tensor_get(logits,lg.data(),0,ggml_nbytes(logits)); + // greedy CTC: argmax per frame -> collapse consecutive -> drop blank + int prev=-1; + for(int n=0;nbest){best=col[v];am=v;} + if(am!=prev && am!=m.c.blank) printf("%d ", am); + prev=am; + } + printf("\n"); + fprintf(stderr,"[sensevoice] N=%d (q%d+T%d) encode %.2fs\n",N,nq,T,(t1-t0)/1e6); + ggml_gallocr_free(ga); ggml_free(c); ggml_backend_free(be); + if(m.ctx_w) ggml_free(m.ctx_w); + return 0; +}