mirror of
https://github.com/modelscope/FunASR.git
synced 2026-10-12 02:19:18 +00:00
feat: built-in FSMN-VAD (--vad) for all three runtimes — single-binary segmentation (#2998)
Add a native ggml FSMN-VAD front end so the binaries segment long audio themselves, with no Python at runtime. New `--vad fsmn-vad.gguf` flag on llama-funasr-cli, llama-funasr-sensevoice and llama-funasr-paraformer runs FSMN-VAD (fbank + LFR + CMVN + FSMN encoder + E2EVadModel state machine, in funasr-common/funasr_vad.h) and decodes each speech segment, then concatenates. - funasr-common/funasr_vad.h: single-header VAD; segment boundaries within ~10 ms of PyTorch fsmn-vad.generate on the 184-clip set (DEFAULT_SILENCE_SCHEDULE replicated, chunk-stepped silence schedule + speech_noise_thres 0.5). - funasr-vad/: standalone VAD tool (wav -> [start_ms,end_ms] segments). - export_vad_gguf.py: export FSMN-VAD encoder + CMVN to GGUF (1.7 MB, 24 tensors). - BENCHMARKS.md: bare binary with built-in VAD reaches the reference end-to-end — full-184 micro-CER SenseVoice 8.01 / Paraformer 9.85 / Fun-ASR-Nano 8.30. Additive only: no existing code is modified outside runtime/llama.cpp/.
This commit is contained in:
zhifu gao
@@ -38,12 +38,15 @@ separates model-load from compute.
|
||||
- **Segmentation differs by system, each using its natural strategy:** FunASR uses an
|
||||
`fsmn-vad` front end (segments → ASR → concatenate); whisper.cpp uses its built-in
|
||||
30 s windowing. This is a fair system-level comparison.
|
||||
- **Engine-internal VAD is on the FunASR runtime roadmap.** Today the runtime reaches
|
||||
the reference numbers via the `fsmn-vad` front end (or `--chunk`); a native ggml VAD
|
||||
is planned so the bare binary hits the reference end-to-end.
|
||||
- **Bare binary, no VAD (whole-clip)**, for full disclosure: SenseVoiceSmall **9.99 %**,
|
||||
Paraformer **12.82 %** (micro, normalize_zh, full 184). Long clips decoded as one
|
||||
segment are out-of-distribution; this is exactly what VAD fixes.
|
||||
- **Engine-internal VAD is now implemented** — a native ggml FSMN-VAD built into the
|
||||
binaries (`--vad fsmn-vad.gguf`). The **bare binary, with no Python front end**, now
|
||||
reaches the reference end-to-end: SenseVoiceSmall **8.01 %**, Paraformer **9.85 %**,
|
||||
Fun-ASR-Nano **8.30 %** (micro, normalize_zh, full 184). The built-in C++ VAD matches
|
||||
the PyTorch `fsmn-vad` front end (segment boundaries within ~10 ms, slightly better
|
||||
CER), so the runtime is now fully self-contained.
|
||||
- For full disclosure, **bare binary with no VAD at all (whole-clip)** is higher —
|
||||
SenseVoiceSmall 9.99 %, Paraformer 12.82 % — because long clips decoded as one segment
|
||||
are out-of-distribution; that is exactly what the built-in VAD fixes.
|
||||
|
||||
## Why FunASR wins on Chinese
|
||||
|
||||
|
||||
@@ -36,6 +36,9 @@ echo 'add_subdirectory(<example-dir>)' >> examples/CMakeLists.txt
|
||||
cmake -B build -DGGML_NATIVE=ON -DLLAMA_CURL=OFF
|
||||
cmake --build build -j --target <target>
|
||||
```
|
||||
The shared **FSMN-VAD** front end builds the same way (`funasr-vad/` + `funasr-common/`,
|
||||
target `llama-funasr-vad`); export weights with `export_vad_gguf.py`. Pass
|
||||
`--vad fsmn-vad.gguf` to any of the three tools for built-in long-audio segmentation.
|
||||
|
||||
## Validation
|
||||
|
||||
@@ -44,9 +47,11 @@ SenseVoice CTC token ids identical; Paraformer text identical; Fun-ASR-Nano aggr
|
||||
CER matches PyTorch within 0.02% under identical conditions). See per-model READMEs.
|
||||
|
||||
## Status / notes
|
||||
- WAV input currently assumes 16 kHz mono PCM16.
|
||||
- For long audio, Fun-ASR-Nano supports `--chunk` windowing; a proper VAD front end
|
||||
is on the roadmap.
|
||||
- Any audio in (wav/mp3/flac, any rate/channels) via the bundled miniaudio loader.
|
||||
- **Built-in FSMN-VAD (`--vad fsmn-vad.gguf`)** segments long audio inside the binary
|
||||
(native ggml, no Python front end); all three tools support it. Bare-binary full-184
|
||||
micro-CER: SenseVoice **8.01** / Paraformer **9.85** / Fun-ASR-Nano **8.30** (see
|
||||
[BENCHMARKS.md](BENCHMARKS.md)). `--chunk` fixed-window remains a simpler fallback.
|
||||
- This adds a new `runtime/llama.cpp/` directory only; no existing code is modified.
|
||||
|
||||
## Further reading
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Export FSMN-VAD (encoder + CMVN) to GGUF for the ggml C++ runtime."""
|
||||
import argparse, os, re
|
||||
import numpy as np, torch, gguf
|
||||
|
||||
def parse_mvn(path):
|
||||
b=[np.array([float(x) for x in m.split()],np.float32) for m in re.findall(r"\[([^\]]*)\]",open(path).read())]
|
||||
v=[x for x in b if x.size>1]; return v[0], v[1] # shift, scale (both 400-dim)
|
||||
|
||||
def main():
|
||||
ap=argparse.ArgumentParser()
|
||||
ap.add_argument("--model_pt",required=True); ap.add_argument("--mvn",required=True); ap.add_argument("--out",required=True)
|
||||
a=ap.parse_args()
|
||||
sd=torch.load(a.model_pt,map_location="cpu"); sd=sd.get("state_dict",sd)
|
||||
w=gguf.GGUFWriter(a.out,"fsmn-vad")
|
||||
w.add_uint32("vad.input_dim",400); w.add_uint32("vad.input_affine_dim",140)
|
||||
w.add_uint32("vad.linear_dim",250); w.add_uint32("vad.proj_dim",128)
|
||||
w.add_uint32("vad.fsmn_layers",4); w.add_uint32("vad.lorder",20)
|
||||
w.add_uint32("vad.output_affine_dim",140); w.add_uint32("vad.output_dim",248)
|
||||
w.add_uint32("vad.n_mels",80); w.add_uint32("vad.lfr_m",5); w.add_uint32("vad.lfr_n",1)
|
||||
shift,scale=parse_mvn(a.mvn); w.add_tensor("cmvn.shift",shift); w.add_tensor("cmvn.scale",scale)
|
||||
n=0
|
||||
for k,v in sd.items():
|
||||
if not k.startswith("encoder."): continue
|
||||
arr=v.detach().to(torch.float32).contiguous().numpy()
|
||||
if k.endswith("conv_left.weight"): # (C,1,lorder,1) -> (lorder,C) tap-major
|
||||
arr=np.ascontiguousarray(arr[:,0,:,0].T)
|
||||
w.add_tensor(k,arr); n+=1
|
||||
print(f"writing {n} tensors (+cmvn) to {a.out}")
|
||||
w.write_header_to_file(); w.write_kv_data_to_file(); w.write_tensors_to_file(); w.close()
|
||||
print(f"done: {a.out} ({os.path.getsize(a.out)/1e6:.1f} MB)")
|
||||
|
||||
if __name__=="__main__": main()
|
||||
@@ -26,6 +26,9 @@
|
||||
// any audio (wav/mp3/flac, any rate/channels) -> 16 kHz mono f32, via miniaudio
|
||||
#define FUNASR_AUDIO_IMPLEMENTATION
|
||||
#include "funasr_audio.h"
|
||||
// built-in FSMN-VAD front end (single-binary --vad segmentation)
|
||||
#include "funasr_vad.h"
|
||||
#include <utility>
|
||||
|
||||
// ======================= kaldi fbank + LFR =======================
|
||||
static const int FS=16000, WINLEN=400, SHIFT=160, NFFT=512, NMEL=80, LFR_M=7, LFR_N=6;
|
||||
@@ -164,15 +167,18 @@ static int decode_batch(llama_context*ctx,int n,llama_token*tok,float*embd,int n
|
||||
}
|
||||
|
||||
int main(int argc,char**argv){
|
||||
std::string enc_path,llm_path,wav_path; int npred=512; double chunk_sec=0; float rep=1.0f;
|
||||
std::string enc_path,llm_path,wav_path,vad_path; int npred=512; double chunk_sec=0; float rep=1.0f;
|
||||
int vad_maxseg=30000;
|
||||
for(int i=1;i<argc;i++){
|
||||
if(!strcmp(argv[i],"--enc")&&i+1<argc)enc_path=argv[++i];
|
||||
else if(!strcmp(argv[i],"-m")&&i+1<argc)llm_path=argv[++i];
|
||||
else if(!strcmp(argv[i],"-a")&&i+1<argc)wav_path=argv[++i];
|
||||
else if(!strcmp(argv[i],"-n")&&i+1<argc)npred=atoi(argv[++i]);
|
||||
else if(!strcmp(argv[i],"--chunk")&&i+1<argc)chunk_sec=atof(argv[++i]);
|
||||
else if(!strcmp(argv[i],"--vad")&&i+1<argc)vad_path=argv[++i];
|
||||
else if(!strcmp(argv[i],"--vad-maxseg")&&i+1<argc)vad_maxseg=atoi(argv[++i]);
|
||||
else if(!strcmp(argv[i],"--rep")&&i+1<argc)rep=atof(argv[++i]);
|
||||
else {fprintf(stderr,"usage: %s --enc enc.gguf -m llm.gguf -a audio.wav [-n npred] [--chunk sec]\n",argv[0]);return 1;}
|
||||
else {fprintf(stderr,"usage: %s --enc enc.gguf -m llm.gguf -a audio.wav [-n npred] [--chunk sec] [--vad fsmn-vad.gguf [--vad-maxseg ms]]\n",argv[0]);return 1;}
|
||||
}
|
||||
if(enc_path.empty()||llm_path.empty()||wav_path.empty()){fprintf(stderr,"missing args\n");return 1;}
|
||||
|
||||
@@ -199,12 +205,24 @@ int main(int argc,char**argv){
|
||||
std::vector<llama_token> v(n); llama_tokenize(vocab,s,strlen(s),v.data(),n,false,true); return v;};
|
||||
auto pre=tokenize(prefix); auto suf=tokenize(suffix);
|
||||
|
||||
// split into chunks (chunk_sec<=0 -> whole file)
|
||||
int chunk_n = chunk_sec > 0 ? std::max(1, (int)(chunk_sec*16000)) : (int)wav.size();
|
||||
// Build the list of [offset,len] windows to transcribe (in samples).
|
||||
// --vad : FSMN-VAD speech segments (single-binary front end, replaces fixed chunking)
|
||||
// --chunk sec : fixed-size chunks ; otherwise the whole file in one window
|
||||
std::vector<std::pair<int,int>> wins; // {sample offset, sample len}
|
||||
if(!vad_path.empty()){
|
||||
std::vector<std::pair<int,int>> segs; // ms
|
||||
if(!funasr_vad_segments(vad_path,wav,vad_maxseg,segs)){fprintf(stderr,"vad failed\n");return 1;}
|
||||
for(auto&s:segs){ int off=(int)((int64_t)s.first*16000/1000), end=(int)((int64_t)s.second*16000/1000);
|
||||
if(end>(int)wav.size())end=wav.size(); if(end-off>0) wins.push_back({off,end-off}); }
|
||||
fprintf(stderr,"[vad] %zu segments\n",wins.size());
|
||||
} else {
|
||||
int chunk_n = chunk_sec > 0 ? std::max(1, (int)(chunk_sec*16000)) : (int)wav.size();
|
||||
for(size_t off=0; off<wav.size(); off+=chunk_n) wins.push_back({(int)off,(int)std::min((size_t)chunk_n,wav.size()-off)});
|
||||
}
|
||||
std::string full;
|
||||
for (size_t off = 0; off < wav.size(); off += chunk_n) {
|
||||
int len = std::min((size_t)chunk_n, wav.size()-off);
|
||||
if (len < WINLEN) break; // too short for one frame
|
||||
for (auto& w : wins) {
|
||||
int off = w.first, len = w.second;
|
||||
if (len < WINLEN) continue; // too short for one frame
|
||||
std::vector<float> seg(wav.begin()+off, wav.begin()+off+len);
|
||||
int T=0; auto fbank=compute_fbank(seg,T);
|
||||
int D=0; auto adp=run_encoder(em,fbank,T,560,D);
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
// funasr_vad.h — single-header FSMN-VAD for the funasr ggml runtime.
|
||||
// Exposes funasr_vad_segments(): 16k mono wav -> speech segments [start_ms,end_ms].
|
||||
// Front end (80-mel fbank + LFR m5n1 + CMVN) and FSMN encoder validated bit-exact vs
|
||||
// PyTorch fsmn-vad; the host state machine reproduces E2EVadModel (DEFAULT_SILENCE_SCHEDULE,
|
||||
// chunk-stepped) to within 1 frame (10ms) of fsmn-vad.generate on the 184-clip set.
|
||||
#pragma once
|
||||
#include "ggml.h"
|
||||
#include "ggml-cpu.h"
|
||||
#include "ggml-alloc.h"
|
||||
#include "ggml-backend.h"
|
||||
#include "gguf.h"
|
||||
#include <cmath>
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace funasr_vad_impl {
|
||||
static const int FS=16000,WINLEN=400,SHIFT=160,NFFT=512,NMEL=80;
|
||||
static const float PREEMPH=0.97f,LOWF=20.0f,HIGHF=8000.0f;
|
||||
static inline float melf(float f){return 1127.0f*logf(1.0f+f/700.0f);}
|
||||
static void fftc(std::vector<float>&re,std::vector<float>&im,int n){for(int i=1,j=0;i<n;i++){int b=n>>1;for(;j&b;b>>=1)j^=b;j^=b;if(i<j){std::swap(re[i],re[j]);std::swap(im[i],im[j]);}}
|
||||
for(int len=2;len<=n;len<<=1){double a=-2.0*M_PI/len;float wr=cosf(a),wi=sinf(a);for(int i=0;i<n;i+=len){float cr=1,ci=0;for(int k=0;k<len/2;k++){float ur=re[i+k],ui=im[i+k];
|
||||
float vr=re[i+k+len/2]*cr-im[i+k+len/2]*ci,vi=re[i+k+len/2]*ci+im[i+k+len/2]*cr;re[i+k]=ur+vr;im[i+k]=ui+vi;re[i+k+len/2]=ur-vr;im[i+k+len/2]=ui-vi;float nc=cr*wr-ci*wi;ci=cr*wi+ci*wr;cr=nc;}}}}
|
||||
static std::vector<std::vector<float>> fbank80(std::vector<float> wav){
|
||||
for(auto&v:wav)v*=32768.0f; std::vector<float>win(WINLEN);
|
||||
for(int i=0;i<WINLEN;i++)win[i]=0.54f-0.46f*cosf(2.0f*M_PI*i/(WINLEN-1));
|
||||
const int NB=NFFT/2+1; float bw=(float)FS/NFFT,ml=melf(LOWF),mh=melf(HIGHF),dm=(mh-ml)/(NMEL+1);
|
||||
std::vector<std::vector<float>>fb(NMEL,std::vector<float>(NB,0.0f));
|
||||
for(int m=0;m<NMEL;m++){float L=ml+m*dm,C=ml+(m+1)*dm,R=ml+(m+2)*dm;for(int k=0;k<NB;k++){float mf=melf(bw*k);if(mf>L&&mf<R)fb[m][k]=mf<=C?(mf-L)/(C-L):(R-mf)/(R-C);}}
|
||||
int N=wav.size(),T=(N-WINLEN)/SHIFT+1; if(T<1)T=0; std::vector<std::vector<float>>feat(T,std::vector<float>(NMEL));
|
||||
std::vector<float>re(NFFT),im(NFFT),fr(WINLEN);const float fl=1.1920929e-07f;
|
||||
for(int t=0;t<T;t++){const float*s=wav.data()+t*SHIFT;double mn=0;for(int i=0;i<WINLEN;i++)mn+=s[i];mn/=WINLEN;
|
||||
for(int i=0;i<WINLEN;i++)fr[i]=s[i]-(float)mn;for(int i=WINLEN-1;i>0;i--)fr[i]-=PREEMPH*fr[i-1];fr[0]-=PREEMPH*fr[0];
|
||||
for(int i=0;i<NFFT;i++){re[i]=i<WINLEN?fr[i]*win[i]:0.0f;im[i]=0.0f;}fftc(re,im,NFFT);
|
||||
for(int m=0;m<NMEL;m++){float e=0;for(int k=0;k<NB;k++)if(fb[m][k]>0)e+=fb[m][k]*(re[k]*re[k]+im[k]*im[k]);feat[t][m]=logf(e>fl?e:fl);}}
|
||||
return feat;
|
||||
}
|
||||
static std::vector<float> lfr(const std::vector<std::vector<float>>&feat,int m,int n,int&T_out){
|
||||
int T=feat.size(),D=NMEL,pad=(m-1)/2; int Tl=T>0?(T+n-1)/n:0;
|
||||
std::vector<std::vector<float>> pf; pf.reserve(T+pad+m);
|
||||
for(int i=0;i<pad;i++)pf.push_back(feat[0]);
|
||||
for(int t=0;t<T;t++)pf.push_back(feat[t]);
|
||||
while((int)pf.size()<(Tl-1)*n+m)pf.push_back(feat[T-1]);
|
||||
std::vector<float> out((size_t)Tl*m*D);
|
||||
for(int i=0;i<Tl;i++)for(int j=0;j<m;j++)memcpy(&out[((size_t)i*m+j)*D],pf[i*n+j].data(),D*sizeof(float));
|
||||
T_out=Tl; return out;
|
||||
}
|
||||
struct vad{ggml_context*ctx=nullptr;std::map<std::string,ggml_tensor*>t;
|
||||
ggml_tensor*g(const std::string&n){auto it=t.find(n);if(it==t.end()){fprintf(stderr,"vad: missing %s\n",n.c_str());return nullptr;}return it->second;}};
|
||||
static ggml_tensor* lin(ggml_context*c,ggml_tensor*w,ggml_tensor*b,ggml_tensor*x){auto y=ggml_mul_mat(c,w,x);return b?ggml_add(c,y,b):y;}
|
||||
} // namespace funasr_vad_impl
|
||||
|
||||
// Run FSMN-VAD on a 16k mono float waveform; fills segs with [start_ms,end_ms] speech spans.
|
||||
// max_seg_ms caps a single segment (e.g. 30000); pass <=0 to use 60000 (model default).
|
||||
inline bool funasr_vad_segments(const std::string& gguf_path, const std::vector<float>& wav,
|
||||
int max_seg_ms, std::vector<std::pair<int,int>>& segs, int nthreads=8){
|
||||
using namespace funasr_vad_impl;
|
||||
segs.clear();
|
||||
vad m; gguf_init_params ip={false,&m.ctx}; gguf_context*gg=gguf_init_from_file(gguf_path.c_str(),ip);
|
||||
if(!gg){fprintf(stderr,"vad: cannot load %s\n",gguf_path.c_str());return false;}
|
||||
auto rd=[&](const char*k,int d){int i=gguf_find_key(gg,k);return i<0?d:(int)gguf_get_val_u32(gg,i);};
|
||||
int idim=rd("vad.input_dim",400),pd=rd("vad.proj_dim",128),nl=rd("vad.fsmn_layers",4),lorder=rd("vad.lorder",20),
|
||||
od=rd("vad.output_dim",248),lm=rd("vad.lfr_m",5),ln=rd("vad.lfr_n",1);
|
||||
for(int i=0;i<gguf_get_n_tensors(gg);i++){const char*nm=gguf_get_tensor_name(gg,i);m.t[nm]=ggml_get_tensor(m.ctx,nm);}
|
||||
gguf_free(gg);
|
||||
|
||||
auto feat=fbank80(wav); int T=0; auto feats=lfr(feat,lm,ln,T); // [T,400]
|
||||
if(T<1){if(m.ctx)ggml_free(m.ctx);return true;} // too short -> no speech
|
||||
float*shift=(float*)m.g("cmvn.shift")->data,*scale=(float*)m.g("cmvn.scale")->data;
|
||||
for(int t=0;t<T;t++)for(int d=0;d<idim;d++)feats[(size_t)t*idim+d]=(feats[(size_t)t*idim+d]+shift[d])*scale[d];
|
||||
|
||||
ggml_backend_t be=ggml_backend_cpu_init();
|
||||
ggml_init_params cp={(size_t)512*1024*1024,nullptr,true}; ggml_context*c=ggml_init(cp);
|
||||
ggml_tensor*x=ggml_new_tensor_2d(c,GGML_TYPE_F32,idim,T); ggml_set_input(x);
|
||||
ggml_tensor*h=lin(c,m.g("encoder.in_linear1.linear.weight"),m.g("encoder.in_linear1.linear.bias"),x);
|
||||
h=lin(c,m.g("encoder.in_linear2.linear.weight"),m.g("encoder.in_linear2.linear.bias"),h); h=ggml_relu(c,h);
|
||||
for(int i=0;i<nl;i++){std::string p="encoder.fsmn."+std::to_string(i)+".";
|
||||
ggml_tensor*z=ggml_mul_mat(c,m.g(p+"linear.linear.weight"),h);
|
||||
ggml_tensor*fk=m.g(p+"fsmn_block.conv_left.weight"); ggml_tensor*zp=ggml_pad_ext(c,z,0,0,lorder-1,0,0,0,0,0); ggml_tensor*acc=z;
|
||||
for(int j=0;j<lorder;j++){auto sl=ggml_view_2d(c,zp,pd,T,zp->nb[1],(size_t)j*zp->nb[1]);auto wj=ggml_view_1d(c,fk,pd,(size_t)j*fk->nb[1]);acc=ggml_add(c,acc,ggml_mul(c,ggml_cont(c,sl),wj));}
|
||||
ggml_tensor*a=lin(c,m.g(p+"affine.linear.weight"),m.g(p+"affine.linear.bias"),acc); h=ggml_relu(c,a);}
|
||||
h=lin(c,m.g("encoder.out_linear1.linear.weight"),m.g("encoder.out_linear1.linear.bias"),h);
|
||||
h=lin(c,m.g("encoder.out_linear2.linear.weight"),m.g("encoder.out_linear2.linear.bias"),h);
|
||||
h=ggml_soft_max(c,h); ggml_set_output(h);
|
||||
ggml_cgraph*gf=ggml_new_graph(c); ggml_build_forward_expand(gf,h);
|
||||
ggml_gallocr_t ga=ggml_gallocr_new(ggml_backend_cpu_buffer_type()); ggml_gallocr_alloc_graph(ga,gf);
|
||||
ggml_backend_tensor_set(x,feats.data(),0,ggml_nbytes(x)); ggml_backend_cpu_set_n_threads(be,nthreads);
|
||||
bool ok=ggml_backend_graph_compute(be,gf)==GGML_STATUS_SUCCESS;
|
||||
std::vector<float> sc((size_t)od*T); if(ok)ggml_backend_tensor_get(h,sc.data(),0,ggml_nbytes(h));
|
||||
ggml_gallocr_free(ga);ggml_free(c);ggml_backend_free(be);if(m.ctx)ggml_free(m.ctx);
|
||||
if(!ok)return false;
|
||||
|
||||
// ===== E2EVadModel state machine (host) -> speech segments [start_ms,end_ms] =====
|
||||
const int FR=10; // ms per frame (frame_in_ms)
|
||||
const int win=20; // window_size_ms 200 / 10
|
||||
const int s2s=15, sp2s=15; // sil_to_speech / speech_to_sil thres (150/10)
|
||||
const int lookahead_end=100/FR; // lookahead_time_end_point 100/10 = 10 (do_extend)
|
||||
int max_seg = (max_seg_ms>0 ? max_seg_ms : 60000)/FR; // max_single_segment frames
|
||||
const int start_lookback = win + 200/FR; // LatencyFrmNumAtStartPoint = 40
|
||||
// End-silence threshold from DEFAULT_SILENCE_SCHEDULE, stepped at chunk boundaries (chunk_size
|
||||
// 60000ms = 6000 frames). At each boundary, if mid-segment (InSpeech) or in_speech latched,
|
||||
// accumulated_ms += 60000; threshold = schedule(accumulated). Reset to 0 on each segment emit.
|
||||
const int CHUNK=60000/FR;
|
||||
int acc=0, insp=0; int max_end_sil, end_lookback;
|
||||
auto recompute=[&](){
|
||||
int s; if(acc<=10000)s=2000; else if(acc<=20000)s=1000; else if(acc<=30000)s=800;
|
||||
else if(acc<=40000)s=600; else if(acc<=50000)s=400; else if(acc<=60000)s=200; else s=100;
|
||||
int ms=s-150; if(ms<0)ms=0; max_end_sil=ms/FR;
|
||||
end_lookback=max_end_sil-lookahead_end-1; if(end_lookback<0)end_lookback=0;
|
||||
};
|
||||
recompute();
|
||||
std::vector<int> wbuf(win,0); int wpos=0,wsum=0,pre=0;
|
||||
int st=0, cstart=-1, csil=0, prev_end=0;
|
||||
auto reset=[&](){ std::fill(wbuf.begin(),wbuf.end(),0); wpos=0; wsum=0; pre=0; csil=0; st=0; cstart=-1; acc=0; insp=0; };
|
||||
auto emit=[&](int s,int e){ if(s<prev_end)s=prev_end; if(s<0)s=0; if(e>T)e=T; if(e>s){segs.push_back({s,e}); prev_end=e;} };
|
||||
for(int t=0;t<T;t++){
|
||||
if(t>0 && t%CHUNK==0){ if(st==1||insp){acc+=60000; insp=1;} recompute(); }
|
||||
float sil=sc[(size_t)t*od+0];
|
||||
int fs = ((1.0f-sil) >= sil + 0.5f) ? 1 : 0; // speech_noise_thres=0.5
|
||||
wsum -= wbuf[wpos]; wsum += fs; wbuf[wpos]=fs; wpos=(wpos+1)%win;
|
||||
int ch;
|
||||
if(pre==0 && wsum>=s2s){pre=1; ch=3;}
|
||||
else if(pre==1 && wsum<=sp2s){pre=0; ch=1;}
|
||||
else ch = pre==0?0:2;
|
||||
if(ch==3){ csil=0;
|
||||
if(st==0){ cstart=t-start_lookback; if(cstart<prev_end)cstart=prev_end; if(cstart<0)cstart=0; st=1; }
|
||||
else if(st==1 && t-cstart+1>max_seg){ emit(cstart,t); reset(); }
|
||||
} else if(ch==1||ch==2){ csil=0;
|
||||
if(st==1 && t-cstart+1>max_seg){ emit(cstart,t); reset(); }
|
||||
} else { csil++;
|
||||
if(st==1){
|
||||
if(csil>=max_end_sil){ emit(cstart, t-end_lookback); reset(); }
|
||||
else if(t-cstart+1>max_seg){ emit(cstart,t); reset(); }
|
||||
}
|
||||
}
|
||||
}
|
||||
if(st==1) emit(cstart,T);
|
||||
// convert frame indices -> ms
|
||||
for(auto&s:segs){ s.first*=FR; s.second*=FR; }
|
||||
return true;
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
set(TARGET llama-funasr-vad)
|
||||
add_executable(${TARGET} funasr-vad.cpp)
|
||||
install(TARGETS ${TARGET} RUNTIME)
|
||||
target_include_directories(${TARGET} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../funasr-common)
|
||||
target_link_libraries(${TARGET} PRIVATE ggml ${CMAKE_THREAD_LIBS_INIT})
|
||||
target_compile_features(${TARGET} PRIVATE cxx_std_17)
|
||||
@@ -0,0 +1,25 @@
|
||||
// funasr-vad: FSMN-VAD on ggml. WAV(any fmt/rate) -> speech segments [start_ms,end_ms].
|
||||
// Front end + FSMN encoder validated bit-exact vs PyTorch; state machine reproduces
|
||||
// E2EVadModel segmentation to within 1 frame (10ms) of fsmn-vad.generate on the 184-clip set.
|
||||
#define FUNASR_AUDIO_IMPLEMENTATION
|
||||
#include "funasr_audio.h"
|
||||
#include "funasr_vad.h"
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
int main(int argc,char**argv){
|
||||
std::string gp,wp;
|
||||
for(int i=1;i<argc;i++){if(!strcmp(argv[i],"-m")&&i+1<argc)gp=argv[++i];else if(!strcmp(argv[i],"-a")&&i+1<argc)wp=argv[++i];}
|
||||
if(gp.empty()||wp.empty()){fprintf(stderr,"usage: %s -m fsmn-vad.gguf -a audio.wav\n",argv[0]);return 1;}
|
||||
std::vector<float> wav; if(!funasr_load_audio_16k_mono(wp.c_str(),wav)){fprintf(stderr,"read audio failed\n");return 1;}
|
||||
int max_seg = getenv("VAD_MAXSEG") ? atoi(getenv("VAD_MAXSEG")) : 30000;
|
||||
std::vector<std::pair<int,int>> segs;
|
||||
if(!funasr_vad_segments(gp,wav,max_seg,segs)){fprintf(stderr,"vad failed\n");return 1;}
|
||||
for(auto&s:segs) printf("%d %d\n", s.first, s.second);
|
||||
fprintf(stderr,"[vad] %zu segments (max_seg=%dms)\n",segs.size(),max_seg);
|
||||
return 0;
|
||||
}
|
||||
@@ -22,6 +22,8 @@ static const float LN_EPS = 1e-5f;
|
||||
// ===== audio loader: any wav/mp3/flac, any rate/channels -> 16k mono (miniaudio) =====
|
||||
#define FUNASR_AUDIO_IMPLEMENTATION
|
||||
#include "funasr_audio.h"
|
||||
#include "funasr_vad.h" // built-in FSMN-VAD front end (--vad segmentation)
|
||||
#include <utility>
|
||||
static const int FS=16000,WINLEN=400,SHIFT=160,NFFT=512,NMEL=80,LFR_M=7,LFR_N=6;
|
||||
static const float PREEMPH=0.97f,LOWF=20.0f,HIGHF=8000.0f;
|
||||
static inline float melf(float f){return 1127.0f*logf(1.0f+f/700.0f);}
|
||||
@@ -118,9 +120,11 @@ static std::vector<float> run_graph(ggml_context*c,ggml_tensor*out,ggml_tensor*i
|
||||
ggml_gallocr_free(ga);ggml_backend_free(be);return r;}
|
||||
|
||||
int main(int argc,char**argv){
|
||||
std::string gguf_path,wav_path;
|
||||
std::string gguf_path,wav_path,vad_path; int vad_maxseg=30000;
|
||||
for(int i=1;i<argc;i++){if(!strcmp(argv[i],"-m")&&i+1<argc)gguf_path=argv[++i];else if(!strcmp(argv[i],"-a")&&i+1<argc)wav_path=argv[++i];
|
||||
else{fprintf(stderr,"usage: %s -m paraformer.gguf -a audio.wav\n",argv[0]);return 1;}}
|
||||
else if(!strcmp(argv[i],"--vad")&&i+1<argc)vad_path=argv[++i];
|
||||
else if(!strcmp(argv[i],"--vad-maxseg")&&i+1<argc)vad_maxseg=atoi(argv[++i]);
|
||||
else{fprintf(stderr,"usage: %s -m paraformer.gguf -a audio.wav [--vad fsmn-vad.gguf [--vad-maxseg ms]]\n",argv[0]);return 1;}}
|
||||
if(gguf_path.empty()||wav_path.empty()){fprintf(stderr,"missing args\n");return 1;}
|
||||
model m;gguf_init_params gp={false,&m.ctx_w};gguf_context*gg=gguf_init_from_file(gguf_path.c_str(),gp);if(!gg){fprintf(stderr,"gguf load failed\n");return 1;}
|
||||
auto rdi=[&](const char*k,int d){int i=gguf_find_key(gg,k);return i<0?d:(int)gguf_get_val_u32(gg,i);};
|
||||
@@ -130,80 +134,70 @@ int main(int argc,char**argv){
|
||||
for(int i=0;i<gguf_get_n_tensors(gg);i++){const char*nm=gguf_get_tensor_name(gg,i);m.t[nm]=ggml_get_tensor(m.ctx_w,nm);}gguf_free(gg);
|
||||
const int D=m.c.d_model,F=560,V=m.c.vocab;
|
||||
|
||||
// fbank
|
||||
std::vector<float>wav;if(!funasr_load_audio_16k_mono(wav_path.c_str(),wav)){fprintf(stderr,"read audio failed\n");return 1;}
|
||||
int64_t t0=ggml_time_us();int T=0;auto fb=compute_fbank(wav,T);
|
||||
// CMVN (Paraformer applies it): (x+shift)*scale
|
||||
float*shift=(float*)m.g("cmvn.shift")->data,*scale=(float*)m.g("cmvn.scale")->data;
|
||||
for(int t=0;t<T;t++)for(int d=0;d<F;d++)fb[(size_t)t*F+d]=(fb[(size_t)t*F+d]+shift[d])*scale[d];
|
||||
// encoder pre-scale + posenc
|
||||
{float sc=sqrtf((float)D);for(auto&v:fb)v*=sc;}add_posenc(fb,T,F);
|
||||
|
||||
// ===== encoder graph =====
|
||||
std::vector<float>enc;
|
||||
{ggml_init_params cp={(size_t)1024*1024*1024,nullptr,true};ggml_context*c=ggml_init(cp);
|
||||
ggml_tensor*x=ggml_new_tensor_2d(c,GGML_TYPE_F32,F,T);ggml_set_input(x);
|
||||
ggml_tensor*h=enc_layer(c,m,"encoder.encoders0.0.",x,T,false);
|
||||
for(int i=0;i<m.c.enc_blocks-1;i++)h=enc_layer(c,m,"encoder.encoders."+std::to_string(i)+".",h,T,true);
|
||||
h=lnorm(c,h,m.g("encoder.after_norm.weight"),m.g("encoder.after_norm.bias"));ggml_set_output(h);
|
||||
enc=run_graph(c,h,x,fb.data(),nullptr,nullptr);ggml_free(c);} // enc: [D, T] -> stored row-major as T rows? ggml out ne0=D,ne1=T
|
||||
// run_graph returns row-major [N=ne1 rows, D=ne0]; here rows=T, cols=D
|
||||
int64_t t1=ggml_time_us();
|
||||
// enc[t*D + d]
|
||||
|
||||
// ===== CIF predictor (host) =====
|
||||
float*cw=(float*)m.g("predictor.cif_conv1d.weight")->data; // [512,512,3] = [o][i][j]
|
||||
float*cb=(float*)m.g("predictor.cif_conv1d.bias")->data; // [512]
|
||||
float*ow=(float*)m.g("predictor.cif_output.weight")->data; // [1,512]
|
||||
float ob=((float*)m.g("predictor.cif_output.bias")->data)[0];
|
||||
// conv1d (k=3,pad1) + residual + relu, then cif_output -> sigmoid -> alpha
|
||||
std::vector<float> alphas(T);
|
||||
std::vector<float> outp((size_t)T*D);
|
||||
for(int t=0;t<T;t++){
|
||||
for(int o=0;o<D;o++){
|
||||
float acc=cb[o];
|
||||
for(int j=0;j<3;j++){int tt=t+j-1; if(tt<0||tt>=T)continue; const float*ev=&enc[(size_t)tt*D];
|
||||
const float*wo=&cw[(size_t)o*D*3]; for(int i=0;i<D;i++) acc+=wo[i*3+j]*ev[i];}
|
||||
outp[(size_t)t*D+o]=acc+enc[(size_t)t*D+o]; // + context residual
|
||||
}
|
||||
float a=ob; for(int o=0;o<D;o++){float r=outp[(size_t)t*D+o]; if(r<0)r=0; a+=ow[o]*r;} // relu then linear
|
||||
float s=1.0f/(1.0f+expf(-a)); alphas[t]=s>0?s:0; // sigmoid, smooth=1 noise=0 -> relu(s)=s
|
||||
}
|
||||
// tail: append zero hidden frame + alpha=tail_threshold
|
||||
std::vector<float> hid=enc; hid.resize((size_t)(T+1)*D,0.0f);
|
||||
std::vector<float> al=alphas; al.push_back(m.c.tail);
|
||||
int L=T+1;
|
||||
// integrate-and-fire
|
||||
std::vector<float> acoustic; acoustic.reserve(64*D);
|
||||
float integrate=0; std::vector<float> frame(D,0.0f);
|
||||
for(int t=0;t<L;t++){
|
||||
float alpha=al[t]; float dc=1.0f-integrate; integrate+=alpha;
|
||||
bool fire=integrate>=m.c.thresh; float cur=fire?dc:alpha; float rem=alpha-cur;
|
||||
for(int d=0;d<D;d++) frame[d]+=cur*hid[(size_t)t*D+d];
|
||||
if(fire){ acoustic.insert(acoustic.end(),frame.begin(),frame.end()); integrate-=1.0f;
|
||||
for(int d=0;d<D;d++) frame[d]=rem*hid[(size_t)t*D+d]; }
|
||||
}
|
||||
int N=acoustic.size()/D;
|
||||
if(N<1){printf("\n");fprintf(stderr,"[paraformer] no tokens\n");return 0;}
|
||||
|
||||
// ===== decoder graph =====
|
||||
std::vector<float> logits;
|
||||
{ggml_init_params cp={(size_t)2048*1024*1024,nullptr,true};ggml_context*c=ggml_init(cp);
|
||||
ggml_tensor*tgt=ggml_new_tensor_2d(c,GGML_TYPE_F32,D,N);ggml_set_input(tgt);
|
||||
ggml_tensor*mem=ggml_new_tensor_2d(c,GGML_TYPE_F32,D,T);ggml_set_input(mem);
|
||||
ggml_tensor*x=tgt;
|
||||
for(int i=0;i<m.c.dec_att;i++)x=dec_layer(c,m,"decoder.decoders."+std::to_string(i)+".",x,mem,N,T);
|
||||
for(int i=0;i<m.c.dec3;i++)x=dec3_layer(c,m,"decoder.decoders3."+std::to_string(i)+".",x);
|
||||
x=lnorm(c,x,m.g("decoder.after_norm.weight"),m.g("decoder.after_norm.bias"));
|
||||
x=lin(c,m.g("decoder.output_layer.weight"),m.g("decoder.output_layer.bias"),x); // [V,N]
|
||||
ggml_set_output(x);
|
||||
logits=run_graph(c,x,tgt,acoustic.data(),mem,enc.data());ggml_free(c);}
|
||||
int64_t t2=ggml_time_us();
|
||||
// Full pipeline (fbank -> CMVN -> encoder -> CIF -> decoder) on one wav window; prints token IDs.
|
||||
auto run_seg=[&](const std::vector<float>& wav){
|
||||
int T=0;auto fb=compute_fbank(wav,T); if(T<1)return;
|
||||
for(int t=0;t<T;t++)for(int d=0;d<F;d++)fb[(size_t)t*F+d]=(fb[(size_t)t*F+d]+shift[d])*scale[d]; // CMVN
|
||||
{float sc=sqrtf((float)D);for(auto&v:fb)v*=sc;}add_posenc(fb,T,F);
|
||||
std::vector<float>enc;
|
||||
{ggml_init_params cp={(size_t)1024*1024*1024,nullptr,true};ggml_context*c=ggml_init(cp);
|
||||
ggml_tensor*x=ggml_new_tensor_2d(c,GGML_TYPE_F32,F,T);ggml_set_input(x);
|
||||
ggml_tensor*h=enc_layer(c,m,"encoder.encoders0.0.",x,T,false);
|
||||
for(int i=0;i<m.c.enc_blocks-1;i++)h=enc_layer(c,m,"encoder.encoders."+std::to_string(i)+".",h,T,true);
|
||||
h=lnorm(c,h,m.g("encoder.after_norm.weight"),m.g("encoder.after_norm.bias"));ggml_set_output(h);
|
||||
enc=run_graph(c,h,x,fb.data(),nullptr,nullptr);ggml_free(c);} // enc row-major [T,D]
|
||||
// ===== CIF predictor (host): conv1d(k3,pad1)+residual+relu -> cif_output -> sigmoid -> alpha =====
|
||||
std::vector<float> outp((size_t)T*D); std::vector<float> alphas(T);
|
||||
for(int t=0;t<T;t++){
|
||||
for(int o=0;o<D;o++){ float acc=cb[o];
|
||||
for(int j=0;j<3;j++){int tt=t+j-1; if(tt<0||tt>=T)continue; const float*ev=&enc[(size_t)tt*D];
|
||||
const float*wo=&cw[(size_t)o*D*3]; for(int i=0;i<D;i++) acc+=wo[i*3+j]*ev[i];}
|
||||
outp[(size_t)t*D+o]=acc+enc[(size_t)t*D+o]; }
|
||||
float a=ob; for(int o=0;o<D;o++){float r=outp[(size_t)t*D+o]; if(r<0)r=0; a+=ow[o]*r;}
|
||||
float s=1.0f/(1.0f+expf(-a)); alphas[t]=s>0?s:0; }
|
||||
std::vector<float> hid=enc; hid.resize((size_t)(T+1)*D,0.0f);
|
||||
std::vector<float> al=alphas; al.push_back(m.c.tail); int L=T+1;
|
||||
std::vector<float> acoustic; acoustic.reserve(64*D);
|
||||
float integrate=0; std::vector<float> frame(D,0.0f);
|
||||
for(int t=0;t<L;t++){
|
||||
float alpha=al[t]; float dc=1.0f-integrate; integrate+=alpha;
|
||||
bool fire=integrate>=m.c.thresh; float cur=fire?dc:alpha; float rem=alpha-cur;
|
||||
for(int d=0;d<D;d++) frame[d]+=cur*hid[(size_t)t*D+d];
|
||||
if(fire){ acoustic.insert(acoustic.end(),frame.begin(),frame.end()); integrate-=1.0f;
|
||||
for(int d=0;d<D;d++) frame[d]=rem*hid[(size_t)t*D+d]; } }
|
||||
int N=acoustic.size()/D; if(N<1)return;
|
||||
std::vector<float> logits;
|
||||
{ggml_init_params cp={(size_t)2048*1024*1024,nullptr,true};ggml_context*c=ggml_init(cp);
|
||||
ggml_tensor*tgt=ggml_new_tensor_2d(c,GGML_TYPE_F32,D,N);ggml_set_input(tgt);
|
||||
ggml_tensor*mem=ggml_new_tensor_2d(c,GGML_TYPE_F32,D,T);ggml_set_input(mem);
|
||||
ggml_tensor*x=tgt;
|
||||
for(int i=0;i<m.c.dec_att;i++)x=dec_layer(c,m,"decoder.decoders."+std::to_string(i)+".",x,mem,N,T);
|
||||
for(int i=0;i<m.c.dec3;i++)x=dec3_layer(c,m,"decoder.decoders3."+std::to_string(i)+".",x);
|
||||
x=lnorm(c,x,m.g("decoder.after_norm.weight"),m.g("decoder.after_norm.bias"));
|
||||
x=lin(c,m.g("decoder.output_layer.weight"),m.g("decoder.output_layer.bias"),x); // [V,N]
|
||||
ggml_set_output(x);
|
||||
logits=run_graph(c,x,tgt,acoustic.data(),mem,enc.data());ggml_free(c);}
|
||||
for(int n=0;n<N;n++){const float*col=&logits[(size_t)n*V];int am=0;float best=col[0];for(int v=1;v<V;v++)if(col[v]>best){best=col[v];am=v;}printf("%d ",am);}
|
||||
};
|
||||
|
||||
// argmax per slot -> ids
|
||||
for(int n=0;n<N;n++){const float*col=&logits[(size_t)n*V];int am=0;float best=col[0];for(int v=1;v<V;v++)if(col[v]>best){best=col[v];am=v;}printf("%d ",am);}
|
||||
int64_t t0=ggml_time_us();
|
||||
std::vector<float>wav;if(!funasr_load_audio_16k_mono(wav_path.c_str(),wav)){fprintf(stderr,"read audio failed\n");return 1;}
|
||||
if(!vad_path.empty()){
|
||||
std::vector<std::pair<int,int>> segs;
|
||||
if(!funasr_vad_segments(vad_path,wav,vad_maxseg,segs)){fprintf(stderr,"vad failed\n");return 1;}
|
||||
for(auto&s:segs){ int off=(int)((int64_t)s.first*16000/1000), end=(int)((int64_t)s.second*16000/1000);
|
||||
if(end>(int)wav.size())end=wav.size(); if(end-off<WINLEN)continue;
|
||||
std::vector<float> seg(wav.begin()+off,wav.begin()+end); run_seg(seg); }
|
||||
fprintf(stderr,"[paraformer] %zu vad segments\n",segs.size());
|
||||
} else { run_seg(wav); }
|
||||
printf("\n");
|
||||
fprintf(stderr,"[paraformer] T=%d N_tok=%d enc %.2fs dec %.2fs\n",T,N,(t1-t0)/1e6,(t2-t1)/1e6);
|
||||
fprintf(stderr,"[paraformer] done %.2fs\n",(ggml_time_us()-t0)/1e6);
|
||||
if(m.ctx_w) ggml_free(m.ctx_w);
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -23,6 +23,8 @@ static const float LN_EPS = 1e-5f;
|
||||
// ---- audio loader: any wav/mp3/flac, any rate/channels -> 16k mono (miniaudio) ----
|
||||
#define FUNASR_AUDIO_IMPLEMENTATION
|
||||
#include "funasr_audio.h"
|
||||
#include "funasr_vad.h" // built-in FSMN-VAD front end (--vad segmentation)
|
||||
#include <utility>
|
||||
static const int FS=16000,WINLEN=400,SHIFT=160,NFFT=512,NMEL=80,LFR_M=7,LFR_N=6;
|
||||
static const float PREEMPH=0.97f,LOWF=20.0f,HIGHF=8000.0f;
|
||||
static inline float melf(float f){return 1127.0f*logf(1.0f+f/700.0f);}
|
||||
@@ -90,11 +92,13 @@ static void add_posenc(std::vector<float>&x,int T,int depth){
|
||||
}
|
||||
|
||||
int main(int argc,char**argv){
|
||||
std::string gguf_path,fbank_path,wav_path;
|
||||
std::string gguf_path,fbank_path,wav_path,vad_path; int vad_maxseg=30000;
|
||||
for(int i=1;i<argc;i++){ if(!strcmp(argv[i],"-m")&&i+1<argc)gguf_path=argv[++i];
|
||||
else if(!strcmp(argv[i],"-f")&&i+1<argc)fbank_path=argv[++i];
|
||||
else if(!strcmp(argv[i],"-a")&&i+1<argc)wav_path=argv[++i];
|
||||
else {fprintf(stderr,"usage: %s -m sensevoice.gguf (-a audio.wav | -f fbank.bin)\n",argv[0]);return 1;} }
|
||||
else if(!strcmp(argv[i],"--vad")&&i+1<argc)vad_path=argv[++i];
|
||||
else if(!strcmp(argv[i],"--vad-maxseg")&&i+1<argc)vad_maxseg=atoi(argv[++i]);
|
||||
else {fprintf(stderr,"usage: %s -m sensevoice.gguf (-a audio.wav | -f fbank.bin) [--vad fsmn-vad.gguf [--vad-maxseg ms]]\n",argv[0]);return 1;} }
|
||||
if(gguf_path.empty()||(fbank_path.empty()&&wav_path.empty())){fprintf(stderr,"missing args\n");return 1;}
|
||||
|
||||
// load model
|
||||
@@ -110,61 +114,61 @@ int main(int argc,char**argv){
|
||||
gguf_free(gg);
|
||||
const int F=560, D=m.c.d_model, V=m.c.vocab;
|
||||
|
||||
// fbank [T,F]: from wav (-a) or precomputed (-f)
|
||||
int32_t T=0,Fc=F; std::vector<float> fb;
|
||||
if(!wav_path.empty()){
|
||||
std::vector<float> wav; if(!funasr_load_audio_16k_mono(wav_path.c_str(),wav)){fprintf(stderr,"read audio failed\n");return 1;}
|
||||
int t=0; fb=compute_fbank(wav,t); T=t;
|
||||
} else {
|
||||
FILE*f=fopen(fbank_path.c_str(),"rb"); if(!f){fprintf(stderr,"open fbank\n");return 1;}
|
||||
if(fread(&T,4,1,f)!=1||fread(&Fc,4,1,f)!=1){fclose(f);return 1;}
|
||||
fb.resize((size_t)T*Fc); if((int)fread(fb.data(),4,fb.size(),f)!=(int)fb.size()){fclose(f);return 1;} fclose(f);
|
||||
}
|
||||
|
||||
// NOTE: SenseVoiceSmall inference() feeds the RAW log-mel fbank to the encoder;
|
||||
// it does NOT apply am.mvn CMVN (that path is unused at inference). Applying it
|
||||
// makes the encoder predict <|nospeech|>. So no CMVN here.
|
||||
|
||||
// prepend nq query tokens (rows of embed.weight) -> input [nq+T, F]
|
||||
float*emb=(float*)m.g("embed.weight")->data; // [16, 560] row-major
|
||||
int N=nq+T; std::vector<float> inp((size_t)N*F);
|
||||
for(int i=0;i<nq;i++) memcpy(&inp[(size_t)i*F], &emb[(size_t)qtok[i]*F], F*sizeof(float));
|
||||
memcpy(&inp[(size_t)nq*F], fb.data(), (size_t)T*F*sizeof(float));
|
||||
// Run encoder+CTC on one fbank window [T,F]; prints greedy-CTC token IDs (no newline).
|
||||
auto run_seg=[&](const std::vector<float>& fb,int T){
|
||||
int N=nq+T; std::vector<float> inp((size_t)N*F);
|
||||
for(int i=0;i<nq;i++) memcpy(&inp[(size_t)i*F], &emb[(size_t)qtok[i]*F], F*sizeof(float));
|
||||
memcpy(&inp[(size_t)nq*F], fb.data(), (size_t)T*F*sizeof(float));
|
||||
float sc=sqrtf((float)D); for(auto&v:inp)v*=sc; add_posenc(inp,N,F);
|
||||
ggml_backend_t be=ggml_backend_cpu_init();
|
||||
ggml_init_params cp={(size_t)1024*1024*1024,nullptr,true}; ggml_context*c=ggml_init(cp);
|
||||
ggml_tensor*x=ggml_new_tensor_2d(c,GGML_TYPE_F32,F,N); ggml_set_input(x);
|
||||
ggml_tensor*h=sanm_layer(c,m,"encoder.encoders0.0.",x,N,false);
|
||||
for(int i=0;i<m.c.num_blocks-1;i++) h=sanm_layer(c,m,"encoder.encoders."+std::to_string(i)+".",h,N,true);
|
||||
h=lnorm(c,h,m.g("encoder.after_norm.weight"),m.g("encoder.after_norm.bias"));
|
||||
for(int i=0;i<m.c.tp_blocks;i++) h=sanm_layer(c,m,"encoder.tp_encoders."+std::to_string(i)+".",h,N,true);
|
||||
h=lnorm(c,h,m.g("encoder.tp_norm.weight"),m.g("encoder.tp_norm.bias"));
|
||||
ggml_tensor*logits=lin(c,m.g("ctc.ctc_lo.weight"),m.g("ctc.ctc_lo.bias"),h); // [V, N]
|
||||
ggml_set_output(logits);
|
||||
ggml_cgraph*gf=ggml_new_graph_custom(c,32768,false); ggml_build_forward_expand(gf,logits);
|
||||
ggml_gallocr_t ga=ggml_gallocr_new(ggml_backend_cpu_buffer_type()); ggml_gallocr_alloc_graph(ga,gf);
|
||||
ggml_backend_tensor_set(x,inp.data(),0,ggml_nbytes(x)); ggml_backend_cpu_set_n_threads(be,8);
|
||||
if(ggml_backend_graph_compute(be,gf)!=GGML_STATUS_SUCCESS){fprintf(stderr,"compute failed\n");}
|
||||
std::vector<float> lg((size_t)V*N); ggml_backend_tensor_get(logits,lg.data(),0,ggml_nbytes(logits));
|
||||
int prev=-1; // greedy CTC: argmax per frame -> collapse consecutive -> drop blank
|
||||
for(int n=0;n<N;n++){ const float*col=&lg[(size_t)n*V]; int am=0; float best=col[0];
|
||||
for(int v=1;v<V;v++) if(col[v]>best){best=col[v];am=v;}
|
||||
if(am!=prev && am!=m.c.blank) printf("%d ", am); prev=am; }
|
||||
ggml_gallocr_free(ga); ggml_free(c); ggml_backend_free(be);
|
||||
};
|
||||
|
||||
// encoder pre-scale + posenc
|
||||
float sc=sqrtf((float)D); for(auto&v:inp)v*=sc; add_posenc(inp,N,F);
|
||||
|
||||
// build graph
|
||||
ggml_backend_t be=ggml_backend_cpu_init();
|
||||
ggml_init_params cp={(size_t)1024*1024*1024,nullptr,true}; ggml_context*c=ggml_init(cp);
|
||||
ggml_tensor*x=ggml_new_tensor_2d(c,GGML_TYPE_F32,F,N); ggml_set_input(x);
|
||||
ggml_tensor*h=sanm_layer(c,m,"encoder.encoders0.0.",x,N,false);
|
||||
for(int i=0;i<m.c.num_blocks-1;i++) h=sanm_layer(c,m,"encoder.encoders."+std::to_string(i)+".",h,N,true);
|
||||
h=lnorm(c,h,m.g("encoder.after_norm.weight"),m.g("encoder.after_norm.bias"));
|
||||
for(int i=0;i<m.c.tp_blocks;i++) h=sanm_layer(c,m,"encoder.tp_encoders."+std::to_string(i)+".",h,N,true);
|
||||
h=lnorm(c,h,m.g("encoder.tp_norm.weight"),m.g("encoder.tp_norm.bias"));
|
||||
ggml_tensor*logits=lin(c,m.g("ctc.ctc_lo.weight"),m.g("ctc.ctc_lo.bias"),h); // [V, N]
|
||||
ggml_set_output(logits);
|
||||
ggml_cgraph*gf=ggml_new_graph_custom(c,32768,false); ggml_build_forward_expand(gf,logits);
|
||||
ggml_gallocr_t ga=ggml_gallocr_new(ggml_backend_cpu_buffer_type()); ggml_gallocr_alloc_graph(ga,gf);
|
||||
ggml_backend_tensor_set(x,inp.data(),0,ggml_nbytes(x));
|
||||
ggml_backend_cpu_set_n_threads(be,8);
|
||||
int64_t t0=ggml_time_us();
|
||||
if(ggml_backend_graph_compute(be,gf)!=GGML_STATUS_SUCCESS){fprintf(stderr,"compute failed\n");return 1;}
|
||||
int64_t t1=ggml_time_us();
|
||||
|
||||
std::vector<float> lg((size_t)V*N); ggml_backend_tensor_get(logits,lg.data(),0,ggml_nbytes(logits));
|
||||
// greedy CTC: argmax per frame -> collapse consecutive -> drop blank
|
||||
int prev=-1;
|
||||
for(int n=0;n<N;n++){
|
||||
const float*col=&lg[(size_t)n*V]; int am=0; float best=col[0];
|
||||
for(int v=1;v<V;v++) if(col[v]>best){best=col[v];am=v;}
|
||||
if(am!=prev && am!=m.c.blank) printf("%d ", am);
|
||||
prev=am;
|
||||
if(!vad_path.empty()){
|
||||
std::vector<float> wav; if(!funasr_load_audio_16k_mono(wav_path.c_str(),wav)){fprintf(stderr,"read audio failed\n");return 1;}
|
||||
std::vector<std::pair<int,int>> segs;
|
||||
if(!funasr_vad_segments(vad_path,wav,vad_maxseg,segs)){fprintf(stderr,"vad failed\n");return 1;}
|
||||
for(auto&s:segs){ int off=(int)((int64_t)s.first*16000/1000), end=(int)((int64_t)s.second*16000/1000);
|
||||
if(end>(int)wav.size())end=wav.size(); if(end-off<WINLEN)continue;
|
||||
std::vector<float> seg(wav.begin()+off,wav.begin()+end); int t=0; auto fb=compute_fbank(seg,t); run_seg(fb,t); }
|
||||
fprintf(stderr,"[sensevoice] %zu vad segments\n",segs.size());
|
||||
} else {
|
||||
int32_t T=0,Fc=F; std::vector<float> fb;
|
||||
if(!wav_path.empty()){
|
||||
std::vector<float> wav; if(!funasr_load_audio_16k_mono(wav_path.c_str(),wav)){fprintf(stderr,"read audio failed\n");return 1;}
|
||||
int t=0; fb=compute_fbank(wav,t); T=t;
|
||||
} else {
|
||||
FILE*f=fopen(fbank_path.c_str(),"rb"); if(!f){fprintf(stderr,"open fbank\n");return 1;}
|
||||
if(fread(&T,4,1,f)!=1||fread(&Fc,4,1,f)!=1){fclose(f);return 1;}
|
||||
fb.resize((size_t)T*Fc); if((int)fread(fb.data(),4,fb.size(),f)!=(int)fb.size()){fclose(f);return 1;} fclose(f);
|
||||
}
|
||||
run_seg(fb,T);
|
||||
}
|
||||
printf("\n");
|
||||
fprintf(stderr,"[sensevoice] N=%d (q%d+T%d) encode %.2fs\n",N,nq,T,(t1-t0)/1e6);
|
||||
ggml_gallocr_free(ga); ggml_free(c); ggml_backend_free(be);
|
||||
fprintf(stderr,"[sensevoice] done %.2fs\n",(ggml_time_us()-t0)/1e6);
|
||||
if(m.ctx_w) ggml_free(m.ctx_w);
|
||||
return 0;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user