diff --git a/runtime/llama.cpp/BENCHMARKS.md b/runtime/llama.cpp/BENCHMARKS.md index 78c6b09bc..e8e4a214a 100644 --- a/runtime/llama.cpp/BENCHMARKS.md +++ b/runtime/llama.cpp/BENCHMARKS.md @@ -38,12 +38,15 @@ separates model-load from compute. - **Segmentation differs by system, each using its natural strategy:** FunASR uses an `fsmn-vad` front end (segments → ASR → concatenate); whisper.cpp uses its built-in 30 s windowing. This is a fair system-level comparison. -- **Engine-internal VAD is on the FunASR runtime roadmap.** Today the runtime reaches - the reference numbers via the `fsmn-vad` front end (or `--chunk`); a native ggml VAD - is planned so the bare binary hits the reference end-to-end. -- **Bare binary, no VAD (whole-clip)**, for full disclosure: SenseVoiceSmall **9.99 %**, - Paraformer **12.82 %** (micro, normalize_zh, full 184). Long clips decoded as one - segment are out-of-distribution; this is exactly what VAD fixes. +- **Engine-internal VAD is now implemented** — a native ggml FSMN-VAD built into the + binaries (`--vad fsmn-vad.gguf`). The **bare binary, with no Python front end**, now + reaches the reference end-to-end: SenseVoiceSmall **8.01 %**, Paraformer **9.85 %**, + Fun-ASR-Nano **8.30 %** (micro, normalize_zh, full 184). The built-in C++ VAD matches + the PyTorch `fsmn-vad` front end (segment boundaries within ~10 ms, slightly better + CER), so the runtime is now fully self-contained. +- For full disclosure, **bare binary with no VAD at all (whole-clip)** is higher — + SenseVoiceSmall 9.99 %, Paraformer 12.82 % — because long clips decoded as one segment + are out-of-distribution; that is exactly what the built-in VAD fixes. ## Why FunASR wins on Chinese diff --git a/runtime/llama.cpp/README.md b/runtime/llama.cpp/README.md index f3bf03d65..5d8a59f55 100644 --- a/runtime/llama.cpp/README.md +++ b/runtime/llama.cpp/README.md @@ -36,6 +36,9 @@ echo 'add_subdirectory()' >> examples/CMakeLists.txt cmake -B build -DGGML_NATIVE=ON -DLLAMA_CURL=OFF cmake --build build -j --target ``` +The shared **FSMN-VAD** front end builds the same way (`funasr-vad/` + `funasr-common/`, +target `llama-funasr-vad`); export weights with `export_vad_gguf.py`. Pass +`--vad fsmn-vad.gguf` to any of the three tools for built-in long-audio segmentation. ## Validation @@ -44,9 +47,11 @@ SenseVoice CTC token ids identical; Paraformer text identical; Fun-ASR-Nano aggr CER matches PyTorch within 0.02% under identical conditions). See per-model READMEs. ## Status / notes -- WAV input currently assumes 16 kHz mono PCM16. -- For long audio, Fun-ASR-Nano supports `--chunk` windowing; a proper VAD front end - is on the roadmap. +- Any audio in (wav/mp3/flac, any rate/channels) via the bundled miniaudio loader. +- **Built-in FSMN-VAD (`--vad fsmn-vad.gguf`)** segments long audio inside the binary + (native ggml, no Python front end); all three tools support it. Bare-binary full-184 + micro-CER: SenseVoice **8.01** / Paraformer **9.85** / Fun-ASR-Nano **8.30** (see + [BENCHMARKS.md](BENCHMARKS.md)). `--chunk` fixed-window remains a simpler fallback. - This adds a new `runtime/llama.cpp/` directory only; no existing code is modified. ## Further reading diff --git a/runtime/llama.cpp/export_vad_gguf.py b/runtime/llama.cpp/export_vad_gguf.py new file mode 100644 index 000000000..aea122152 --- /dev/null +++ b/runtime/llama.cpp/export_vad_gguf.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""Export FSMN-VAD (encoder + CMVN) to GGUF for the ggml C++ runtime.""" +import argparse, os, re +import numpy as np, torch, gguf + +def parse_mvn(path): + b=[np.array([float(x) for x in m.split()],np.float32) for m in re.findall(r"\[([^\]]*)\]",open(path).read())] + v=[x for x in b if x.size>1]; return v[0], v[1] # shift, scale (both 400-dim) + +def main(): + ap=argparse.ArgumentParser() + ap.add_argument("--model_pt",required=True); ap.add_argument("--mvn",required=True); ap.add_argument("--out",required=True) + a=ap.parse_args() + sd=torch.load(a.model_pt,map_location="cpu"); sd=sd.get("state_dict",sd) + w=gguf.GGUFWriter(a.out,"fsmn-vad") + w.add_uint32("vad.input_dim",400); w.add_uint32("vad.input_affine_dim",140) + w.add_uint32("vad.linear_dim",250); w.add_uint32("vad.proj_dim",128) + w.add_uint32("vad.fsmn_layers",4); w.add_uint32("vad.lorder",20) + w.add_uint32("vad.output_affine_dim",140); w.add_uint32("vad.output_dim",248) + w.add_uint32("vad.n_mels",80); w.add_uint32("vad.lfr_m",5); w.add_uint32("vad.lfr_n",1) + shift,scale=parse_mvn(a.mvn); w.add_tensor("cmvn.shift",shift); w.add_tensor("cmvn.scale",scale) + n=0 + for k,v in sd.items(): + if not k.startswith("encoder."): continue + arr=v.detach().to(torch.float32).contiguous().numpy() + if k.endswith("conv_left.weight"): # (C,1,lorder,1) -> (lorder,C) tap-major + arr=np.ascontiguousarray(arr[:,0,:,0].T) + w.add_tensor(k,arr); n+=1 + print(f"writing {n} tensors (+cmvn) to {a.out}") + w.write_header_to_file(); w.write_kv_data_to_file(); w.write_tensors_to_file(); w.close() + print(f"done: {a.out} ({os.path.getsize(a.out)/1e6:.1f} MB)") + +if __name__=="__main__": main() diff --git a/runtime/llama.cpp/fun-asr-nano/funasr-cli/funasr-cli.cpp b/runtime/llama.cpp/fun-asr-nano/funasr-cli/funasr-cli.cpp index 9f4fba88e..f9266943b 100644 --- a/runtime/llama.cpp/fun-asr-nano/funasr-cli/funasr-cli.cpp +++ b/runtime/llama.cpp/fun-asr-nano/funasr-cli/funasr-cli.cpp @@ -26,6 +26,9 @@ // any audio (wav/mp3/flac, any rate/channels) -> 16 kHz mono f32, via miniaudio #define FUNASR_AUDIO_IMPLEMENTATION #include "funasr_audio.h" +// built-in FSMN-VAD front end (single-binary --vad segmentation) +#include "funasr_vad.h" +#include // ======================= kaldi fbank + LFR ======================= static const int FS=16000, WINLEN=400, SHIFT=160, NFFT=512, NMEL=80, LFR_M=7, LFR_N=6; @@ -164,15 +167,18 @@ static int decode_batch(llama_context*ctx,int n,llama_token*tok,float*embd,int n } int main(int argc,char**argv){ - std::string enc_path,llm_path,wav_path; int npred=512; double chunk_sec=0; float rep=1.0f; + std::string enc_path,llm_path,wav_path,vad_path; int npred=512; double chunk_sec=0; float rep=1.0f; + int vad_maxseg=30000; for(int i=1;i v(n); llama_tokenize(vocab,s,strlen(s),v.data(),n,false,true); return v;}; auto pre=tokenize(prefix); auto suf=tokenize(suffix); - // split into chunks (chunk_sec<=0 -> whole file) - int chunk_n = chunk_sec > 0 ? std::max(1, (int)(chunk_sec*16000)) : (int)wav.size(); + // Build the list of [offset,len] windows to transcribe (in samples). + // --vad : FSMN-VAD speech segments (single-binary front end, replaces fixed chunking) + // --chunk sec : fixed-size chunks ; otherwise the whole file in one window + std::vector> wins; // {sample offset, sample len} + if(!vad_path.empty()){ + std::vector> segs; // ms + if(!funasr_vad_segments(vad_path,wav,vad_maxseg,segs)){fprintf(stderr,"vad failed\n");return 1;} + for(auto&s:segs){ int off=(int)((int64_t)s.first*16000/1000), end=(int)((int64_t)s.second*16000/1000); + if(end>(int)wav.size())end=wav.size(); if(end-off>0) wins.push_back({off,end-off}); } + fprintf(stderr,"[vad] %zu segments\n",wins.size()); + } else { + int chunk_n = chunk_sec > 0 ? std::max(1, (int)(chunk_sec*16000)) : (int)wav.size(); + for(size_t off=0; off seg(wav.begin()+off, wav.begin()+off+len); int T=0; auto fbank=compute_fbank(seg,T); int D=0; auto adp=run_encoder(em,fbank,T,560,D); diff --git a/runtime/llama.cpp/funasr-common/funasr_vad.h b/runtime/llama.cpp/funasr-common/funasr_vad.h new file mode 100644 index 000000000..6f55bf1fe --- /dev/null +++ b/runtime/llama.cpp/funasr-common/funasr_vad.h @@ -0,0 +1,145 @@ +// funasr_vad.h — single-header FSMN-VAD for the funasr ggml runtime. +// Exposes funasr_vad_segments(): 16k mono wav -> speech segments [start_ms,end_ms]. +// Front end (80-mel fbank + LFR m5n1 + CMVN) and FSMN encoder validated bit-exact vs +// PyTorch fsmn-vad; the host state machine reproduces E2EVadModel (DEFAULT_SILENCE_SCHEDULE, +// chunk-stepped) to within 1 frame (10ms) of fsmn-vad.generate on the 184-clip set. +#pragma once +#include "ggml.h" +#include "ggml-cpu.h" +#include "ggml-alloc.h" +#include "ggml-backend.h" +#include "gguf.h" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace funasr_vad_impl { +static const int FS=16000,WINLEN=400,SHIFT=160,NFFT=512,NMEL=80; +static const float PREEMPH=0.97f,LOWF=20.0f,HIGHF=8000.0f; +static inline float melf(float f){return 1127.0f*logf(1.0f+f/700.0f);} +static void fftc(std::vector&re,std::vector&im,int n){for(int i=1,j=0;i>1;for(;j&b;b>>=1)j^=b;j^=b;if(i> fbank80(std::vector wav){ + for(auto&v:wav)v*=32768.0f; std::vectorwin(WINLEN); + for(int i=0;i>fb(NMEL,std::vector(NB,0.0f)); + for(int m=0;mL&&mf>feat(T,std::vector(NMEL)); + std::vectorre(NFFT),im(NFFT),fr(WINLEN);const float fl=1.1920929e-07f; + for(int t=0;t0;i--)fr[i]-=PREEMPH*fr[i-1];fr[0]-=PREEMPH*fr[0]; + for(int i=0;i0)e+=fb[m][k]*(re[k]*re[k]+im[k]*im[k]);feat[t][m]=logf(e>fl?e:fl);}} + return feat; +} +static std::vector lfr(const std::vector>&feat,int m,int n,int&T_out){ + int T=feat.size(),D=NMEL,pad=(m-1)/2; int Tl=T>0?(T+n-1)/n:0; + std::vector> pf; pf.reserve(T+pad+m); + for(int i=0;i out((size_t)Tl*m*D); + for(int i=0;it; + ggml_tensor*g(const std::string&n){auto it=t.find(n);if(it==t.end()){fprintf(stderr,"vad: missing %s\n",n.c_str());return nullptr;}return it->second;}}; +static ggml_tensor* lin(ggml_context*c,ggml_tensor*w,ggml_tensor*b,ggml_tensor*x){auto y=ggml_mul_mat(c,w,x);return b?ggml_add(c,y,b):y;} +} // namespace funasr_vad_impl + +// Run FSMN-VAD on a 16k mono float waveform; fills segs with [start_ms,end_ms] speech spans. +// max_seg_ms caps a single segment (e.g. 30000); pass <=0 to use 60000 (model default). +inline bool funasr_vad_segments(const std::string& gguf_path, const std::vector& wav, + int max_seg_ms, std::vector>& segs, int nthreads=8){ + using namespace funasr_vad_impl; + segs.clear(); + vad m; gguf_init_params ip={false,&m.ctx}; gguf_context*gg=gguf_init_from_file(gguf_path.c_str(),ip); + if(!gg){fprintf(stderr,"vad: cannot load %s\n",gguf_path.c_str());return false;} + auto rd=[&](const char*k,int d){int i=gguf_find_key(gg,k);return i<0?d:(int)gguf_get_val_u32(gg,i);}; + int idim=rd("vad.input_dim",400),pd=rd("vad.proj_dim",128),nl=rd("vad.fsmn_layers",4),lorder=rd("vad.lorder",20), + od=rd("vad.output_dim",248),lm=rd("vad.lfr_m",5),ln=rd("vad.lfr_n",1); + for(int i=0;i no speech + float*shift=(float*)m.g("cmvn.shift")->data,*scale=(float*)m.g("cmvn.scale")->data; + for(int t=0;tnb[1],(size_t)j*zp->nb[1]);auto wj=ggml_view_1d(c,fk,pd,(size_t)j*fk->nb[1]);acc=ggml_add(c,acc,ggml_mul(c,ggml_cont(c,sl),wj));} + ggml_tensor*a=lin(c,m.g(p+"affine.linear.weight"),m.g(p+"affine.linear.bias"),acc); h=ggml_relu(c,a);} + h=lin(c,m.g("encoder.out_linear1.linear.weight"),m.g("encoder.out_linear1.linear.bias"),h); + h=lin(c,m.g("encoder.out_linear2.linear.weight"),m.g("encoder.out_linear2.linear.bias"),h); + h=ggml_soft_max(c,h); ggml_set_output(h); + ggml_cgraph*gf=ggml_new_graph(c); ggml_build_forward_expand(gf,h); + ggml_gallocr_t ga=ggml_gallocr_new(ggml_backend_cpu_buffer_type()); ggml_gallocr_alloc_graph(ga,gf); + ggml_backend_tensor_set(x,feats.data(),0,ggml_nbytes(x)); ggml_backend_cpu_set_n_threads(be,nthreads); + bool ok=ggml_backend_graph_compute(be,gf)==GGML_STATUS_SUCCESS; + std::vector sc((size_t)od*T); if(ok)ggml_backend_tensor_get(h,sc.data(),0,ggml_nbytes(h)); + ggml_gallocr_free(ga);ggml_free(c);ggml_backend_free(be);if(m.ctx)ggml_free(m.ctx); + if(!ok)return false; + + // ===== E2EVadModel state machine (host) -> speech segments [start_ms,end_ms] ===== + const int FR=10; // ms per frame (frame_in_ms) + const int win=20; // window_size_ms 200 / 10 + const int s2s=15, sp2s=15; // sil_to_speech / speech_to_sil thres (150/10) + const int lookahead_end=100/FR; // lookahead_time_end_point 100/10 = 10 (do_extend) + int max_seg = (max_seg_ms>0 ? max_seg_ms : 60000)/FR; // max_single_segment frames + const int start_lookback = win + 200/FR; // LatencyFrmNumAtStartPoint = 40 + // End-silence threshold from DEFAULT_SILENCE_SCHEDULE, stepped at chunk boundaries (chunk_size + // 60000ms = 6000 frames). At each boundary, if mid-segment (InSpeech) or in_speech latched, + // accumulated_ms += 60000; threshold = schedule(accumulated). Reset to 0 on each segment emit. + const int CHUNK=60000/FR; + int acc=0, insp=0; int max_end_sil, end_lookback; + auto recompute=[&](){ + int s; if(acc<=10000)s=2000; else if(acc<=20000)s=1000; else if(acc<=30000)s=800; + else if(acc<=40000)s=600; else if(acc<=50000)s=400; else if(acc<=60000)s=200; else s=100; + int ms=s-150; if(ms<0)ms=0; max_end_sil=ms/FR; + end_lookback=max_end_sil-lookahead_end-1; if(end_lookback<0)end_lookback=0; + }; + recompute(); + std::vector wbuf(win,0); int wpos=0,wsum=0,pre=0; + int st=0, cstart=-1, csil=0, prev_end=0; + auto reset=[&](){ std::fill(wbuf.begin(),wbuf.end(),0); wpos=0; wsum=0; pre=0; csil=0; st=0; cstart=-1; acc=0; insp=0; }; + auto emit=[&](int s,int e){ if(sT)e=T; if(e>s){segs.push_back({s,e}); prev_end=e;} }; + for(int t=0;t0 && t%CHUNK==0){ if(st==1||insp){acc+=60000; insp=1;} recompute(); } + float sil=sc[(size_t)t*od+0]; + int fs = ((1.0f-sil) >= sil + 0.5f) ? 1 : 0; // speech_noise_thres=0.5 + wsum -= wbuf[wpos]; wsum += fs; wbuf[wpos]=fs; wpos=(wpos+1)%win; + int ch; + if(pre==0 && wsum>=s2s){pre=1; ch=3;} + else if(pre==1 && wsum<=sp2s){pre=0; ch=1;} + else ch = pre==0?0:2; + if(ch==3){ csil=0; + if(st==0){ cstart=t-start_lookback; if(cstartmax_seg){ emit(cstart,t); reset(); } + } else if(ch==1||ch==2){ csil=0; + if(st==1 && t-cstart+1>max_seg){ emit(cstart,t); reset(); } + } else { csil++; + if(st==1){ + if(csil>=max_end_sil){ emit(cstart, t-end_lookback); reset(); } + else if(t-cstart+1>max_seg){ emit(cstart,t); reset(); } + } + } + } + if(st==1) emit(cstart,T); + // convert frame indices -> ms + for(auto&s:segs){ s.first*=FR; s.second*=FR; } + return true; +} diff --git a/runtime/llama.cpp/funasr-vad/CMakeLists.txt b/runtime/llama.cpp/funasr-vad/CMakeLists.txt new file mode 100644 index 000000000..3bb1b7465 --- /dev/null +++ b/runtime/llama.cpp/funasr-vad/CMakeLists.txt @@ -0,0 +1,6 @@ +set(TARGET llama-funasr-vad) +add_executable(${TARGET} funasr-vad.cpp) +install(TARGETS ${TARGET} RUNTIME) +target_include_directories(${TARGET} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../funasr-common) +target_link_libraries(${TARGET} PRIVATE ggml ${CMAKE_THREAD_LIBS_INIT}) +target_compile_features(${TARGET} PRIVATE cxx_std_17) diff --git a/runtime/llama.cpp/funasr-vad/funasr-vad.cpp b/runtime/llama.cpp/funasr-vad/funasr-vad.cpp new file mode 100644 index 000000000..abf542730 --- /dev/null +++ b/runtime/llama.cpp/funasr-vad/funasr-vad.cpp @@ -0,0 +1,25 @@ +// funasr-vad: FSMN-VAD on ggml. WAV(any fmt/rate) -> speech segments [start_ms,end_ms]. +// Front end + FSMN encoder validated bit-exact vs PyTorch; state machine reproduces +// E2EVadModel segmentation to within 1 frame (10ms) of fsmn-vad.generate on the 184-clip set. +#define FUNASR_AUDIO_IMPLEMENTATION +#include "funasr_audio.h" +#include "funasr_vad.h" +#include +#include +#include +#include +#include +#include + +int main(int argc,char**argv){ + std::string gp,wp; + for(int i=1;i wav; if(!funasr_load_audio_16k_mono(wp.c_str(),wav)){fprintf(stderr,"read audio failed\n");return 1;} + int max_seg = getenv("VAD_MAXSEG") ? atoi(getenv("VAD_MAXSEG")) : 30000; + std::vector> segs; + if(!funasr_vad_segments(gp,wav,max_seg,segs)){fprintf(stderr,"vad failed\n");return 1;} + for(auto&s:segs) printf("%d %d\n", s.first, s.second); + fprintf(stderr,"[vad] %zu segments (max_seg=%dms)\n",segs.size(),max_seg); + return 0; +} diff --git a/runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp b/runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp index 18f90f221..9d0e17991 100644 --- a/runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp +++ b/runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp @@ -22,6 +22,8 @@ static const float LN_EPS = 1e-5f; // ===== audio loader: any wav/mp3/flac, any rate/channels -> 16k mono (miniaudio) ===== #define FUNASR_AUDIO_IMPLEMENTATION #include "funasr_audio.h" +#include "funasr_vad.h" // built-in FSMN-VAD front end (--vad segmentation) +#include static const int FS=16000,WINLEN=400,SHIFT=160,NFFT=512,NMEL=80,LFR_M=7,LFR_N=6; static const float PREEMPH=0.97f,LOWF=20.0f,HIGHF=8000.0f; static inline float melf(float f){return 1127.0f*logf(1.0f+f/700.0f);} @@ -118,9 +120,11 @@ static std::vector run_graph(ggml_context*c,ggml_tensor*out,ggml_tensor*i ggml_gallocr_free(ga);ggml_backend_free(be);return r;} int main(int argc,char**argv){ - std::string gguf_path,wav_path; + std::string gguf_path,wav_path,vad_path; int vad_maxseg=30000; for(int i=1;iwav;if(!funasr_load_audio_16k_mono(wav_path.c_str(),wav)){fprintf(stderr,"read audio failed\n");return 1;} - int64_t t0=ggml_time_us();int T=0;auto fb=compute_fbank(wav,T); - // CMVN (Paraformer applies it): (x+shift)*scale float*shift=(float*)m.g("cmvn.shift")->data,*scale=(float*)m.g("cmvn.scale")->data; - for(int t=0;tenc; - {ggml_init_params cp={(size_t)1024*1024*1024,nullptr,true};ggml_context*c=ggml_init(cp); - ggml_tensor*x=ggml_new_tensor_2d(c,GGML_TYPE_F32,F,T);ggml_set_input(x); - ggml_tensor*h=enc_layer(c,m,"encoder.encoders0.0.",x,T,false); - for(int i=0;i stored row-major as T rows? ggml out ne0=D,ne1=T - // run_graph returns row-major [N=ne1 rows, D=ne0]; here rows=T, cols=D - int64_t t1=ggml_time_us(); - // enc[t*D + d] - - // ===== CIF predictor (host) ===== float*cw=(float*)m.g("predictor.cif_conv1d.weight")->data; // [512,512,3] = [o][i][j] float*cb=(float*)m.g("predictor.cif_conv1d.bias")->data; // [512] float*ow=(float*)m.g("predictor.cif_output.weight")->data; // [1,512] float ob=((float*)m.g("predictor.cif_output.bias")->data)[0]; - // conv1d (k=3,pad1) + residual + relu, then cif_output -> sigmoid -> alpha - std::vector alphas(T); - std::vector outp((size_t)T*D); - for(int t=0;t=T)continue; const float*ev=&enc[(size_t)tt*D]; - const float*wo=&cw[(size_t)o*D*3]; for(int i=0;i0?s:0; // sigmoid, smooth=1 noise=0 -> relu(s)=s - } - // tail: append zero hidden frame + alpha=tail_threshold - std::vector hid=enc; hid.resize((size_t)(T+1)*D,0.0f); - std::vector al=alphas; al.push_back(m.c.tail); - int L=T+1; - // integrate-and-fire - std::vector acoustic; acoustic.reserve(64*D); - float integrate=0; std::vector frame(D,0.0f); - for(int t=0;t=m.c.thresh; float cur=fire?dc:alpha; float rem=alpha-cur; - for(int d=0;d logits; - {ggml_init_params cp={(size_t)2048*1024*1024,nullptr,true};ggml_context*c=ggml_init(cp); - ggml_tensor*tgt=ggml_new_tensor_2d(c,GGML_TYPE_F32,D,N);ggml_set_input(tgt); - ggml_tensor*mem=ggml_new_tensor_2d(c,GGML_TYPE_F32,D,T);ggml_set_input(mem); - ggml_tensor*x=tgt; - for(int i=0;i CMVN -> encoder -> CIF -> decoder) on one wav window; prints token IDs. + auto run_seg=[&](const std::vector& wav){ + int T=0;auto fb=compute_fbank(wav,T); if(T<1)return; + for(int t=0;tenc; + {ggml_init_params cp={(size_t)1024*1024*1024,nullptr,true};ggml_context*c=ggml_init(cp); + ggml_tensor*x=ggml_new_tensor_2d(c,GGML_TYPE_F32,F,T);ggml_set_input(x); + ggml_tensor*h=enc_layer(c,m,"encoder.encoders0.0.",x,T,false); + for(int i=0;i cif_output -> sigmoid -> alpha ===== + std::vector outp((size_t)T*D); std::vector alphas(T); + for(int t=0;t=T)continue; const float*ev=&enc[(size_t)tt*D]; + const float*wo=&cw[(size_t)o*D*3]; for(int i=0;i0?s:0; } + std::vector hid=enc; hid.resize((size_t)(T+1)*D,0.0f); + std::vector al=alphas; al.push_back(m.c.tail); int L=T+1; + std::vector acoustic; acoustic.reserve(64*D); + float integrate=0; std::vector frame(D,0.0f); + for(int t=0;t=m.c.thresh; float cur=fire?dc:alpha; float rem=alpha-cur; + for(int d=0;d logits; + {ggml_init_params cp={(size_t)2048*1024*1024,nullptr,true};ggml_context*c=ggml_init(cp); + ggml_tensor*tgt=ggml_new_tensor_2d(c,GGML_TYPE_F32,D,N);ggml_set_input(tgt); + ggml_tensor*mem=ggml_new_tensor_2d(c,GGML_TYPE_F32,D,T);ggml_set_input(mem); + ggml_tensor*x=tgt; + for(int i=0;ibest){best=col[v];am=v;}printf("%d ",am);} + }; - // argmax per slot -> ids - for(int n=0;nbest){best=col[v];am=v;}printf("%d ",am);} + int64_t t0=ggml_time_us(); + std::vectorwav;if(!funasr_load_audio_16k_mono(wav_path.c_str(),wav)){fprintf(stderr,"read audio failed\n");return 1;} + if(!vad_path.empty()){ + std::vector> segs; + if(!funasr_vad_segments(vad_path,wav,vad_maxseg,segs)){fprintf(stderr,"vad failed\n");return 1;} + for(auto&s:segs){ int off=(int)((int64_t)s.first*16000/1000), end=(int)((int64_t)s.second*16000/1000); + if(end>(int)wav.size())end=wav.size(); if(end-off seg(wav.begin()+off,wav.begin()+end); run_seg(seg); } + fprintf(stderr,"[paraformer] %zu vad segments\n",segs.size()); + } else { run_seg(wav); } printf("\n"); - fprintf(stderr,"[paraformer] T=%d N_tok=%d enc %.2fs dec %.2fs\n",T,N,(t1-t0)/1e6,(t2-t1)/1e6); + fprintf(stderr,"[paraformer] done %.2fs\n",(ggml_time_us()-t0)/1e6); if(m.ctx_w) ggml_free(m.ctx_w); return 0; } diff --git a/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp b/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp index 47f663459..45eb8ac4b 100644 --- a/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp +++ b/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp @@ -23,6 +23,8 @@ static const float LN_EPS = 1e-5f; // ---- audio loader: any wav/mp3/flac, any rate/channels -> 16k mono (miniaudio) ---- #define FUNASR_AUDIO_IMPLEMENTATION #include "funasr_audio.h" +#include "funasr_vad.h" // built-in FSMN-VAD front end (--vad segmentation) +#include static const int FS=16000,WINLEN=400,SHIFT=160,NFFT=512,NMEL=80,LFR_M=7,LFR_N=6; static const float PREEMPH=0.97f,LOWF=20.0f,HIGHF=8000.0f; static inline float melf(float f){return 1127.0f*logf(1.0f+f/700.0f);} @@ -90,11 +92,13 @@ static void add_posenc(std::vector&x,int T,int depth){ } int main(int argc,char**argv){ - std::string gguf_path,fbank_path,wav_path; + std::string gguf_path,fbank_path,wav_path,vad_path; int vad_maxseg=30000; for(int i=1;i fb; - if(!wav_path.empty()){ - std::vector wav; if(!funasr_load_audio_16k_mono(wav_path.c_str(),wav)){fprintf(stderr,"read audio failed\n");return 1;} - int t=0; fb=compute_fbank(wav,t); T=t; - } else { - FILE*f=fopen(fbank_path.c_str(),"rb"); if(!f){fprintf(stderr,"open fbank\n");return 1;} - if(fread(&T,4,1,f)!=1||fread(&Fc,4,1,f)!=1){fclose(f);return 1;} - fb.resize((size_t)T*Fc); if((int)fread(fb.data(),4,fb.size(),f)!=(int)fb.size()){fclose(f);return 1;} fclose(f); - } - // NOTE: SenseVoiceSmall inference() feeds the RAW log-mel fbank to the encoder; // it does NOT apply am.mvn CMVN (that path is unused at inference). Applying it // makes the encoder predict <|nospeech|>. So no CMVN here. - - // prepend nq query tokens (rows of embed.weight) -> input [nq+T, F] float*emb=(float*)m.g("embed.weight")->data; // [16, 560] row-major - int N=nq+T; std::vector inp((size_t)N*F); - for(int i=0;i& fb,int T){ + int N=nq+T; std::vector inp((size_t)N*F); + for(int i=0;i lg((size_t)V*N); ggml_backend_tensor_get(logits,lg.data(),0,ggml_nbytes(logits)); + int prev=-1; // greedy CTC: argmax per frame -> collapse consecutive -> drop blank + for(int n=0;nbest){best=col[v];am=v;} + if(am!=prev && am!=m.c.blank) printf("%d ", am); prev=am; } + ggml_gallocr_free(ga); ggml_free(c); ggml_backend_free(be); + }; - // encoder pre-scale + posenc - float sc=sqrtf((float)D); for(auto&v:inp)v*=sc; add_posenc(inp,N,F); - - // build graph - ggml_backend_t be=ggml_backend_cpu_init(); - ggml_init_params cp={(size_t)1024*1024*1024,nullptr,true}; ggml_context*c=ggml_init(cp); - ggml_tensor*x=ggml_new_tensor_2d(c,GGML_TYPE_F32,F,N); ggml_set_input(x); - ggml_tensor*h=sanm_layer(c,m,"encoder.encoders0.0.",x,N,false); - for(int i=0;i lg((size_t)V*N); ggml_backend_tensor_get(logits,lg.data(),0,ggml_nbytes(logits)); - // greedy CTC: argmax per frame -> collapse consecutive -> drop blank - int prev=-1; - for(int n=0;nbest){best=col[v];am=v;} - if(am!=prev && am!=m.c.blank) printf("%d ", am); - prev=am; + if(!vad_path.empty()){ + std::vector wav; if(!funasr_load_audio_16k_mono(wav_path.c_str(),wav)){fprintf(stderr,"read audio failed\n");return 1;} + std::vector> segs; + if(!funasr_vad_segments(vad_path,wav,vad_maxseg,segs)){fprintf(stderr,"vad failed\n");return 1;} + for(auto&s:segs){ int off=(int)((int64_t)s.first*16000/1000), end=(int)((int64_t)s.second*16000/1000); + if(end>(int)wav.size())end=wav.size(); if(end-off seg(wav.begin()+off,wav.begin()+end); int t=0; auto fb=compute_fbank(seg,t); run_seg(fb,t); } + fprintf(stderr,"[sensevoice] %zu vad segments\n",segs.size()); + } else { + int32_t T=0,Fc=F; std::vector fb; + if(!wav_path.empty()){ + std::vector wav; if(!funasr_load_audio_16k_mono(wav_path.c_str(),wav)){fprintf(stderr,"read audio failed\n");return 1;} + int t=0; fb=compute_fbank(wav,t); T=t; + } else { + FILE*f=fopen(fbank_path.c_str(),"rb"); if(!f){fprintf(stderr,"open fbank\n");return 1;} + if(fread(&T,4,1,f)!=1||fread(&Fc,4,1,f)!=1){fclose(f);return 1;} + fb.resize((size_t)T*Fc); if((int)fread(fb.data(),4,fb.size(),f)!=(int)fb.size()){fclose(f);return 1;} fclose(f); + } + run_seg(fb,T); } printf("\n"); - fprintf(stderr,"[sensevoice] N=%d (q%d+T%d) encode %.2fs\n",N,nq,T,(t1-t0)/1e6); - ggml_gallocr_free(ga); ggml_free(c); ggml_backend_free(be); + fprintf(stderr,"[sensevoice] done %.2fs\n",(ggml_time_us()-t0)/1e6); if(m.ctx_w) ggml_free(m.ctx_w); return 0; }