modelscope/FunASR
1
0
Fork 0
mirror of https://github.com/modelscope/FunASR.git synced 2026-10-12 02:19:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add SenseVoice llama.cpp backend selector (#3247)

Browse Source
This commit is contained in:
zhifu gao
2026-07-18 19:16:03 +08:00
committed by GitHub
parent d41ca43b33
commit 57721fc8ce
3 changed files with 82 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/llama.cpp/README.md
+18
View File
@@ -58,6 +58,24 @@ Pre-converted GGUF on Hugging Face: [SenseVoiceSmall-GGUF](https://huggingface.c
cmake -B build -DCMAKE_BUILD_TYPE=Release # fetches pinned llama.cpp; static, self-contained
cmake --build build -j # -> build/bin/llama-funasr-* (all tools)
```
### Optional CUDA backend for SenseVoiceSmall
The prebuilt release ZIPs are CPU-first portable packages. To experiment with
SenseVoiceSmall graph execution on NVIDIA GPUs, build from source with ggml CUDA
enabled and select the backend at runtime:
```bash
cmake -B build-cuda -DCMAKE_BUILD_TYPE=Release -DGGML_CUDA=ON
cmake --build build-cuda -j --target llama-funasr-sensevoice
./build-cuda/bin/llama-funasr-sensevoice \
-m sensevoice-small-f16.gguf -a sample.wav --backend cuda
```
`--backend cpu` remains the default and is what the current cross-platform
prebuilt binaries use. A binary built without `-DGGML_CUDA=ON` exits with a clear
message if `--backend cuda` is requested.
## Build (shared)
```bash
git clone https://github.com/ggml-org/llama.cpp && cd llama.cpp
runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp
+42 -6
View File
@@ -60,6 +60,39 @@ struct cfg { int d_model=512,n_head=4,num_blocks=50,tp_blocks=20,kernel=11,vocab
struct model { cfg c; ggml_context*ctx_w=nullptr; std::map<std::string,ggml_tensor*> t;
ggml_tensor* g(const std::string&n){auto it=t.find(n);if(it==t.end()){fprintf(stderr,"missing %s\n",n.c_str());exit(1);}return it->second;} };
struct graph_backend {
ggml_backend_t backend=nullptr;
ggml_backend_buffer_type_t buffer_type=nullptr;
bool is_cpu=true;
};
static graph_backend make_graph_backend(const std::string&name){
graph_backend out;
if(name=="cpu"){
out.backend=ggml_backend_cpu_init();
out.buffer_type=ggml_backend_get_default_buffer_type(out.backend);
out.is_cpu=true;
} else if(name=="cuda"){
ggml_backend_load_all();
ggml_backend_dev_t dev=ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU);
if(!dev){
fprintf(stderr,"CUDA backend requested, but no GPU backend is available; build with -DGGML_CUDA=ON\n");
exit(1);
}
out.backend=ggml_backend_dev_init(dev,nullptr);
out.buffer_type=ggml_backend_get_default_buffer_type(out.backend);
out.is_cpu=false;
} else {
fprintf(stderr,"unsupported backend '%s' (expected cpu|cuda)\n",name.c_str());
exit(1);
}
if(!out.backend||!out.buffer_type){
fprintf(stderr,"failed to initialize %s backend\n",name.c_str());
exit(1);
}
return out;
}
static ggml_tensor* lin(ggml_context*c,ggml_tensor*w,ggml_tensor*b,ggml_tensor*x){auto y=ggml_mul_mat(c,w,x);return b?ggml_add(c,y,b):y;}
static ggml_tensor* lnorm(ggml_context*c,ggml_tensor*x,ggml_tensor*g,ggml_tensor*b){return ggml_add(c,ggml_mul(c,ggml_norm(c,x,LN_EPS),g),b);}
static ggml_tensor* sanm_attn(ggml_context*c,model&m,const std::string&p,ggml_tensor*x,int T){
@@ -104,15 +137,18 @@ static std::string detok_sv(const std::vector<int>&ids,const std::vector<std::st
int main(int argc,char**argv){
std::string gguf_path,fbank_path,wav_path,vad_path; int vad_maxseg=30000; bool ids_mode=false,keep_tags=false;
std::string backend_name="cpu";
for(int i=1;i<argc;i++){ if(!strcmp(argv[i],"-m")&&i+1<argc)gguf_path=argv[++i];
else if(!strcmp(argv[i],"-f")&&i+1<argc)fbank_path=argv[++i];
else if(!strcmp(argv[i],"-a")&&i+1<argc)wav_path=argv[++i];
else if(!strcmp(argv[i],"--vad")&&i+1<argc)vad_path=argv[++i];
else if(!strcmp(argv[i],"--vad-maxseg")&&i+1<argc)vad_maxseg=atoi(argv[++i]);
else if(!strcmp(argv[i],"--backend")&&i+1<argc)backend_name=argv[++i];
else if(!strcmp(argv[i],"--ids"))ids_mode=true;
else if(!strcmp(argv[i],"--keep-tags"))keep_tags=true;
else {fprintf(stderr,"usage: %s -m sensevoice.gguf (-a audio.wav | -f fbank.bin) [--vad fsmn-vad.gguf [--vad-maxseg ms]] [--ids] [--keep-tags]\n",argv[0]);return 1;} }
else {fprintf(stderr,"usage: %s -m sensevoice.gguf (-a audio.wav | -f fbank.bin) [--vad fsmn-vad.gguf [--vad-maxseg ms]] [--backend cpu|cuda] [--ids] [--keep-tags]\n",argv[0]);return 1;} }
if(gguf_path.empty()||(fbank_path.empty()&&wav_path.empty())){fprintf(stderr,"missing args\n");return 1;}
graph_backend graph_be=make_graph_backend(backend_name);
// load model
model m; gguf_init_params gp={false,&m.ctx_w}; gguf_context*gg=gguf_init_from_file(gguf_path.c_str(),gp);
@@ -139,7 +175,6 @@ int main(int argc,char**argv){
for(int i=0;i<nq;i++) memcpy(&inp[(size_t)i*F], &emb[(size_t)qtok[i]*F], F*sizeof(float));
memcpy(&inp[(size_t)nq*F], fb.data(), (size_t)T*F*sizeof(float));
float sc=sqrtf((float)D); for(auto&v:inp)v*=sc; add_posenc(inp,N,F);
ggml_backend_t be=ggml_backend_cpu_init();
ggml_init_params cp={(size_t)1024*1024*1024,nullptr,true}; ggml_context*c=ggml_init(cp);
ggml_tensor*x=ggml_new_tensor_2d(c,GGML_TYPE_F32,F,N); ggml_set_input(x);
ggml_tensor*h=sanm_layer(c,m,"encoder.encoders0.0.",x,N,false);
@@ -150,9 +185,9 @@ int main(int argc,char**argv){
ggml_tensor*logits=lin(c,m.g("ctc.ctc_lo.weight"),m.g("ctc.ctc_lo.bias"),h); // [V, N]
ggml_set_output(logits);
ggml_cgraph*gf=ggml_new_graph_custom(c,32768,false); ggml_build_forward_expand(gf,logits);
ggml_gallocr_t ga=ggml_gallocr_new(ggml_backend_cpu_buffer_type()); ggml_gallocr_alloc_graph(ga,gf);
ggml_backend_tensor_set(x,inp.data(),0,ggml_nbytes(x)); ggml_backend_cpu_set_n_threads(be,8);
if(ggml_backend_graph_compute(be,gf)!=GGML_STATUS_SUCCESS){fprintf(stderr,"compute failed\n");}
ggml_gallocr_t ga=ggml_gallocr_new(graph_be.buffer_type); ggml_gallocr_alloc_graph(ga,gf);
ggml_backend_tensor_set(x,inp.data(),0,ggml_nbytes(x)); if(graph_be.is_cpu) ggml_backend_cpu_set_n_threads(graph_be.backend,8);
if(ggml_backend_graph_compute(graph_be.backend,gf)!=GGML_STATUS_SUCCESS){fprintf(stderr,"compute failed\n");}
std::vector<float> lg((size_t)V*N); ggml_backend_tensor_get(logits,lg.data(),0,ggml_nbytes(logits));
std::vector<int> seg_ids; int prev=-1; // greedy CTC: argmax per frame -> collapse -> drop blank
for(int n=0;n<N;n++){ const float*col=&lg[(size_t)n*V]; int am=0; float best=col[0];
@@ -160,7 +195,7 @@ int main(int argc,char**argv){
if(am!=prev && am!=m.c.blank) seg_ids.push_back(am); prev=am; }
if(emit_ids){ for(int id:seg_ids) printf("%d ",id); }
else { std::string t=detok_sv(seg_ids,vocab,keep_tags); printf("%s",t.c_str()); }
ggml_gallocr_free(ga); ggml_free(c); ggml_backend_free(be);
ggml_gallocr_free(ga); ggml_free(c);
};
int64_t t0=ggml_time_us();
@@ -186,6 +221,7 @@ int main(int argc,char**argv){
}
printf("\n");
fprintf(stderr,"[sensevoice] done %.2fs\n",(ggml_time_us()-t0)/1e6);
ggml_backend_free(graph_be.backend);
if(m.ctx_w) ggml_free(m.ctx_w);
return 0;
}
runtime/llama.cpp/tests/test_backend_flags.py
+22
View File
@@ -0,0 +1,22 @@
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SENSEVOICE = ROOT / "sensevoice" / "funasr-sensevoice" / "funasr-sensevoice.cpp"
def test_sensevoice_exposes_backend_flag():
source = SENSEVOICE.read_text(encoding="utf-8")
assert "--backend" in source
assert "cpu|cuda" in source
def test_sensevoice_does_not_hardcode_cpu_graph_backend():
source = SENSEVOICE.read_text(encoding="utf-8")
run_seg_body = source.split("auto run_seg=", maxsplit=1)[1].split("int64_t t0=", maxsplit=1)[0]
assert "graph_be.backend" in run_seg_body
assert "graph_be.buffer_type" in run_seg_body
assert "ggml_backend_cpu_init()" not in run_seg_body
assert "ggml_backend_cpu_buffer_type()" not in run_seg_body