diff --git a/runtime/llama.cpp/README.md b/runtime/llama.cpp/README.md index c007e30cd..1004000c0 100644 --- a/runtime/llama.cpp/README.md +++ b/runtime/llama.cpp/README.md @@ -58,6 +58,24 @@ Pre-converted GGUF on Hugging Face: [SenseVoiceSmall-GGUF](https://huggingface.c cmake -B build -DCMAKE_BUILD_TYPE=Release # fetches pinned llama.cpp; static, self-contained cmake --build build -j # -> build/bin/llama-funasr-* (all tools) ``` + +### Optional CUDA backend for SenseVoiceSmall + +The prebuilt release ZIPs are CPU-first portable packages. To experiment with +SenseVoiceSmall graph execution on NVIDIA GPUs, build from source with ggml CUDA +enabled and select the backend at runtime: + +```bash +cmake -B build-cuda -DCMAKE_BUILD_TYPE=Release -DGGML_CUDA=ON +cmake --build build-cuda -j --target llama-funasr-sensevoice +./build-cuda/bin/llama-funasr-sensevoice \ + -m sensevoice-small-f16.gguf -a sample.wav --backend cuda +``` + +`--backend cpu` remains the default and is what the current cross-platform +prebuilt binaries use. A binary built without `-DGGML_CUDA=ON` exits with a clear +message if `--backend cuda` is requested. + ## Build (shared) ```bash git clone https://github.com/ggml-org/llama.cpp && cd llama.cpp diff --git a/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp b/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp index 778eaee56..4d915ea40 100644 --- a/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp +++ b/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp @@ -60,6 +60,39 @@ struct cfg { int d_model=512,n_head=4,num_blocks=50,tp_blocks=20,kernel=11,vocab struct model { cfg c; ggml_context*ctx_w=nullptr; std::map t; ggml_tensor* g(const std::string&n){auto it=t.find(n);if(it==t.end()){fprintf(stderr,"missing %s\n",n.c_str());exit(1);}return it->second;} }; +struct graph_backend { + ggml_backend_t backend=nullptr; + ggml_backend_buffer_type_t buffer_type=nullptr; + bool is_cpu=true; +}; + +static graph_backend make_graph_backend(const std::string&name){ + graph_backend out; + if(name=="cpu"){ + out.backend=ggml_backend_cpu_init(); + out.buffer_type=ggml_backend_get_default_buffer_type(out.backend); + out.is_cpu=true; + } else if(name=="cuda"){ + ggml_backend_load_all(); + ggml_backend_dev_t dev=ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU); + if(!dev){ + fprintf(stderr,"CUDA backend requested, but no GPU backend is available; build with -DGGML_CUDA=ON\n"); + exit(1); + } + out.backend=ggml_backend_dev_init(dev,nullptr); + out.buffer_type=ggml_backend_get_default_buffer_type(out.backend); + out.is_cpu=false; + } else { + fprintf(stderr,"unsupported backend '%s' (expected cpu|cuda)\n",name.c_str()); + exit(1); + } + if(!out.backend||!out.buffer_type){ + fprintf(stderr,"failed to initialize %s backend\n",name.c_str()); + exit(1); + } + return out; +} + static ggml_tensor* lin(ggml_context*c,ggml_tensor*w,ggml_tensor*b,ggml_tensor*x){auto y=ggml_mul_mat(c,w,x);return b?ggml_add(c,y,b):y;} static ggml_tensor* lnorm(ggml_context*c,ggml_tensor*x,ggml_tensor*g,ggml_tensor*b){return ggml_add(c,ggml_mul(c,ggml_norm(c,x,LN_EPS),g),b);} static ggml_tensor* sanm_attn(ggml_context*c,model&m,const std::string&p,ggml_tensor*x,int T){ @@ -104,15 +137,18 @@ static std::string detok_sv(const std::vector&ids,const std::vector lg((size_t)V*N); ggml_backend_tensor_get(logits,lg.data(),0,ggml_nbytes(logits)); std::vector seg_ids; int prev=-1; // greedy CTC: argmax per frame -> collapse -> drop blank for(int n=0;n