Merge pull request #3550 from modelscope/codex/fix-sensevoice-empty-output

fix(runtime): decode SenseVoice F16 query embeddings

runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp

@@ -241,11 +241,22 @@ int main(int argc,char**argv){
   // NOTE: SenseVoiceSmall inference() feeds the RAW log-mel fbank to the encoder;
   // it does NOT apply am.mvn CMVN (that path is unused at inference). Applying it
   // makes the encoder predict <|nospeech|>. So no CMVN here.
-  float*emb=(float*)m.g("embed.weight")->data;   // [16, 560] row-major
+  ggml_tensor*embed=m.g("embed.weight");   // [16, 560] row-major
+  if(embed->ne[0]!=F){fprintf(stderr,"unexpected embed width: %lld\n",(long long)embed->ne[0]);return 1;}
+  for(int id:qtok) if(id<0||id>=embed->ne[1]){fprintf(stderr,"query token out of embed range: %d\n",id);return 1;}
+  std::vector<float> embed_f32((size_t)ggml_nelements(embed));
+  if(embed->type==GGML_TYPE_F32){
+    memcpy(embed_f32.data(),embed->data,embed_f32.size()*sizeof(float));
+  } else if(embed->type==GGML_TYPE_F16){
+    const ggml_fp16_t*src=(const ggml_fp16_t*)embed->data;
+    for(size_t i=0;i<embed_f32.size();i++)embed_f32[i]=ggml_fp16_to_fp32(src[i]);
+  } else {
+    fprintf(stderr,"unsupported embed type: %s\n",ggml_type_name(embed->type));return 1;
+  }
   // Run encoder+CTC on one fbank window [T,F]; returns decoded text string.
   auto run_seg=[&](const std::vector<float>& fb,int T) -> std::string {
     int N=nq+T; std::vector<float> inp((size_t)N*F);
-    for(int i=0;i<nq;i++) memcpy(&inp[(size_t)i*F], &emb[(size_t)qtok[i]*F], F*sizeof(float));
+    for(int i=0;i<nq;i++) memcpy(&inp[(size_t)i*F], &embed_f32[(size_t)qtok[i]*F], F*sizeof(float));
     memcpy(&inp[(size_t)nq*F], fb.data(), (size_t)T*F*sizeof(float));
     float sc=sqrtf((float)D); for(auto&v:inp)v*=sc; add_posenc(inp,N,F);
     trace_stage("[sensevoice] building graph: %d frames",N);

runtime/llama.cpp/tests/golden/sensevoice.txt

@@ -1 +1 @@
-我想问我在滨海新区有房
+我想问我在滨海新区有房。

runtime/llama.cpp/tests/test_backend_flags.py

@@ -12,6 +12,16 @@ def test_sensevoice_exposes_backend_flag():
     assert "cpu|cuda|vulkan" in source
 
 
+def test_sensevoice_reads_query_embeddings_using_their_ggml_type():
+    source = SENSEVOICE.read_text(encoding="utf-8")
+
+    assert 'ggml_tensor*embed=m.g("embed.weight")' in source
+    assert "embed->type==GGML_TYPE_F32" in source
+    assert "embed->type==GGML_TYPE_F16" in source
+    assert "ggml_fp16_to_fp32" in source
+    assert 'float*emb=(float*)m.g("embed.weight")->data' not in source
+
+
 def test_sensevoice_does_not_hardcode_cpu_graph_backend():
     source = SENSEVOICE.read_text(encoding="utf-8")
     run_seg_body = source.split("auto run_seg=", maxsplit=1)[1].split("int64_t t0=", maxsplit=1)[0]