modelscope/FunASR
1
0
Fork 0
mirror of https://github.com/modelscope/FunASR.git synced 2026-10-12 02:19:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: print transcription text in the binaries (embed vocab, in-binary detok) (#3004)

Browse Source 
SenseVoice and Paraformer binaries now output text directly (whisper-cli style),
no Python detok step.

- export_{sensevoice,paraformer}_gguf.py: embed the vocab as a GGUF string array
  (sv.vocab = SentencePiece pieces; pf.vocab = tokens.json) so the binary is self-contained.
- funasr-sensevoice.cpp: CTC ids -> text (pieces + ▁->space), drop <|...|> meta by
  default; --ids (raw), --keep-tags (keep meta tags).
- funasr-paraformer.cpp: ids -> text (tokens.json, drop @@ continuations); --ids for raw.
- both fall back to ids if the GGUF has no vocab. READMEs updated to text output.

Verified: C++ text == reference detok (sentencepiece / tokens.json) on sample and the
full 002 clip; ids unchanged so CER is unchanged.
This commit is contained in:
zhifu gao
2026-06-20 16:39:25 +08:00
committed by GitHub
parent 3610b759ce
commit 807bcd5647
6 changed files with 63 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/llama.cpp/paraformer/README.md
+1 -2
View File
@@ -60,8 +60,7 @@ python runtime/llama.cpp/export_paraformer_gguf.py --wtype f16 \
**3. Transcribe:**
```bash
build/bin/llama-funasr-paraformer -m paraformer.gguf -a audio.wav > ids.txt
python runtime/llama.cpp/detok.py <model>/tokens.json ids.txt
build/bin/llama-funasr-paraformer -m paraformer.gguf -a audio.wav # prints transcription text (--ids for raw)
```
Expected output:
```
runtime/llama.cpp/paraformer/export_paraformer_gguf.py
+8
View File
@@ -20,6 +20,7 @@ def main():
ap = argparse.ArgumentParser()
ap.add_argument("--model_pt", required=True); ap.add_argument("--mvn", required=True)
ap.add_argument("--out", required=True); ap.add_argument("--wtype", default="f32", choices=["f32","f16"])
ap.add_argument("--tokens", default=None, help="tokens.json (id->token); default: next to model_pt")
a = ap.parse_args()
sd = torch.load(a.model_pt, map_location="cpu"); sd = sd.get("state_dict", sd)
w = gguf.GGUFWriter(a.out, "paraformer")
@@ -28,6 +29,13 @@ def main():
w.add_uint32("pf.dec.num_blocks", 16); w.add_uint32("pf.dec.att_layer_num", 16)
w.add_uint32("pf.dec.decoders3", 1); w.add_uint32("pf.dec.attention_heads", 4)
w.add_uint32("pf.dec.kernel_size", 11); w.add_uint32("pf.vocab_size", 8404)
import json, glob
tp = a.tokens or (glob.glob(os.path.join(os.path.dirname(a.model_pt), "tokens.json")) + [None])[0]
if tp and os.path.exists(tp):
toks = json.load(open(tp)); w.add_array("pf.vocab", toks)
print(f"embedded pf.vocab ({len(toks)} tokens) from {tp}")
else:
print("WARNING: tokens.json not found - gguf will have no vocab (binary falls back to ids)")
w.add_float32("pf.predictor.tail_threshold", 0.45)
w.add_float32("pf.predictor.threshold", 1.0)
shift, scale = parse_mvn(a.mvn)
runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp
+18 -3
View File
@@ -119,20 +119,32 @@ static std::vector<float> run_graph(ggml_context*c,ggml_tensor*out,ggml_tensor*i
int D=out->ne[0],N=out->ne[1];std::vector<float>r((size_t)D*N);ggml_backend_tensor_get(out,r.data(),0,ggml_nbytes(out));
ggml_gallocr_free(ga);ggml_backend_free(be);return r;}
// Paraformer detok: tokens.json is BPE; join, drop "@@" continuations, "▁"(U+2581)->space.
static std::string pf_trim(const std::string&s){size_t a=s.find_first_not_of(' ');if(a==std::string::npos)return "";size_t b=s.find_last_not_of(' ');return s.substr(a,b-a+1);}
static std::string detok_pf(const std::vector<int>&ids,const std::vector<std::string>&vocab){
std::string s; for(int id:ids){ if(id==1||id==2)continue; if(id>=0&&id<(int)vocab.size())s+=vocab[id]; }
size_t p; while((p=s.find("@@"))!=std::string::npos)s.erase(p,2);
const std::string lb="\xe2\x96\x81"; while((p=s.find(lb))!=std::string::npos)s.replace(p,3," ");
return pf_trim(s);
}
int main(int argc,char**argv){
std::string gguf_path,wav_path,vad_path; int vad_maxseg=30000;
std::string gguf_path,wav_path,vad_path; int vad_maxseg=30000; bool ids_mode=false;
for(int i=1;i<argc;i++){if(!strcmp(argv[i],"-m")&&i+1<argc)gguf_path=argv[++i];else if(!strcmp(argv[i],"-a")&&i+1<argc)wav_path=argv[++i];
else if(!strcmp(argv[i],"--vad")&&i+1<argc)vad_path=argv[++i];
else if(!strcmp(argv[i],"--vad-maxseg")&&i+1<argc)vad_maxseg=atoi(argv[++i]);
else{fprintf(stderr,"usage: %s -m paraformer.gguf -a audio.wav [--vad fsmn-vad.gguf [--vad-maxseg ms]]\n",argv[0]);return 1;}}
else if(!strcmp(argv[i],"--ids"))ids_mode=true;
else{fprintf(stderr,"usage: %s -m paraformer.gguf -a audio.wav [--vad fsmn-vad.gguf [--vad-maxseg ms]] [--ids]\n",argv[0]);return 1;}}
if(gguf_path.empty()||wav_path.empty()){fprintf(stderr,"missing args\n");return 1;}
model m;gguf_init_params gp={false,&m.ctx_w};gguf_context*gg=gguf_init_from_file(gguf_path.c_str(),gp);if(!gg){fprintf(stderr,"gguf load failed\n");return 1;}
auto rdi=[&](const char*k,int d){int i=gguf_find_key(gg,k);return i<0?d:(int)gguf_get_val_u32(gg,i);};
auto rdf=[&](const char*k,float d){int i=gguf_find_key(gg,k);return i<0?d:gguf_get_val_f32(gg,i);};
m.c.enc_blocks=rdi("pf.enc.num_blocks",50);m.c.dec_blocks=rdi("pf.dec.num_blocks",16);m.c.dec_att=rdi("pf.dec.att_layer_num",16);
m.c.dec3=rdi("pf.dec.decoders3",1);m.c.vocab=rdi("pf.vocab_size",8404);m.c.tail=rdf("pf.predictor.tail_threshold",0.45f);m.c.thresh=rdf("pf.predictor.threshold",1.0f);
std::vector<std::string> vocab; {int ki=gguf_find_key(gg,"pf.vocab"); if(ki>=0){int nv=gguf_get_arr_n(gg,ki); vocab.resize(nv); for(int i=0;i<nv;i++)vocab[i]=gguf_get_arr_str(gg,ki,i);}}
for(int i=0;i<gguf_get_n_tensors(gg);i++){const char*nm=gguf_get_tensor_name(gg,i);m.t[nm]=ggml_get_tensor(m.ctx_w,nm);}gguf_free(gg);
const int D=m.c.d_model,F=560,V=m.c.vocab;
bool emit_ids = ids_mode || vocab.empty(); // fall back to ids if the gguf has no vocab
float*shift=(float*)m.g("cmvn.shift")->data,*scale=(float*)m.g("cmvn.scale")->data;
float*cw=(float*)m.g("predictor.cif_conv1d.weight")->data; // [512,512,3] = [o][i][j]
@@ -183,7 +195,10 @@ int main(int argc,char**argv){
x=lin(c,m.g("decoder.output_layer.weight"),m.g("decoder.output_layer.bias"),x); // [V,N]
ggml_set_output(x);
logits=run_graph(c,x,tgt,acoustic.data(),mem,enc.data());ggml_free(c);}
for(int n=0;n<N;n++){const float*col=&logits[(size_t)n*V];int am=0;float best=col[0];for(int v=1;v<V;v++)if(col[v]>best){best=col[v];am=v;}printf("%d ",am);}
std::vector<int> seg_ids; seg_ids.reserve(N);
for(int n=0;n<N;n++){const float*col=&logits[(size_t)n*V];int am=0;float best=col[0];for(int v=1;v<V;v++)if(col[v]>best){best=col[v];am=v;}seg_ids.push_back(am);}
if(emit_ids){ for(int id:seg_ids) printf("%d ",id); }
else { std::string t=detok_pf(seg_ids,vocab); printf("%s",t.c_str()); }
};
int64_t t0=ggml_time_us();
runtime/llama.cpp/sensevoice/README.md
+3 -4
View File
@@ -61,13 +61,12 @@ python runtime/llama.cpp/export_sensevoice_gguf.py --wtype f16 \
**3. Transcribe:**
```bash
build/bin/llama-funasr-sensevoice -m sensevoice-small.gguf -a audio.wav > ids.txt
python runtime/llama.cpp/detok.py <model>/chn_jpn_yue_eng_ko_spectok.bpe.model ids.txt
build/bin/llama-funasr-sensevoice -m sensevoice-small.gguf -a audio.wav # prints transcription text
# --keep-tags keeps the <|lang|>/<|emotion|>/<|event|> tags; --ids prints raw CTC ids
```
Expected output:
```
<|zh|><|NEUTRAL|><|Speech|><|woitn|>我想问我在滨海新区有房我一直没有照顾孩子...你觉得这是正常的想法吗
[sensevoice] N=746 (q4+T742) encode 1.32s
我想问我在滨海新区有房我一直没有照顾孩子...你觉得这是正常的想法吗
```
The leading `<|...|>` tags are the predicted language / emotion / event / ITN.
runtime/llama.cpp/sensevoice/export_sensevoice_gguf.py
+12 -1
View File
@@ -23,6 +23,7 @@ def main():
ap.add_argument("--mvn", required=True)
ap.add_argument("--out", required=True)
ap.add_argument("--wtype", default="f32", choices=["f32", "f16"])
ap.add_argument("--spm", default=None, help="sentencepiece .bpe.model; default: next to model_pt")
args = ap.parse_args()
sd = torch.load(args.model_pt, map_location="cpu"); sd = sd.get("state_dict", sd)
@@ -36,7 +37,17 @@ def main():
w.add_uint32("sv.vocab_size", 25055)
w.add_uint32("sv.blank_id", 0)
# query token embed indices used at inference: [lid(auto=0), 1, 2, textnorm(woitn=15)]
w.add_array("sv.query_tokens", [0, 1, 2, 15])
w.add_array("sv.query_tokens", [0, 1, 2, 14]) # 14=withitn (use_itn=True), matches authoritative
import glob
spm_path = args.spm or (glob.glob(os.path.join(os.path.dirname(args.model_pt), "*.bpe.model")) + [None])[0]
if spm_path and os.path.exists(spm_path):
import sentencepiece as spm
sp = spm.SentencePieceProcessor(model_file=spm_path)
pieces = [sp.id_to_piece(i) for i in range(sp.get_piece_size())]
w.add_array("sv.vocab", pieces)
print(f"embedded sv.vocab ({len(pieces)} pieces) from {spm_path}")
else:
print("WARNING: *.bpe.model not found - gguf will have no vocab (binary falls back to ids)")
shift, scale = parse_mvn(args.mvn)
w.add_tensor("cmvn.shift", shift) # (560,)
runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp
+21 -4
View File
@@ -91,14 +91,27 @@ static void add_posenc(std::vector<float>&x,int T,int depth){
x[(size_t)t*depth+i]+=(float)sin(st);x[(size_t)t*depth+depth/2+i]+=(float)cos(st);}}
}
// SenseVoice detok: sentencepiece pieces (no byte-fallback in this vocab) -> join,
// "▁"(U+2581)->space; meta tokens <|lang|>/<|emo|>/<|event|>/<|itn|> dropped unless --keep-tags.
static std::string sv_trim(const std::string&s){size_t a=s.find_first_not_of(' ');if(a==std::string::npos)return "";size_t b=s.find_last_not_of(' ');return s.substr(a,b-a+1);}
static std::string detok_sv(const std::vector<int>&ids,const std::vector<std::string>&vocab,bool keep_tags){
std::string s; for(int id:ids){ if(id<0||id>=(int)vocab.size())continue; const std::string&p=vocab[id];
if(!keep_tags && p.size()>=2 && p[0]=='<' && p[1]=='|') continue; // skip <|...|> meta
s+=p; }
const std::string lb="\xe2\x96\x81"; size_t pp; while((pp=s.find(lb))!=std::string::npos)s.replace(pp,3," ");
return sv_trim(s);
}
int main(int argc,char**argv){
std::string gguf_path,fbank_path,wav_path,vad_path; int vad_maxseg=30000;
std::string gguf_path,fbank_path,wav_path,vad_path; int vad_maxseg=30000; bool ids_mode=false,keep_tags=false;
for(int i=1;i<argc;i++){ if(!strcmp(argv[i],"-m")&&i+1<argc)gguf_path=argv[++i];
else if(!strcmp(argv[i],"-f")&&i+1<argc)fbank_path=argv[++i];
else if(!strcmp(argv[i],"-a")&&i+1<argc)wav_path=argv[++i];
else if(!strcmp(argv[i],"--vad")&&i+1<argc)vad_path=argv[++i];
else if(!strcmp(argv[i],"--vad-maxseg")&&i+1<argc)vad_maxseg=atoi(argv[++i]);
else {fprintf(stderr,"usage: %s -m sensevoice.gguf (-a audio.wav | -f fbank.bin) [--vad fsmn-vad.gguf [--vad-maxseg ms]]\n",argv[0]);return 1;} }
else if(!strcmp(argv[i],"--ids"))ids_mode=true;
else if(!strcmp(argv[i],"--keep-tags"))keep_tags=true;
else {fprintf(stderr,"usage: %s -m sensevoice.gguf (-a audio.wav | -f fbank.bin) [--vad fsmn-vad.gguf [--vad-maxseg ms]] [--ids] [--keep-tags]\n",argv[0]);return 1;} }
if(gguf_path.empty()||(fbank_path.empty()&&wav_path.empty())){fprintf(stderr,"missing args\n");return 1;}
// load model
@@ -110,9 +123,11 @@ int main(int argc,char**argv){
m.c.kernel=rd("sv.kernel_size",11); m.c.vocab=rd("sv.vocab_size",25055); m.c.blank=rd("sv.blank_id",0);
int qi=gguf_find_key(gg,"sv.query_tokens"); int nq=qi<0?0:(int)gguf_get_arr_n(gg,qi);
std::vector<int> qtok(nq); for(int i=0;i<nq;i++) qtok[i]=((const int32_t*)gguf_get_arr_data(gg,qi))[i];
std::vector<std::string> vocab; {int ki=gguf_find_key(gg,"sv.vocab"); if(ki>=0){int nv=gguf_get_arr_n(gg,ki); vocab.resize(nv); for(int i=0;i<nv;i++)vocab[i]=gguf_get_arr_str(gg,ki,i);}}
for(int i=0;i<gguf_get_n_tensors(gg);i++){const char*nm=gguf_get_tensor_name(gg,i);m.t[nm]=ggml_get_tensor(m.ctx_w,nm);}
gguf_free(gg);
const int F=560, D=m.c.d_model, V=m.c.vocab;
bool emit_ids = ids_mode || vocab.empty(); // fall back to ids if the gguf has no vocab
// NOTE: SenseVoiceSmall inference() feeds the RAW log-mel fbank to the encoder;
// it does NOT apply am.mvn CMVN (that path is unused at inference). Applying it
@@ -139,10 +154,12 @@ int main(int argc,char**argv){
ggml_backend_tensor_set(x,inp.data(),0,ggml_nbytes(x)); ggml_backend_cpu_set_n_threads(be,8);
if(ggml_backend_graph_compute(be,gf)!=GGML_STATUS_SUCCESS){fprintf(stderr,"compute failed\n");}
std::vector<float> lg((size_t)V*N); ggml_backend_tensor_get(logits,lg.data(),0,ggml_nbytes(logits));
int prev=-1; // greedy CTC: argmax per frame -> collapse consecutive -> drop blank
std::vector<int> seg_ids; int prev=-1; // greedy CTC: argmax per frame -> collapse -> drop blank
for(int n=0;n<N;n++){ const float*col=&lg[(size_t)n*V]; int am=0; float best=col[0];
for(int v=1;v<V;v++) if(col[v]>best){best=col[v];am=v;}
if(am!=prev && am!=m.c.blank) printf("%d ", am); prev=am; }
if(am!=prev && am!=m.c.blank) seg_ids.push_back(am); prev=am; }
if(emit_ids){ for(int id:seg_ids) printf("%d ",id); }
else { std::string t=detok_sv(seg_ids,vocab,keep_tags); printf("%s",t.c_str()); }
ggml_gallocr_free(ga); ggml_free(c); ggml_backend_free(be);
};