modelscope/FunASR
1
0
Fork 0
mirror of https://github.com/modelscope/FunASR.git synced 2026-10-12 02:19:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: detok review findings (null gguf_get_arr_str guard + utf-8 tokens.json read) (#3005)

Browse Source
This commit is contained in:
zhifu gao
2026-06-20 17:39:32 +08:00
committed by GitHub
parent 807bcd5647
commit 9cc0ebcc66
3 changed files with 4 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/llama.cpp/paraformer/export_paraformer_gguf.py
+2 -1
View File
@@ -32,7 +32,8 @@ def main():
import json, glob
tp = a.tokens or (glob.glob(os.path.join(os.path.dirname(a.model_pt), "tokens.json")) + [None])[0]
if tp and os.path.exists(tp):
toks = json.load(open(tp)); w.add_array("pf.vocab", toks)
with open(tp, encoding="utf-8") as f: toks = json.load(f)
w.add_array("pf.vocab", toks)
print(f"embedded pf.vocab ({len(toks)} tokens) from {tp}")
else:
print("WARNING: tokens.json not found - gguf will have no vocab (binary falls back to ids)")
runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp
+1 -1
View File
@@ -141,7 +141,7 @@ int main(int argc,char**argv){
auto rdf=[&](const char*k,float d){int i=gguf_find_key(gg,k);return i<0?d:gguf_get_val_f32(gg,i);};
m.c.enc_blocks=rdi("pf.enc.num_blocks",50);m.c.dec_blocks=rdi("pf.dec.num_blocks",16);m.c.dec_att=rdi("pf.dec.att_layer_num",16);
m.c.dec3=rdi("pf.dec.decoders3",1);m.c.vocab=rdi("pf.vocab_size",8404);m.c.tail=rdf("pf.predictor.tail_threshold",0.45f);m.c.thresh=rdf("pf.predictor.threshold",1.0f);
std::vector<std::string> vocab; {int ki=gguf_find_key(gg,"pf.vocab"); if(ki>=0){int nv=gguf_get_arr_n(gg,ki); vocab.resize(nv); for(int i=0;i<nv;i++)vocab[i]=gguf_get_arr_str(gg,ki,i);}}
std::vector<std::string> vocab; {int ki=gguf_find_key(gg,"pf.vocab"); if(ki>=0){int nv=gguf_get_arr_n(gg,ki); vocab.resize(nv); for(int i=0;i<nv;i++){const char*s=gguf_get_arr_str(gg,ki,i); vocab[i]=s?s:"";}}}
for(int i=0;i<gguf_get_n_tensors(gg);i++){const char*nm=gguf_get_tensor_name(gg,i);m.t[nm]=ggml_get_tensor(m.ctx_w,nm);}gguf_free(gg);
const int D=m.c.d_model,F=560,V=m.c.vocab;
bool emit_ids = ids_mode || vocab.empty(); // fall back to ids if the gguf has no vocab
runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp
+1 -1
View File
@@ -123,7 +123,7 @@ int main(int argc,char**argv){
m.c.kernel=rd("sv.kernel_size",11); m.c.vocab=rd("sv.vocab_size",25055); m.c.blank=rd("sv.blank_id",0);
int qi=gguf_find_key(gg,"sv.query_tokens"); int nq=qi<0?0:(int)gguf_get_arr_n(gg,qi);
std::vector<int> qtok(nq); for(int i=0;i<nq;i++) qtok[i]=((const int32_t*)gguf_get_arr_data(gg,qi))[i];
std::vector<std::string> vocab; {int ki=gguf_find_key(gg,"sv.vocab"); if(ki>=0){int nv=gguf_get_arr_n(gg,ki); vocab.resize(nv); for(int i=0;i<nv;i++)vocab[i]=gguf_get_arr_str(gg,ki,i);}}
std::vector<std::string> vocab; {int ki=gguf_find_key(gg,"sv.vocab"); if(ki>=0){int nv=gguf_get_arr_n(gg,ki); vocab.resize(nv); for(int i=0;i<nv;i++){const char*s=gguf_get_arr_str(gg,ki,i); vocab[i]=s?s:"";}}}
for(int i=0;i<gguf_get_n_tensors(gg);i++){const char*nm=gguf_get_tensor_name(gg,i);m.t[nm]=ggml_get_tensor(m.ctx_w,nm);}
gguf_free(gg);
const int F=560, D=m.c.d_model, V=m.c.vocab;