diff --git a/runtime/llama.cpp/paraformer/export_paraformer_gguf.py b/runtime/llama.cpp/paraformer/export_paraformer_gguf.py index 8497d503d..8a61aace2 100644 --- a/runtime/llama.cpp/paraformer/export_paraformer_gguf.py +++ b/runtime/llama.cpp/paraformer/export_paraformer_gguf.py @@ -32,7 +32,8 @@ def main(): import json, glob tp = a.tokens or (glob.glob(os.path.join(os.path.dirname(a.model_pt), "tokens.json")) + [None])[0] if tp and os.path.exists(tp): - toks = json.load(open(tp)); w.add_array("pf.vocab", toks) + with open(tp, encoding="utf-8") as f: toks = json.load(f) + w.add_array("pf.vocab", toks) print(f"embedded pf.vocab ({len(toks)} tokens) from {tp}") else: print("WARNING: tokens.json not found - gguf will have no vocab (binary falls back to ids)") diff --git a/runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp b/runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp index 810d5d078..10c0c4d03 100644 --- a/runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp +++ b/runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp @@ -141,7 +141,7 @@ int main(int argc,char**argv){ auto rdf=[&](const char*k,float d){int i=gguf_find_key(gg,k);return i<0?d:gguf_get_val_f32(gg,i);}; m.c.enc_blocks=rdi("pf.enc.num_blocks",50);m.c.dec_blocks=rdi("pf.dec.num_blocks",16);m.c.dec_att=rdi("pf.dec.att_layer_num",16); m.c.dec3=rdi("pf.dec.decoders3",1);m.c.vocab=rdi("pf.vocab_size",8404);m.c.tail=rdf("pf.predictor.tail_threshold",0.45f);m.c.thresh=rdf("pf.predictor.threshold",1.0f); - std::vector vocab; {int ki=gguf_find_key(gg,"pf.vocab"); if(ki>=0){int nv=gguf_get_arr_n(gg,ki); vocab.resize(nv); for(int i=0;i vocab; {int ki=gguf_find_key(gg,"pf.vocab"); if(ki>=0){int nv=gguf_get_arr_n(gg,ki); vocab.resize(nv); for(int i=0;i qtok(nq); for(int i=0;i vocab; {int ki=gguf_find_key(gg,"sv.vocab"); if(ki>=0){int nv=gguf_get_arr_n(gg,ki); vocab.resize(nv); for(int i=0;i vocab; {int ki=gguf_find_key(gg,"sv.vocab"); if(ki>=0){int nv=gguf_get_arr_n(gg,ki); vocab.resize(nv); for(int i=0;i