diff --git a/examples/industrial_data_pretraining/bicif_paraformer/export.py b/examples/industrial_data_pretraining/bicif_paraformer/export.py index 44849b066..e4eb3829d 100644 --- a/examples/industrial_data_pretraining/bicif_paraformer/export.py +++ b/examples/industrial_data_pretraining/bicif_paraformer/export.py @@ -12,7 +12,7 @@ model = AutoModel( device="cpu", ) -res = model.export(type="torchscripts", quantize=False) +res = model.export(type="torchscript", quantize=False) print(res) diff --git a/examples/industrial_data_pretraining/llm_asr/demo_speech2text.sh b/examples/industrial_data_pretraining/llm_asr/demo_speech2text.sh index 57299fc65..3eba6d3c2 100644 --- a/examples/industrial_data_pretraining/llm_asr/demo_speech2text.sh +++ b/examples/industrial_data_pretraining/llm_asr/demo_speech2text.sh @@ -62,4 +62,3 @@ for data_set in "common_voice_zh-CN_speech2text.jsonl" "common_voice_en_speech2t }& done -wait \ No newline at end of file diff --git a/examples/industrial_data_pretraining/paraformer/export.py b/examples/industrial_data_pretraining/paraformer/export.py index a91e9e457..6334e3b8c 100644 --- a/examples/industrial_data_pretraining/paraformer/export.py +++ b/examples/industrial_data_pretraining/paraformer/export.py @@ -13,7 +13,7 @@ model = AutoModel( model="iic/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404", ) -res = model.export(type="torchscripts", quantize=False) +res = model.export(type="torchscript", quantize=False) # res = model.export(type="bladedisc", input=f"{model.model_path}/example/asr_example.wav") print(res) diff --git a/funasr/download/runtime_sdk_download_tool.py b/funasr/download/runtime_sdk_download_tool.py index 96c67355e..0db17c725 100644 --- a/funasr/download/runtime_sdk_download_tool.py +++ b/funasr/download/runtime_sdk_download_tool.py @@ -10,7 +10,7 @@ def main(): parser.add_argument("--model-name", type=str, required=True) parser.add_argument("--export-dir", type=str, required=True) parser.add_argument("--export", type=str2bool, default=True, help="whether to export model") - parser.add_argument("--type", type=str, default="onnx", help='["onnx", "torch"]') + parser.add_argument("--type", type=str, default="onnx", help='["onnx", "torchscript", "bladedisc"]') parser.add_argument("--device", type=str, default="cpu", help='["cpu", "cuda"]') parser.add_argument("--quantize", type=str2bool, default=False, help="export quantized model") parser.add_argument("--fallback-num", type=int, default=0, help="amp fallback number") @@ -37,11 +37,17 @@ def main(): model_file = os.path.join(model_dir, "model.onnx") if args.quantize: model_file = os.path.join(model_dir, "model_quant.onnx") + if args.type == "torchscript": + model_file = os.path.join(model_dir, "model.torchscript") + args.device = "cuda" + elif args.type == "bladedisc": + model_file = os.path.join(model_dir, "model_blade.torchscript") + args.device = "cuda" if not os.path.exists(model_file): - print(".onnx is not exist, begin to export onnx") + print("model is not exist, begin to export " + model_file) from funasr import AutoModel - export_model = AutoModel(model=args.model_name, output_dir=output_dir) + export_model = AutoModel(model=args.model_name, output_dir=output_dir, device=args.device) export_model.export( quantize=args.quantize, type=args.type, diff --git a/funasr/utils/export_utils.py b/funasr/utils/export_utils.py index 72b150f79..a6d079871 100644 --- a/funasr/utils/export_utils.py +++ b/funasr/utils/export_utils.py @@ -23,7 +23,7 @@ def export( export_dir=export_dir, **kwargs, ) - elif type == "torchscripts": + elif type == "torchscript": device = "cuda" if torch.cuda.is_available() else "cpu" print("Exporting torchscripts on device {}".format(device)) _torchscripts(m, path=export_dir, device=device) @@ -100,7 +100,7 @@ def _torchscripts(model, path, device="cuda"): dummy_input = tuple([i.cuda() for i in dummy_input]) model_script = torch.jit.trace(model, dummy_input) - model_script.save(os.path.join(path, f"{model.export_name}.torchscripts")) + model_script.save(os.path.join(path, f"{model.export_name}.torchscript")) def _bladedisc_opt(model, model_inputs, enable_fp16=True): @@ -193,4 +193,4 @@ def _bladedisc_opt_for_encdec(model, path, enable_fp16): model.encoder = _bladedisc_opt(model.encoder, input_data[:2]) model.decoder = _bladedisc_opt(model.decoder, tuple(decoder_inputs)) model_script = torch.jit.trace(model, input_data) - model_script.save(os.path.join(path, f"{model.export_name}_blade.torchscripts")) + model_script.save(os.path.join(path, f"{model.export_name}_blade.torchscript")) diff --git a/runtime/onnxruntime/bin/funasr-onnx-offline-rtf.cpp b/runtime/onnxruntime/bin/funasr-onnx-offline-rtf.cpp index c252bc7a0..d8d94734c 100644 --- a/runtime/onnxruntime/bin/funasr-onnx-offline-rtf.cpp +++ b/runtime/onnxruntime/bin/funasr-onnx-offline-rtf.cpp @@ -52,7 +52,7 @@ void runReg(FUNASR_HANDLE asr_handle, vector wav_list, vector wa std::vector> hotwords_embedding = CompileHotwordEmbedding(asr_handle, nn_hotwords_); // warm up - for (size_t i = 0; i < 10; i++) + for (size_t i = 0; i < 1; i++) { FUNASR_RESULT result=FunOfflineInfer(asr_handle, wav_list[0].c_str(), RASR_NONE, nullptr, hotwords_embedding, audio_fs, true, decoder_handle); if(result){ diff --git a/runtime/onnxruntime/include/com-define.h b/runtime/onnxruntime/include/com-define.h index 77a7b020b..5f71f7b3c 100644 --- a/runtime/onnxruntime/include/com-define.h +++ b/runtime/onnxruntime/include/com-define.h @@ -48,6 +48,7 @@ namespace funasr { #define MODEL_NAME "model.onnx" // hotword embedding compile model #define MODEL_EB_NAME "model_eb.onnx" +#define TORCH_MODEL_EB_NAME "model_eb.torchscript" #define QUANT_MODEL_NAME "model_quant.onnx" #define VAD_CMVN_NAME "am.mvn" #define VAD_CONFIG_NAME "config.yaml" @@ -55,9 +56,9 @@ namespace funasr { // gpu models #define INFER_GPU "gpu" #define BATCHSIZE "batch-size" -#define TORCH_MODEL_NAME "model.torchscripts" -#define TORCH_QUANT_MODEL_NAME "model_quant.torchscripts" -#define BLADE_MODEL_NAME "model.blade.fp16.pt" +#define TORCH_MODEL_NAME "model.torchscript" +#define TORCH_QUANT_MODEL_NAME "model_quant.torchscript" +#define BLADE_MODEL_NAME "model_blade.torchscript" #define BLADEDISC "bladedisc" #define AM_CMVN_NAME "am.mvn" diff --git a/runtime/onnxruntime/src/offline-stream.cpp b/runtime/onnxruntime/src/offline-stream.cpp index 35eb1ba3a..166d3c9ea 100644 --- a/runtime/onnxruntime/src/offline-stream.cpp +++ b/runtime/onnxruntime/src/offline-stream.cpp @@ -33,7 +33,8 @@ OfflineStream::OfflineStream(std::map& model_path, int string am_cmvn_path; string am_config_path; string token_path; - string hw_compile_model_path; + string hw_cpu_model_path; + string hw_gpu_model_path; string seg_dict_path; if(use_gpu){ @@ -50,33 +51,31 @@ OfflineStream::OfflineStream(std::map& model_path, int } bool enable_hotword = false; - hw_compile_model_path = PathAppend(model_path.at(MODEL_DIR), MODEL_EB_NAME); + hw_cpu_model_path = PathAppend(model_path.at(MODEL_DIR), MODEL_EB_NAME); + hw_gpu_model_path = PathAppend(model_path.at(MODEL_DIR), TORCH_MODEL_EB_NAME); seg_dict_path = PathAppend(model_path.at(MODEL_DIR), MODEL_SEG_DICT); - if (access(hw_compile_model_path.c_str(), F_OK) == 0) { // if model_eb.onnx exist, hotword enabled + if (access(hw_cpu_model_path.c_str(), F_OK) == 0) { // if model_eb.onnx exist, hotword enabled enable_hotword = true; - asr_handle->InitHwCompiler(hw_compile_model_path, thread_num); + asr_handle->InitHwCompiler(hw_cpu_model_path, thread_num); asr_handle->InitSegDict(seg_dict_path); } - if (enable_hotword) { - am_model_path = PathAppend(model_path.at(MODEL_DIR), MODEL_NAME); - if(model_path.find(QUANTIZE) != model_path.end() && model_path.at(QUANTIZE) == "true"){ + if (use_gpu && access(hw_gpu_model_path.c_str(), F_OK) == 0) { // if model_eb.torchscript exist, hotword enabled + enable_hotword = true; + asr_handle->InitHwCompiler(hw_gpu_model_path, thread_num); + asr_handle->InitSegDict(seg_dict_path); + } + + am_model_path = PathAppend(model_path.at(MODEL_DIR), MODEL_NAME); + if(model_path.find(QUANTIZE) != model_path.end() && model_path.at(QUANTIZE) == "true"){ am_model_path = PathAppend(model_path.at(MODEL_DIR), QUANT_MODEL_NAME); - } - } else { - am_model_path = PathAppend(model_path.at(MODEL_DIR), MODEL_NAME); - if(model_path.find(QUANTIZE) != model_path.end() && model_path.at(QUANTIZE) == "true"){ - am_model_path = PathAppend(model_path.at(MODEL_DIR), QUANT_MODEL_NAME); - } - if(use_gpu){ + } + if(use_gpu){ am_model_path = PathAppend(model_path.at(MODEL_DIR), TORCH_MODEL_NAME); - if(model_path.find(QUANTIZE) != model_path.end() && model_path.at(QUANTIZE) == "true"){ - am_model_path = PathAppend(model_path.at(MODEL_DIR), TORCH_QUANT_MODEL_NAME); - } if(model_path.find(BLADEDISC) != model_path.end() && model_path.at(BLADEDISC) == "true"){ am_model_path = PathAppend(model_path.at(MODEL_DIR), BLADE_MODEL_NAME); } - } } + am_cmvn_path = PathAppend(model_path.at(MODEL_DIR), AM_CMVN_NAME); am_config_path = PathAppend(model_path.at(MODEL_DIR), AM_CONFIG_NAME); token_path = PathAppend(model_path.at(MODEL_DIR), TOKEN_PATH); diff --git a/runtime/onnxruntime/src/paraformer-torch.cpp b/runtime/onnxruntime/src/paraformer-torch.cpp index a5f719485..466d80a90 100644 --- a/runtime/onnxruntime/src/paraformer-torch.cpp +++ b/runtime/onnxruntime/src/paraformer-torch.cpp @@ -50,6 +50,11 @@ void ParaformerTorch::InitAsr(const std::string &am_model, const std::string &am torch::jit::script::Module model = torch::jit::load(am_model, device); model_ = std::make_shared(std::move(model)); LOG(INFO) << "Successfully load model from " << am_model; + torch::NoGradGuard no_grad; + model_->eval(); + torch::jit::setGraphExecutorOptimize(false); + torch::jit::FusionStrategy static0 = {{torch::jit::FusionBehavior::STATIC, 0}}; + torch::jit::setFusionStrategy(static0); } catch (std::exception const &e) { LOG(ERROR) << "Error when load am model: " << am_model << e.what(); exit(-1); @@ -100,6 +105,27 @@ void ParaformerTorch::LoadConfigFromYaml(const char* filename){ void ParaformerTorch::InitHwCompiler(const std::string &hw_model, int thread_num) { // TODO + torch::DeviceType device = at::kCPU; + #ifdef USE_GPU + if (!torch::cuda::is_available()) { + // LOG(ERROR) << "CUDA is not available! Please check your GPU settings"; + exit(-1); + } else { + // LOG(INFO) << "CUDA is available, running on GPU"; + device = at::kCUDA; + } + #endif + + try { + torch::jit::script::Module model = torch::jit::load(hw_model, device); + hw_model_ = std::make_shared(std::move(model)); + LOG(INFO) << "Successfully load model from " << hw_model; + torch::NoGradGuard no_grad; + hw_model_->eval(); + } catch (std::exception const &e) { + LOG(ERROR) << "Error when load hw model: " << hw_model << e.what(); + exit(-1); + } use_hotword = true; } @@ -111,15 +137,19 @@ ParaformerTorch::~ParaformerTorch() { if(vocab){ delete vocab; + vocab = nullptr; } if(lm_vocab){ delete lm_vocab; + lm_vocab = nullptr; } if(seg_dict){ delete seg_dict; + seg_dict = nullptr; } if(phone_set_){ delete phone_set_; + phone_set_ = nullptr; } } @@ -267,6 +297,9 @@ void ParaformerTorch::LfrCmvn(std::vector> &asr_feats) { std::vector ParaformerTorch::Forward(float** din, int* len, bool input_finished, const std::vector> &hw_emb, void* decoder_handle, int batch_in) { + vector results; + string result=""; + WfstDecoder* wfst_decoder = (WfstDecoder*)decoder_handle; int32_t in_feat_dim = fbank_opts_.mel_opts.num_bins; int32_t feature_dim = lfr_m*in_feat_dim; @@ -295,8 +328,13 @@ std::vector ParaformerTorch::Forward(float** din, int* len, bool in feats_batch.emplace_back(flattened); } - torch::NoGradGuard no_grad; - model_->eval(); + if(max_frames == 0){ + for(int index=0; index all_feats(batch_in * max_frames * feature_dim); for(int index=0; index ParaformerTorch::Forward(float** din, int* len, bool in #endif std::vector inputs = {feats, feat_lens}; - vector results; + std::vector batch_embedding; + std::vector embedding; + try{ + if (use_hotword) { + if(hw_emb.size()<=0){ + LOG(ERROR) << "hw_emb is null"; + for(int index=0; index(hw_emb.size()), static_cast(hw_emb[0].size())}, torch::kFloat).contiguous(); + #ifdef USE_GPU + tensor_hw_emb = tensor_hw_emb.to(at::kCUDA); + #endif + inputs.emplace_back(tensor_hw_emb); + } + }catch (std::exception const &e) + { + LOG(ERROR)<forward(inputs).toTuple()->elements(); torch::Tensor am_scores; torch::Tensor valid_token_lens; @@ -329,28 +411,31 @@ std::vector ParaformerTorch::Forward(float** din, int* len, bool in am_scores = outputs[0].toTensor(); valid_token_lens = outputs[1].toTensor(); #endif + + torch::Tensor us_alphas_tensor; + torch::Tensor us_peaks_tensor; + if(outputs.size() == 4){ + #ifdef USE_GPU + us_alphas_tensor = outputs[2].toTensor().to(at::kCPU); + us_peaks_tensor = outputs[3].toTensor().to(at::kCPU); + #else + us_alphas_tensor = outputs[2].toTensor(); + us_peaks_tensor = outputs[3].toTensor(); + #endif + } + // timestamp for(int index=0; index(); - std::vector us_alphas(paraformer_length[index]); + std::vector us_alphas(paraformer_length[index]*3); for (int i = 0; i < us_alphas.size(); i++) { us_alphas[i] = us_alphas_data[i]; } float* us_peaks_data = us_peaks_tensor[index].data_ptr(); - std::vector us_peaks(paraformer_length[index]); + std::vector us_peaks(paraformer_length[index]*3); for (int i = 0; i < us_peaks.size(); i++) { us_peaks[i] = us_peaks_data[i]; } @@ -387,8 +472,98 @@ std::vector ParaformerTorch::Forward(float** din, int* len, bool in } std::vector> ParaformerTorch::CompileHotwordEmbedding(std::string &hotwords) { - // TODO - std::vector> result(1, std::vector(10, 0.0f)); + int embedding_dim = encoder_size; + std::vector> hw_emb; + if (!use_hotword) { + std::vector vec(embedding_dim, 0); + hw_emb.push_back(vec); + return hw_emb; + } + int max_hotword_len = 10; + std::vector hotword_matrix; + std::vector lengths; + int hotword_size = 1; + int real_hw_size = 0; + if (!hotwords.empty()) { + std::vector hotword_array = split(hotwords, ' '); + hotword_size = hotword_array.size() + 1; + hotword_matrix.reserve(hotword_size * max_hotword_len); + for (auto hotword : hotword_array) { + std::vector chars; + if (EncodeConverter::IsAllChineseCharactor((const U8CHAR_T*)hotword.c_str(), hotword.size())) { + KeepChineseCharacterAndSplit(hotword, chars); + } else { + // for english + std::vector words = split(hotword, ' '); + for (auto word : words) { + std::vector tokens = seg_dict->GetTokensByWord(word); + chars.insert(chars.end(), tokens.begin(), tokens.end()); + } + } + if(chars.size()==0){ + continue; + } + std::vector hw_vector(max_hotword_len, 0); + int vector_len = std::min(max_hotword_len, (int)chars.size()); + int chs_oov = false; + for (int i=0; iString2Id(chars[i]); + if(hw_vector[i] == -1){ + chs_oov = true; + break; + } + } + if(chs_oov){ + LOG(INFO) << "OOV: " << hotword; + continue; + } + LOG(INFO) << hotword; + lengths.push_back(vector_len); + real_hw_size += 1; + hotword_matrix.insert(hotword_matrix.end(), hw_vector.begin(), hw_vector.end()); + } + hotword_size = real_hw_size + 1; + } + std::vector blank_vec(max_hotword_len, 0); + blank_vec[0] = 1; + hotword_matrix.insert(hotword_matrix.end(), blank_vec.begin(), blank_vec.end()); + lengths.push_back(1); + + torch::Tensor feats = + torch::from_blob(hotword_matrix.data(), + {hotword_size, max_hotword_len}, torch::kInt32).contiguous(); + + // 2. forward + #ifdef USE_GPU + feats = feats.to(at::kCUDA); + #endif + std::vector inputs = {feats}; + std::vector> result; + try { + auto output = hw_model_->forward(inputs); + torch::Tensor emb_tensor; + #ifdef USE_GPU + emb_tensor = output.toTensor().to(at::kCPU); + #else + emb_tensor = output.toTensor(); + #endif + assert(emb_tensor.size(0) == max_hotword_len); + assert(emb_tensor.size(1) == hotword_size); + embedding_dim = emb_tensor.size(2); + + float* floatData = emb_tensor.data_ptr(); + for (int j = 0; j < hotword_size; j++) + { + int start_pos = hotword_size * (lengths[j] - 1) * embedding_dim + j * embedding_dim; + std::vector embedding; + embedding.insert(embedding.begin(), floatData + start_pos, floatData + start_pos + embedding_dim); + result.push_back(embedding); + } + } + catch (std::exception const &e) + { + LOG(ERROR)< model_ = nullptr; + std::shared_ptr hw_model_ = nullptr; std::vector encoder_outs_; bool use_hotword; diff --git a/runtime/onnxruntime/src/util.cpp b/runtime/onnxruntime/src/util.cpp index a12570b86..483795e7a 100644 --- a/runtime/onnxruntime/src/util.cpp +++ b/runtime/onnxruntime/src/util.cpp @@ -870,6 +870,15 @@ void TimestampOnnx( std::vector& us_alphas, sum -=(1.0 - 1e-4); } } + // fix case: sum > 1 + int cif_idx = cif_peak.size()-1; + while(sum>=1.0 - 1e-4 && cif_idx >= 0 ){ + if(cif_peak[cif_idx] < 1.0 - 1e-4){ + cif_peak[cif_idx] = sum; + sum -=(1.0 - 1e-4); + } + cif_idx--; + } fire_place.clear(); for (int i = 0; i < num_frames; i++) { diff --git a/runtime/python/libtorch/README.md b/runtime/python/libtorch/README.md index a96846e66..1d15d2b28 100644 --- a/runtime/python/libtorch/README.md +++ b/runtime/python/libtorch/README.md @@ -41,7 +41,7 @@ pip install -e ./ ## Run the demo -- Model_dir: the model path, which contains `model.torchscripts`, `config.yaml`, `am.mvn`. +- Model_dir: the model path, which contains `model.torchscript`, `config.yaml`, `am.mvn`. - Input: wav formt file, support formats: `str, np.ndarray, List[str]` - Output: `List[str]`: recognition result. - Example: diff --git a/runtime/python/libtorch/funasr_torch/paraformer_bin.py b/runtime/python/libtorch/funasr_torch/paraformer_bin.py index 5fa3cc969..16c040678 100644 --- a/runtime/python/libtorch/funasr_torch/paraformer_bin.py +++ b/runtime/python/libtorch/funasr_torch/paraformer_bin.py @@ -46,11 +46,11 @@ class Paraformer: model_dir ) - model_file = os.path.join(model_dir, "model.torchscripts") + model_file = os.path.join(model_dir, "model.torchscript") if quantize: - model_file = os.path.join(model_dir, "model_quant.torchscripts") + model_file = os.path.join(model_dir, "model_quant.torchscript") if not os.path.exists(model_file): - print(".torchscripts does not exist, begin to export torchscripts") + print(".torchscripts does not exist, begin to export torchscript") try: from funasr import AutoModel except: @@ -268,11 +268,11 @@ class ContextualParaformer(Paraformer): ) if quantize: - model_bb_file = os.path.join(model_dir, "model_bb_quant.torchscripts") - model_eb_file = os.path.join(model_dir, "model_eb_quant.torchscripts") + model_bb_file = os.path.join(model_dir, "model_bb_quant.torchscript") + model_eb_file = os.path.join(model_dir, "model_eb_quant.torchscript") else: - model_bb_file = os.path.join(model_dir, "model_bb.torchscripts") - model_eb_file = os.path.join(model_dir, "model_eb.torchscripts") + model_bb_file = os.path.join(model_dir, "model_bb.torchscript") + model_eb_file = os.path.join(model_dir, "model_eb.torchscript") if not (os.path.exists(model_eb_file) and os.path.exists(model_bb_file)): print(".onnx does not exist, begin to export onnx") @@ -282,7 +282,7 @@ class ContextualParaformer(Paraformer): raise "You are exporting onnx, please install funasr and try it again. To install funasr, you could:\n" "\npip3 install -U funasr\n" "For the users in China, you could install with the command:\n" "\npip3 install -U funasr -i https://mirror.sjtu.edu.cn/pypi/web/simple" model = AutoModel(model=model_dir) - model_dir = model.export(type="torchscripts", quantize=quantize, **kwargs) + model_dir = model.export(type="torchscript", quantize=quantize, **kwargs) config_file = os.path.join(model_dir, "config.yaml") cmvn_file = os.path.join(model_dir, "am.mvn") diff --git a/runtime/websocket/bin/funasr-wss-server.cpp b/runtime/websocket/bin/funasr-wss-server.cpp index 0d475da8d..3c5b81cff 100644 --- a/runtime/websocket/bin/funasr-wss-server.cpp +++ b/runtime/websocket/bin/funasr-wss-server.cpp @@ -45,7 +45,7 @@ int main(int argc, char* argv[]) { false, "/workspace/models", "string"); TCLAP::ValueArg model_dir( "", MODEL_DIR, - "default: /workspace/models/asr, the asr model path, which contains model_quant.onnx, config.yaml, am.mvn", + "default: /workspace/models/asr, the asr model path, which contains *.onnx/*.torchscript, config.yaml, am.mvn", false, "/workspace/models/asr", "string"); TCLAP::ValueArg model_revision( "", "model-revision", @@ -67,7 +67,7 @@ int main(int argc, char* argv[]) { TCLAP::ValueArg vad_revision( "", "vad-revision", "VAD model revision", - false, "v2.0.4", "string"); + false, "v2.0.6", "string"); TCLAP::ValueArg vad_quant( "", VAD_QUANT, "true (Default), load the model of model_quant.onnx in vad_dir. If set " @@ -198,8 +198,9 @@ int main(int argc, char* argv[]) { std::string s_punc_quant = model_path[PUNC_QUANT]; std::string s_itn_path = model_path[ITN_DIR]; std::string s_lm_path = model_path[LM_DIR]; + std::string s_blade = model_path[BLADEDISC]; - std::string python_cmd = "python -m funasr.download.runtime_sdk_download_tool --type onnx "; + std::string python_cmd = "python -m funasr.download.runtime_sdk_download_tool "; if(vad_dir.isSet() && !s_vad_path.empty()){ std::string python_cmd_vad; @@ -208,12 +209,12 @@ int main(int argc, char* argv[]) { if (access(s_vad_path.c_str(), F_OK) == 0){ // local - python_cmd_vad = python_cmd + " --quantize " + s_vad_quant + " --model-name " + s_vad_path + " --export-dir ./ " + " --model_revision " + model_path["vad-revision"]; + python_cmd_vad = python_cmd + " --type onnx " + " --quantize " + s_vad_quant + " --model-name " + s_vad_path + " --export-dir ./ " + " --model_revision " + model_path["vad-revision"]; down_vad_path = s_vad_path; }else{ // modelscope LOG(INFO) << "Download model: " << s_vad_path << " from modelscope: "; - python_cmd_vad = python_cmd + " --quantize " + s_vad_quant + " --model-name " + s_vad_path + " --export-dir " + s_download_model_dir + " --model_revision " + model_path["vad-revision"]; + python_cmd_vad = python_cmd + " --type onnx " + " --quantize " + s_vad_quant + " --model-name " + s_vad_path + " --export-dir " + s_download_model_dir + " --model_revision " + model_path["vad-revision"]; down_vad_path = s_download_model_dir+"/"+s_vad_path; } @@ -241,6 +242,7 @@ int main(int argc, char* argv[]) { std::string python_cmd_asr; std::string down_asr_path; std::string down_asr_model; + std::string model_type = "onnx"; // modify model-revision by model name size_t found = s_asr_path.find("speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404"); @@ -260,26 +262,41 @@ int main(int argc, char* argv[]) { s_lm_path=""; } + if (use_gpu_){ + model_type = "torchscript"; + if (s_blade=="true" || s_blade=="True" || s_blade=="TRUE"){ + model_type = "bladedisc"; + } + } + if (access(s_asr_path.c_str(), F_OK) == 0){ // local - python_cmd_asr = python_cmd + " --quantize " + s_asr_quant + " --model-name " + s_asr_path + " --export-dir ./ " + " --model_revision " + model_path["model-revision"]; + python_cmd_asr = python_cmd + " --type " + model_type + " --quantize " + s_asr_quant + " --model-name " + s_asr_path + " --export-dir ./ " + " --model_revision " + model_path["model-revision"]; down_asr_path = s_asr_path; }else{ // modelscope LOG(INFO) << "Download model: " << s_asr_path << " from modelscope: "; - python_cmd_asr = python_cmd + " --quantize " + s_asr_quant + " --model-name " + s_asr_path + " --export-dir " + s_download_model_dir + " --model_revision " + model_path["model-revision"]; + python_cmd_asr = python_cmd + " --type " + model_type + " --quantize " + s_asr_quant + " --model-name " + s_asr_path + " --export-dir " + s_download_model_dir + " --model_revision " + model_path["model-revision"]; down_asr_path = s_download_model_dir+"/"+s_asr_path; } - - int ret = system(python_cmd_asr.c_str()); - if(ret !=0){ - LOG(INFO) << "Failed to download model from modelscope. If you set local asr model path, you can ignore the errors."; - } + down_asr_model = down_asr_path+"/model_quant.onnx"; if(s_asr_quant=="false" || s_asr_quant=="False" || s_asr_quant=="FALSE"){ down_asr_model = down_asr_path+"/model.onnx"; } + if (use_gpu_){ + down_asr_model = down_asr_path+"/model.torchscript"; + if (s_blade=="true" || s_blade=="True" || s_blade=="TRUE"){ + down_asr_model = down_asr_path+"/model_blade.torchscript"; + } + } + + int ret = system(python_cmd_asr.c_str()); + if(ret !=0){ + LOG(INFO) << "Failed to download model from modelscope. If you set local asr model path, you can ignore the errors."; + } + if (access(down_asr_model.c_str(), F_OK) != 0){ LOG(ERROR) << down_asr_model << " do not exists."; exit(-1); @@ -298,7 +315,7 @@ int main(int argc, char* argv[]) { if (access(s_itn_path.c_str(), F_OK) == 0) { // local - python_cmd_itn = python_cmd + " --model-name " + s_itn_path + + python_cmd_itn = python_cmd + " --type onnx " + " --model-name " + s_itn_path + " --export-dir ./ " + " --model_revision " + model_path["itn-revision"] + " --export False "; down_itn_path = s_itn_path; @@ -306,7 +323,7 @@ int main(int argc, char* argv[]) { // modelscope LOG(INFO) << "Download model: " << s_itn_path << " from modelscope : "; - python_cmd_itn = python_cmd + " --model-name " + + python_cmd_itn = python_cmd + " --type onnx " + " --model-name " + s_itn_path + " --export-dir " + s_download_model_dir + " --model_revision " + model_path["itn-revision"] @@ -340,7 +357,7 @@ int main(int argc, char* argv[]) { if (access(s_lm_path.c_str(), F_OK) == 0) { // local - python_cmd_lm = python_cmd + "--quantize " + s_punc_quant + " --model-name " + s_lm_path + + python_cmd_lm = python_cmd + " --type onnx " + " --model-name " + s_lm_path + " --export-dir ./ " + " --model_revision " + model_path["lm-revision"] + " --export False "; down_lm_path = s_lm_path; @@ -348,7 +365,7 @@ int main(int argc, char* argv[]) { // modelscope LOG(INFO) << "Download model: " << s_lm_path << " from modelscope : "; - python_cmd_lm = python_cmd + " --quantize " + s_punc_quant + " --model-name " + + python_cmd_lm = python_cmd + " --type onnx " + " --model-name " + s_lm_path + " --export-dir " + s_download_model_dir + " --model_revision " + model_path["lm-revision"] @@ -383,12 +400,12 @@ int main(int argc, char* argv[]) { if (access(s_punc_path.c_str(), F_OK) == 0){ // local - python_cmd_punc = python_cmd + " --model-name " + s_punc_path + " --export-dir ./ " + " --model_revision " + model_path["punc-revision"]; + python_cmd_punc = python_cmd + " --type onnx " + "--quantize " + s_punc_quant + " --model-name " + s_punc_path + " --export-dir ./ " + " --model_revision " + model_path["punc-revision"]; down_punc_path = s_punc_path; }else{ // modelscope LOG(INFO) << "Download model: " << s_punc_path << " from modelscope: "; - python_cmd_punc = python_cmd + " --model-name " + s_punc_path + " --export-dir " + s_download_model_dir + " --model_revision " + model_path["punc-revision"]; + python_cmd_punc = python_cmd + " --type onnx " + "--quantize " + s_punc_quant + " --model-name " + s_punc_path + " --export-dir " + s_download_model_dir + " --model_revision " + model_path["punc-revision"]; down_punc_path = s_download_model_dir+"/"+s_punc_path; }