modelscope/FunASR
1
0
Fork 0
mirror of https://github.com/modelscope/FunASR.git synced 2026-10-12 02:19:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

support SenseVoiceSmall in 2pass mode

Browse Source
This commit is contained in:
雾聪
2024-10-29 11:40:18 +08:00
parent 2a296ab511
commit 1819303f5e
16 changed files with 373 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files
runtime/onnxruntime/include/funasrruntime.h
+2 -1
View File
@@ -120,7 +120,8 @@ _FUNASRAPI FUNASR_HANDLE FunTpassOnlineInit(FUNASR_HANDLE tpass_handle, std::
_FUNASRAPI FUNASR_RESULT FunTpassInferBuffer(FUNASR_HANDLE handle, FUNASR_HANDLE online_handle, const char* sz_buf,
int n_len, std::vector<std::vector<std::string>> &punc_cache, bool input_finished=true,
int sampling_rate=16000, std::string wav_format="pcm", ASR_TYPE mode=ASR_TWO_PASS,
const std::vector<std::vector<float>> &hw_emb={{0.0}}, bool itn=true, FUNASR_DEC_HANDLE dec_handle=nullptr);
const std::vector<std::vector<float>> &hw_emb={{0.0}}, bool itn=true, FUNASR_DEC_HANDLE dec_handle=nullptr,
std::string svs_lang="auto", bool svs_itn=true);
_FUNASRAPI void FunTpassUninit(FUNASR_HANDLE handle);
_FUNASRAPI void FunTpassOnlineUninit(FUNASR_HANDLE handle);
runtime/onnxruntime/include/model.h
+3 -1
View File
@@ -16,9 +16,11 @@ class Model {
virtual void StartUtterance() = 0;
virtual void EndUtterance() = 0;
virtual void Reset() = 0;
virtual string GreedySearch(float* in, int n_len, int64_t token_nums, bool is_stamp=false, std::vector<float> us_alphas={0}, std::vector<float> us_cif_peak={0}){return "";};
virtual void InitAsr(const std::string &am_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num){};
virtual void InitAsr(const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num){};
virtual void InitAsr(const std::string &am_model, const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num){};
virtual void InitAsr(const std::string &am_model, const std::string &en_model, const std::string &de_model, const std::string &am_cmvn,
const std::string &am_config, const std::string &token_file, const std::string &online_token_file, int thread_num){};
virtual void InitLm(const std::string &lm_file, const std::string &lm_config, const std::string &lex_file){};
virtual void InitFstDecoder(){};
virtual std::string Forward(float *din, int len, bool input_finished, const std::vector<std::vector<float>> &hw_emb={{0.0}}, void* wfst_decoder=nullptr){return "";};
runtime/onnxruntime/include/tpass-stream.h
+2
View File
@@ -26,11 +26,13 @@ class TpassStream {
bool UseVad(){return use_vad;};
bool UsePunc(){return use_punc;};
bool UseITN(){return use_itn;};
std::string GetModelType(){return model_type;};
private:
bool use_vad=false;
bool use_punc=false;
bool use_itn=false;
std::string model_type = MODEL_PARA;
};
TpassStream *CreateTpassStream(std::map<std::string, std::string>& model_path, int thread_num=1);
runtime/onnxruntime/src/funasrruntime.cpp
+28 -17
View File
@@ -482,7 +482,8 @@
_FUNASRAPI FUNASR_RESULT FunTpassInferBuffer(FUNASR_HANDLE handle, FUNASR_HANDLE online_handle, const char* sz_buf,
int n_len, std::vector<std::vector<std::string>> &punc_cache, bool input_finished,
int sampling_rate, std::string wav_format, ASR_TYPE mode,
const std::vector<std::vector<float>> &hw_emb, bool itn, FUNASR_DEC_HANDLE dec_handle)
const std::vector<std::vector<float>> &hw_emb, bool itn, FUNASR_DEC_HANDLE dec_handle,
std::string svs_lang, bool svs_itn)
{
funasr::TpassStream* tpass_stream = (funasr::TpassStream*)handle;
funasr::TpassOnlineStream* tpass_online_stream = (funasr::TpassOnlineStream*)online_handle;
@@ -525,7 +526,7 @@
funasr::AudioFrame* frame = nullptr;
while(audio->FetchChunck(frame) > 0){
string msg = ((funasr::ParaformerOnline*)asr_online_handle)->Forward(frame->data, frame->len, frame->is_final);
string msg = (asr_online_handle)->Forward(frame->data, frame->len, frame->is_final);
if(mode == ASR_ONLINE){
((funasr::ParaformerOnline*)asr_online_handle)->online_res += msg;
if(frame->is_final){
@@ -567,7 +568,12 @@
len = new int[1];
buff[0] = frame->data;
len[0] = frame->len;
vector<string> msgs = ((funasr::Paraformer*)asr_handle)->Forward(buff, len, frame->is_final, hw_emb, dec_handle);
vector<string> msgs;
if(tpass_stream->GetModelType() == MODEL_SVS){
msgs = (tpass_stream->asr_handle)->Forward(buff, len, true, svs_lang, svs_itn, 1);
}else{
msgs = (tpass_stream->asr_handle)->Forward(buff, len, true, hw_emb, dec_handle, 1);
}
string msg = msgs.size()>0?msgs[0]:"";
std::vector<std::string> msg_vec = funasr::SplitStr(msg, " | "); // split with timestamp
if(msg_vec.size()==0){
@@ -589,24 +595,29 @@
p_result->stamp += cur_stamp + "]";
}
string msg_punc = punc_online_handle->AddPunc(msg.c_str(), punc_cache[1]);
if(input_finished){
msg_punc += "。";
}
p_result->tpass_msg = msg_punc;
if (tpass_stream->GetModelType() == MODEL_PARA){
string msg_punc = punc_online_handle->AddPunc(msg.c_str(), punc_cache[1]);
if(input_finished){
msg_punc += "。";
}
p_result->tpass_msg = msg_punc;
#if !defined(__APPLE__)
if(tpass_stream->UseITN() && itn){
string msg_itn = tpass_stream->itn_handle->Normalize(msg_punc);
// TimestampSmooth
if(!(p_result->stamp).empty()){
std::string new_stamp = funasr::TimestampSmooth(p_result->tpass_msg, msg_itn, p_result->stamp);
if(!new_stamp.empty()){
p_result->stamp = new_stamp;
if(tpass_stream->UseITN() && itn){
string msg_itn = tpass_stream->itn_handle->Normalize(msg_punc);
// TimestampSmooth
if(!(p_result->stamp).empty()){
std::string new_stamp = funasr::TimestampSmooth(p_result->tpass_msg, msg_itn, p_result->stamp);
if(!new_stamp.empty()){
p_result->stamp = new_stamp;
}
}
p_result->tpass_msg = msg_itn;
}
p_result->tpass_msg = msg_itn;
}
#endif
}else{
p_result->tpass_msg = msg;
}
if (!(p_result->stamp).empty()){
p_result->stamp_sents = funasr::TimestampSentence(p_result->tpass_msg, p_result->stamp);
}
runtime/onnxruntime/src/paraformer-online.cpp
+77 -29
View File
@@ -9,18 +9,55 @@ using namespace std;
namespace funasr {
ParaformerOnline::ParaformerOnline(Paraformer* para_handle, std::vector<int> chunk_size)
:para_handle_(std::move(para_handle)),chunk_size(chunk_size),session_options_{}{
InitOnline(
para_handle_->fbank_opts_,
para_handle_->encoder_session_,
para_handle_->decoder_session_,
para_handle_->en_szInputNames_,
para_handle_->en_szOutputNames_,
para_handle_->de_szInputNames_,
para_handle_->de_szOutputNames_,
para_handle_->means_list_,
para_handle_->vars_list_);
ParaformerOnline::ParaformerOnline(Model* offline_handle, std::vector<int> chunk_size, std::string model_type)
:offline_handle_(std::move(offline_handle)),chunk_size(chunk_size),session_options_{}{
if(model_type == MODEL_PARA){
Paraformer* para_handle = dynamic_cast<Paraformer*>(offline_handle_);
InitOnline(
para_handle->fbank_opts_,
para_handle->encoder_session_,
para_handle->decoder_session_,
para_handle->en_szInputNames_,
para_handle->en_szOutputNames_,
para_handle->de_szInputNames_,
para_handle->de_szOutputNames_,
para_handle->means_list_,
para_handle->vars_list_,
para_handle->frame_length,
para_handle->frame_shift,
para_handle->n_mels,
para_handle->lfr_m,
para_handle->lfr_n,
para_handle->encoder_size,
para_handle->fsmn_layers,
para_handle->fsmn_lorder,
para_handle->fsmn_dims,
para_handle->cif_threshold,
para_handle->tail_alphas);
}else if(model_type == MODEL_SVS){
SenseVoiceSmall* svs_handle = dynamic_cast<SenseVoiceSmall*>(offline_handle_);
InitOnline(
svs_handle->fbank_opts_,
svs_handle->encoder_session_,
svs_handle->decoder_session_,
svs_handle->en_szInputNames_,
svs_handle->en_szOutputNames_,
svs_handle->de_szInputNames_,
svs_handle->de_szOutputNames_,
svs_handle->means_list_,
svs_handle->vars_list_,
svs_handle->frame_length,
svs_handle->frame_shift,
svs_handle->n_mels,
svs_handle->lfr_m,
svs_handle->lfr_n,
svs_handle->encoder_size,
svs_handle->fsmn_layers,
svs_handle->fsmn_lorder,
svs_handle->fsmn_dims,
svs_handle->cif_threshold,
svs_handle->tail_alphas);
}
InitCache();
}
@@ -33,7 +70,18 @@ void ParaformerOnline::InitOnline(
vector<const char*> &de_szInputNames,
vector<const char*> &de_szOutputNames,
vector<float> &means_list,
vector<float> &vars_list){
vector<float> &vars_list,
int frame_length_,
int frame_shift_,
int n_mels_,
int lfr_m_,
int lfr_n_,
int encoder_size_,
int fsmn_layers_,
int fsmn_lorder_,
int fsmn_dims_,
float cif_threshold_,
float tail_alphas_){
fbank_opts_ = fbank_opts;
encoder_session_ = encoder_session;
decoder_session_ = decoder_session;
@@ -44,27 +92,27 @@ void ParaformerOnline::InitOnline(
means_list_ = means_list;
vars_list_ = vars_list;
frame_length = para_handle_->frame_length;
frame_shift = para_handle_->frame_shift;
n_mels = para_handle_->n_mels;
lfr_m = para_handle_->lfr_m;
lfr_n = para_handle_->lfr_n;
encoder_size = para_handle_->encoder_size;
fsmn_layers = para_handle_->fsmn_layers;
fsmn_lorder = para_handle_->fsmn_lorder;
fsmn_dims = para_handle_->fsmn_dims;
cif_threshold = para_handle_->cif_threshold;
tail_alphas = para_handle_->tail_alphas;
frame_length = frame_length_;
frame_shift = frame_shift_;
n_mels = n_mels_;
lfr_m = lfr_m_;
lfr_n = lfr_n_;
encoder_size = encoder_size_;
fsmn_layers = fsmn_layers_;
fsmn_lorder = fsmn_lorder_;
fsmn_dims = fsmn_dims_;
cif_threshold = cif_threshold_;
tail_alphas = tail_alphas_;
// other vars
sqrt_factor = std::sqrt(encoder_size);
for(int i=0; i<fsmn_lorder*fsmn_dims; i++){
fsmn_init_cache_.emplace_back(0);
}
chunk_len = chunk_size[1]*frame_shift*lfr_n*para_handle_->asr_sample_rate/1000;
chunk_len = chunk_size[1]*frame_shift*lfr_n*offline_handle_->GetAsrSampleRate()/1000;
frame_sample_length_ = para_handle_->asr_sample_rate / 1000 * frame_length;
frame_shift_sample_length_ = para_handle_->asr_sample_rate / 1000 * frame_shift;
frame_sample_length_ = offline_handle_->GetAsrSampleRate() / 1000 * frame_length;
frame_shift_sample_length_ = offline_handle_->GetAsrSampleRate() / 1000 * frame_shift;
}
@@ -464,7 +512,7 @@ string ParaformerOnline::ForwardChunk(std::vector<std::vector<float>> &chunk_fea
std::vector<int64_t> decoder_shape = decoder_tensor[0].GetTensorTypeAndShapeInfo().GetShape();
float* float_data = decoder_tensor[0].GetTensorMutableData<float>();
result = para_handle_->GreedySearch(float_data, list_frame.size(), decoder_shape[2]);
result = offline_handle_->GreedySearch(float_data, list_frame.size(), decoder_shape[2]);
}
}catch (std::exception const &e)
{
@@ -493,7 +541,7 @@ string ParaformerOnline::Forward(float* din, int len, bool input_finished, const
if(is_first_chunk){
is_first_chunk = false;
}
ExtractFeats(para_handle_->asr_sample_rate, wav_feats, waves, input_finished);
ExtractFeats(offline_handle_->GetAsrSampleRate(), wav_feats, waves, input_finished);
if(wav_feats.size() == 0){
return result;
}
runtime/onnxruntime/src/paraformer-online.h
+17 -6
View File
@@ -38,7 +38,18 @@ namespace funasr {
vector<const char*> &de_szInputNames,
vector<const char*> &de_szOutputNames,
vector<float> &means_list,
vector<float> &vars_list);
vector<float> &vars_list,
int frame_length_,
int frame_shift_,
int n_mels_,
int lfr_m_,
int lfr_n_,
int encoder_size_,
int fsmn_layers_,
int fsmn_lorder_,
int fsmn_dims_,
float cif_threshold_,
float tail_alphas_);
void StartUtterance()
{
@@ -48,8 +59,8 @@ namespace funasr {
{
}
Paraformer* para_handle_ = nullptr;
// from para_handle_
Model* offline_handle_ = nullptr;
// from offline_handle_
knf::FbankOptions fbank_opts_;
std::shared_ptr<Ort::Session> encoder_session_ = nullptr;
std::shared_ptr<Ort::Session> decoder_session_ = nullptr;
@@ -60,7 +71,7 @@ namespace funasr {
vector<const char*> de_szOutputNames_;
vector<float> means_list_;
vector<float> vars_list_;
// configs from para_handle_
// configs from offline_handle_
int frame_length = 25;
int frame_shift = 10;
int n_mels = 80;
@@ -100,7 +111,7 @@ namespace funasr {
double sqrt_factor;
public:
ParaformerOnline(Paraformer* para_handle, std::vector<int> chunk_size);
ParaformerOnline(Model* offline_handle, std::vector<int> chunk_size, std::string model_type=MODEL_PARA);
~ParaformerOnline();
void Reset();
void ResetCache();
@@ -112,7 +123,7 @@ namespace funasr {
string Forward(float* din, int len, bool input_finished, const std::vector<std::vector<float>> &hw_emb={{0.0}}, void* wfst_decoder=nullptr);
string Rescoring();
int GetAsrSampleRate() { return para_handle_->asr_sample_rate; };
int GetAsrSampleRate() { return offline_handle_->GetAsrSampleRate(); };
// 2pass
std::string online_res;
runtime/onnxruntime/src/paraformer.cpp
+3 -24
View File
@@ -131,9 +131,10 @@ void Paraformer::InitAsr(const std::string &en_model, const std::string &de_mode
}
// 2pass
void Paraformer::InitAsr(const std::string &am_model, const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num){
void Paraformer::InitAsr(const std::string &am_model, const std::string &en_model, const std::string &de_model,
const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, const std::string &online_token_file, int thread_num){
// online
InitAsr(en_model, de_model, am_cmvn, am_config, token_file, thread_num);
InitAsr(en_model, de_model, am_cmvn, am_config, online_token_file, thread_num);
// offline
try {
@@ -144,28 +145,6 @@ void Paraformer::InitAsr(const std::string &am_model, const std::string &en_mode
exit(-1);
}
// string strName;
// GetInputName(m_session_.get(), strName);
// m_strInputNames.push_back(strName.c_str());
// GetInputName(m_session_.get(), strName,1);
// m_strInputNames.push_back(strName);
// if (use_hotword) {
// GetInputName(m_session_.get(), strName, 2);
// m_strInputNames.push_back(strName);
// }
// // support time stamp
// size_t numOutputNodes = m_session_->GetOutputCount();
// for(int index=0; index<numOutputNodes; index++){
// GetOutputName(m_session_.get(), strName, index);
// m_strOutputNames.push_back(strName);
// }
// for (auto& item : m_strInputNames)
// m_szInputNames.push_back(item.c_str());
// for (auto& item : m_strOutputNames)
// m_szOutputNames.push_back(item.c_str());
GetInputNames(m_session_.get(), m_strInputNames, m_szInputNames);
GetOutputNames(m_session_.get(), m_strOutputNames, m_szOutputNames);
}
runtime/onnxruntime/src/paraformer.h
+2 -1
View File
@@ -46,7 +46,8 @@ namespace funasr {
// online
void InitAsr(const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num);
// 2pass
void InitAsr(const std::string &am_model, const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num);
void InitAsr(const std::string &am_model, const std::string &en_model, const std::string &de_model, const std::string &am_cmvn,
const std::string &am_config, const std::string &token_file, const std::string &online_token_file, int thread_num);
void InitHwCompiler(const std::string &hw_model, int thread_num);
void InitSegDict(const std::string &seg_dict_model);
std::vector<std::vector<float>> CompileHotwordEmbedding(std::string &hotwords);
runtime/onnxruntime/src/sensevoice-small.cpp
+166
View File
@@ -48,6 +48,145 @@ void SenseVoiceSmall::InitAsr(const std::string &am_model, const std::string &am
LoadCmvn(am_cmvn.c_str());
}
// online
void SenseVoiceSmall::InitAsr(const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num){
LoadOnlineConfigFromYaml(am_config.c_str());
// knf options
fbank_opts_.frame_opts.dither = 0;
fbank_opts_.mel_opts.num_bins = n_mels;
fbank_opts_.frame_opts.samp_freq = asr_sample_rate;
fbank_opts_.frame_opts.window_type = window_type;
fbank_opts_.frame_opts.frame_shift_ms = frame_shift;
fbank_opts_.frame_opts.frame_length_ms = frame_length;
fbank_opts_.energy_floor = 0;
fbank_opts_.mel_opts.debug_mel = false;
// session_options_.SetInterOpNumThreads(1);
session_options_.SetIntraOpNumThreads(thread_num);
session_options_.SetGraphOptimizationLevel(ORT_ENABLE_ALL);
// DisableCpuMemArena can improve performance
session_options_.DisableCpuMemArena();
try {
encoder_session_ = std::make_unique<Ort::Session>(env_, ORTSTRING(en_model).c_str(), session_options_);
LOG(INFO) << "Successfully load model from " << en_model;
} catch (std::exception const &e) {
LOG(ERROR) << "Error when load am encoder model: " << e.what();
exit(-1);
}
try {
decoder_session_ = std::make_unique<Ort::Session>(env_, ORTSTRING(de_model).c_str(), session_options_);
LOG(INFO) << "Successfully load model from " << de_model;
} catch (std::exception const &e) {
LOG(ERROR) << "Error when load am decoder model: " << e.what();
exit(-1);
}
// encoder
string strName;
GetInputName(encoder_session_.get(), strName);
en_strInputNames.push_back(strName.c_str());
GetInputName(encoder_session_.get(), strName,1);
en_strInputNames.push_back(strName);
GetOutputName(encoder_session_.get(), strName);
en_strOutputNames.push_back(strName);
GetOutputName(encoder_session_.get(), strName,1);
en_strOutputNames.push_back(strName);
GetOutputName(encoder_session_.get(), strName,2);
en_strOutputNames.push_back(strName);
for (auto& item : en_strInputNames)
en_szInputNames_.push_back(item.c_str());
for (auto& item : en_strOutputNames)
en_szOutputNames_.push_back(item.c_str());
// decoder
int de_input_len = 4 + fsmn_layers;
int de_out_len = 2 + fsmn_layers;
for(int i=0;i<de_input_len; i++){
GetInputName(decoder_session_.get(), strName, i);
de_strInputNames.push_back(strName.c_str());
}
for(int i=0;i<de_out_len; i++){
GetOutputName(decoder_session_.get(), strName,i);
de_strOutputNames.push_back(strName);
}
for (auto& item : de_strInputNames)
de_szInputNames_.push_back(item.c_str());
for (auto& item : de_strOutputNames)
de_szOutputNames_.push_back(item.c_str());
online_vocab = new Vocab(token_file.c_str());
phone_set_ = new PhoneSet(token_file.c_str());
LoadCmvn(am_cmvn.c_str());
}
// 2pass
void SenseVoiceSmall::InitAsr(const std::string &am_model, const std::string &en_model, const std::string &de_model,
const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, const std::string &online_token_file, int thread_num){
// online
InitAsr(en_model, de_model, am_cmvn, am_config, online_token_file, thread_num);
// offline
try {
m_session_ = std::make_unique<Ort::Session>(env_, ORTSTRING(am_model).c_str(), session_options_);
LOG(INFO) << "Successfully load model from " << am_model;
} catch (std::exception const &e) {
LOG(ERROR) << "Error when load am onnx model: " << e.what();
exit(-1);
}
GetInputNames(m_session_.get(), m_strInputNames, m_szInputNames);
GetOutputNames(m_session_.get(), m_strOutputNames, m_szOutputNames);
vocab = new Vocab(token_file.c_str());
}
void SenseVoiceSmall::LoadOnlineConfigFromYaml(const char* filename){
YAML::Node config;
try{
config = YAML::LoadFile(filename);
}catch(exception const &e){
LOG(ERROR) << "Error loading file, yaml file error or not exist.";
exit(-1);
}
try{
YAML::Node frontend_conf = config["frontend_conf"];
YAML::Node encoder_conf = config["encoder_conf"];
YAML::Node decoder_conf = config["decoder_conf"];
YAML::Node predictor_conf = config["predictor_conf"];
this->window_type = frontend_conf["window"].as<string>();
this->n_mels = frontend_conf["n_mels"].as<int>();
this->frame_length = frontend_conf["frame_length"].as<int>();
this->frame_shift = frontend_conf["frame_shift"].as<int>();
this->lfr_m = frontend_conf["lfr_m"].as<int>();
this->lfr_n = frontend_conf["lfr_n"].as<int>();
this->encoder_size = encoder_conf["output_size"].as<int>();
this->fsmn_dims = encoder_conf["output_size"].as<int>();
this->fsmn_layers = decoder_conf["num_blocks"].as<int>();
this->fsmn_lorder = decoder_conf["kernel_size"].as<int>()-1;
this->cif_threshold = predictor_conf["threshold"].as<double>();
this->tail_alphas = predictor_conf["tail_threshold"].as<double>();
this->asr_sample_rate = frontend_conf["fs"].as<int>();
}catch(exception const &e){
LOG(ERROR) << "Error when load argument from vad config YAML.";
exit(-1);
}
}
void SenseVoiceSmall::LoadConfigFromYaml(const char* filename){
YAML::Node config;
@@ -84,6 +223,9 @@ SenseVoiceSmall::~SenseVoiceSmall()
if(vocab){
delete vocab;
}
if(online_vocab){
delete online_vocab;
}
if(lm_vocab){
delete lm_vocab;
}
@@ -212,6 +354,30 @@ string SenseVoiceSmall::CTCSearch(float * in, std::vector<int32_t> paraformer_le
return str_lang + str_emo + str_event + " " + text;
}
string SenseVoiceSmall::GreedySearch(float * in, int n_len, int64_t token_nums, bool is_stamp, std::vector<float> us_alphas, std::vector<float> us_cif_peak)
{
vector<int> hyps;
int Tmax = n_len;
for (int i = 0; i < Tmax; i++) {
int max_idx;
float max_val;
FindMax(in + i * token_nums, token_nums, max_val, max_idx);
hyps.push_back(max_idx);
}
if(!is_stamp){
return online_vocab->Vector2StringV2(hyps, language);
}else{
std::vector<string> char_list;
std::vector<std::vector<float>> timestamp_list;
std::string res_str;
online_vocab->Vector2String(hyps, char_list);
std::vector<string> raw_char(char_list);
TimestampOnnx(us_alphas, us_cif_peak, char_list, res_str, timestamp_list);
return PostProcess(raw_char, timestamp_list);
}
}
void SenseVoiceSmall::LfrCmvn(std::vector<std::vector<float>> &asr_feats) {
std::vector<std::vector<float>> out_feats;
runtime/onnxruntime/src/sensevoice-small.h
+9 -3
View File
@@ -12,12 +12,14 @@ namespace funasr {
class SenseVoiceSmall : public Model {
private:
Vocab* vocab = nullptr;
Vocab* online_vocab = nullptr;
Vocab* lm_vocab = nullptr;
SegDict* seg_dict = nullptr;
PhoneSet* phone_set_ = nullptr;
const float scale = 1.0;
void LoadConfigFromYaml(const char* filename);
void LoadOnlineConfigFromYaml(const char* filename);
void LoadCmvn(const char *filename);
void LfrCmvn(std::vector<std::vector<float>> &asr_feats);
@@ -34,9 +36,10 @@ namespace funasr {
~SenseVoiceSmall();
void InitAsr(const std::string &am_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num);
// online
// void InitAsr(const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num);
void InitAsr(const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num);
// 2pass
// void InitAsr(const std::string &am_model, const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num);
void InitAsr(const std::string &am_model, const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config,
const std::string &token_file, const std::string &online_token_file, int thread_num);
// void InitHwCompiler(const std::string &hw_model, int thread_num);
// void InitSegDict(const std::string &seg_dict_model);
std::vector<std::vector<float>> CompileHotwordEmbedding(std::string &hotwords);
@@ -44,7 +47,8 @@ namespace funasr {
void FbankKaldi(float sample_rate, const float* waves, int len, std::vector<std::vector<float>> &asr_feats);
std::vector<std::string> Forward(float** din, int* len, bool input_finished=true, std::string svs_lang="auto", bool svs_itn=true, int batch_in=1);
string CTCSearch( float * in, std::vector<int32_t> paraformer_length, std::vector<int64_t> outputShape);
string GreedySearch( float* in, int n_len, int64_t token_nums,
bool is_stamp=false, std::vector<float> us_alphas={0}, std::vector<float> us_cif_peak={0});
string Rescoring();
string GetLang(){return language;};
int GetAsrSampleRate() { return asr_sample_rate; };
@@ -100,6 +104,8 @@ namespace funasr {
int asr_sample_rate = MODEL_SAMPLE_RATE;
int batch_size_ = 1;
int blank_id = 0;
float cif_threshold = 1.0;
float tail_alphas = 0.45;
//dict
std::map<std::string, int> lid_map = {
{"auto", 0},
runtime/onnxruntime/src/tpass-online-stream.cpp
+1 -1
View File
@@ -11,7 +11,7 @@ TpassOnlineStream::TpassOnlineStream(TpassStream* tpass_stream, std::vector<int>
}
if(tpass_obj->asr_handle){
asr_online_handle = make_unique<ParaformerOnline>((Paraformer*)(tpass_obj->asr_handle).get(), chunk_size);
asr_online_handle = make_unique<ParaformerOnline>((tpass_obj->asr_handle).get(), chunk_size, tpass_stream->GetModelType());
}else{
LOG(ERROR)<<"asr_handle is null";
exit(-1);
runtime/onnxruntime/src/tpass-stream.cpp
+10 -2
View File
@@ -36,10 +36,17 @@ TpassStream::TpassStream(std::map<std::string, std::string>& model_path, int thr
string am_cmvn_path;
string am_config_path;
string token_path;
string online_token_path;
string hw_compile_model_path;
string seg_dict_path;
asr_handle = make_unique<Paraformer>();
if (model_path.at(MODEL_DIR).find(MODEL_SVS) != std::string::npos)
{
asr_handle = make_unique<SenseVoiceSmall>();
model_type = MODEL_SVS;
}else{
asr_handle = make_unique<Paraformer>();
}
bool enable_hotword = false;
hw_compile_model_path = PathAppend(model_path.at(MODEL_DIR), MODEL_EB_NAME);
@@ -54,6 +61,7 @@ TpassStream::TpassStream(std::map<std::string, std::string>& model_path, int thr
am_model_path = PathAppend(model_path.at(OFFLINE_MODEL_DIR), MODEL_NAME);
en_model_path = PathAppend(model_path.at(ONLINE_MODEL_DIR), ENCODER_NAME);
de_model_path = PathAppend(model_path.at(ONLINE_MODEL_DIR), DECODER_NAME);
online_token_path = PathAppend(model_path.at(ONLINE_MODEL_DIR), TOKEN_PATH);
if(model_path.find(QUANTIZE) != model_path.end() && model_path.at(QUANTIZE) == "true"){
am_model_path = PathAppend(model_path.at(OFFLINE_MODEL_DIR), QUANT_MODEL_NAME);
en_model_path = PathAppend(model_path.at(ONLINE_MODEL_DIR), QUANT_ENCODER_NAME);
@@ -63,7 +71,7 @@ TpassStream::TpassStream(std::map<std::string, std::string>& model_path, int thr
am_config_path = PathAppend(model_path.at(ONLINE_MODEL_DIR), AM_CONFIG_NAME);
token_path = PathAppend(model_path.at(MODEL_DIR), TOKEN_PATH);
asr_handle->InitAsr(am_model_path, en_model_path, de_model_path, am_cmvn_path, am_config_path, token_path, thread_num);
asr_handle->InitAsr(am_model_path, en_model_path, de_model_path, am_cmvn_path, am_config_path, token_path, online_token_path, thread_num);
}else{
LOG(ERROR) <<"Can not find offline-model-dir or online-model-dir";
exit(-1);
runtime/websocket/bin/funasr-wss-client-2pass.cpp
+23 -10
View File
@@ -124,7 +124,7 @@ class WebsocketClient {
void run(const std::string& uri, const std::vector<string>& wav_list,
const std::vector<string>& wav_ids, int audio_fs, std::string asr_mode,
std::vector<int> chunk_size, const std::unordered_map<std::string, int>& hws_map,
bool is_record=false, int use_itn=1) {
bool is_record=false, int use_itn=1, int svs_itn=1) {
// Create a new connection to the given URI
websocketpp::lib::error_code ec;
typename websocketpp::client<T>::connection_ptr con =
@@ -146,9 +146,9 @@ class WebsocketClient {
websocketpp::lib::thread asio_thread(&websocketpp::client<T>::run,
&m_client);
if(is_record){
send_rec_data(asr_mode, chunk_size, hws_map, use_itn);
send_rec_data(asr_mode, chunk_size, hws_map, use_itn, svs_itn);
}else{
send_wav_data(wav_list[0], wav_ids[0], audio_fs, asr_mode, chunk_size, hws_map, use_itn);
send_wav_data(wav_list[0], wav_ids[0], audio_fs, asr_mode, chunk_size, hws_map, use_itn, svs_itn);
}
WaitABit();
@@ -185,7 +185,7 @@ class WebsocketClient {
// send wav to server
void send_wav_data(string wav_path, string wav_id, int audio_fs, std::string asr_mode,
std::vector<int> chunk_vector, const std::unordered_map<std::string, int>& hws_map,
int use_itn) {
int use_itn, int svs_itn) {
uint64_t count = 0;
std::stringstream val;
@@ -241,9 +241,13 @@ class WebsocketClient {
jsonbegin["audio_fs"] = sampling_rate;
jsonbegin["is_speaking"] = true;
jsonbegin["itn"] = true;
jsonbegin["svs_itn"] = true;
if(use_itn == 0){
jsonbegin["itn"] = false;
}
if(svs_itn == 0){
jsonbegin["svs_itn"] = false;
}
if(!hws_map.empty()){
LOG(INFO) << "hotwords: ";
for (const auto& pair : hws_map) {
@@ -335,7 +339,7 @@ class WebsocketClient {
}
void send_rec_data(std::string asr_mode, std::vector<int> chunk_vector,
const std::unordered_map<std::string, int>& hws_map, int use_itn) {
const std::unordered_map<std::string, int>& hws_map, int use_itn, int svs_itn) {
// first message
bool wait = false;
while (1) {
@@ -374,9 +378,13 @@ class WebsocketClient {
jsonbegin["audio_fs"] = sample_rate;
jsonbegin["is_speaking"] = true;
jsonbegin["itn"] = true;
jsonbegin["svs_itn"] = true;
if(use_itn == 0){
jsonbegin["itn"] = false;
}
if(svs_itn == 0){
jsonbegin["svs_itn"] = false;
}
if(!hws_map.empty()){
LOG(INFO) << "hotwords: ";
for (const auto& pair : hws_map) {
@@ -513,6 +521,9 @@ int main(int argc, char* argv[]) {
"", "use-itn",
"use-itn is 1 means use itn, 0 means not use itn", false, 1,
"int");
TCLAP::ValueArg<int> svs_itn_(
"", "svs-itn",
"svs-itn is 1 means use itn and punc, 0 means not use", false, 1, "int");
TCLAP::ValueArg<std::string> hotword_("", HOTWORD,
"the hotword file, one hotword perline, Format: Hotword Weight (could be: 阿里巴巴 20)", false, "", "string");
@@ -526,6 +537,7 @@ int main(int argc, char* argv[]) {
cmd.add(thread_num_);
cmd.add(is_ssl_);
cmd.add(use_itn_);
cmd.add(svs_itn_);
cmd.add(hotword_);
cmd.parse(argc, argv);
@@ -535,6 +547,7 @@ int main(int argc, char* argv[]) {
std::string asr_mode = asr_mode_.getValue();
std::string chunk_size_str = chunk_size_.getValue();
int use_itn = use_itn_.getValue();
int svs_itn = svs_itn_.getValue();
// get chunk_size
std::vector<int> chunk_size;
std::stringstream ss(chunk_size_str);
@@ -577,11 +590,11 @@ int main(int argc, char* argv[]) {
c.m_client.set_tls_init_handler(bind(&OnTlsInit, ::_1));
c.run(uri, tmp_wav_list, tmp_wav_ids, audio_fs, asr_mode, chunk_size, hws_map, true, use_itn);
c.run(uri, tmp_wav_list, tmp_wav_ids, audio_fs, asr_mode, chunk_size, hws_map, true, use_itn, svs_itn);
} else {
WebsocketClient<websocketpp::config::asio_client> c(is_ssl);
c.run(uri, tmp_wav_list, tmp_wav_ids, audio_fs, asr_mode, chunk_size, hws_map, true, use_itn);
c.run(uri, tmp_wav_list, tmp_wav_ids, audio_fs, asr_mode, chunk_size, hws_map, true, use_itn, svs_itn);
}
}else{
@@ -622,17 +635,17 @@ int main(int argc, char* argv[]) {
tmp_wav_ids.emplace_back(wav_ids[wav_i + i]);
client_threads.emplace_back(
[uri, tmp_wav_list, tmp_wav_ids, audio_fs, asr_mode, chunk_size, is_ssl, hws_map, use_itn]() {
[uri, tmp_wav_list, tmp_wav_ids, audio_fs, asr_mode, chunk_size, is_ssl, hws_map, use_itn, svs_itn]() {
if (is_ssl == 1) {
WebsocketClient<websocketpp::config::asio_tls_client> c(is_ssl);
c.m_client.set_tls_init_handler(bind(&OnTlsInit, ::_1));
c.run(uri, tmp_wav_list, tmp_wav_ids, audio_fs, asr_mode, chunk_size, hws_map, false, use_itn);
c.run(uri, tmp_wav_list, tmp_wav_ids, audio_fs, asr_mode, chunk_size, hws_map, false, use_itn, svs_itn);
} else {
WebsocketClient<websocketpp::config::asio_client> c(is_ssl);
c.run(uri, tmp_wav_list, tmp_wav_ids, audio_fs, asr_mode, chunk_size, hws_map, false, use_itn);
c.run(uri, tmp_wav_list, tmp_wav_ids, audio_fs, asr_mode, chunk_size, hws_map, false, use_itn, svs_itn);
}
});
}
runtime/websocket/bin/funasr-wss-server-2pass.cpp
+6
View File
@@ -276,6 +276,12 @@ int main(int argc, char* argv[]) {
s_itn_path="";
s_lm_path="";
}
found = s_offline_asr_path.find(MODEL_SVS);
if (found != std::string::npos) {
model_path["model-revision"]="v2.0.5";
s_lm_path="";
model_path[LM_DIR]="";
}
if (access(s_offline_asr_path.c_str(), F_OK) == 0) {
// local
runtime/websocket/bin/websocket-server-2pass.cpp
+21 -5
View File
@@ -111,7 +111,9 @@ void WebSocketServer::do_decoder(
int audio_fs,
std::string wav_format,
FUNASR_HANDLE& tpass_online_handle,
FUNASR_DEC_HANDLE& decoder_handle) {
FUNASR_DEC_HANDLE& decoder_handle,
std::string svs_lang,
bool sys_itn) {
// lock for each connection
if(!tpass_online_handle){
scoped_lock guard(thread_lock);
@@ -140,7 +142,8 @@ void WebSocketServer::do_decoder(
subvector.data(), subvector.size(),
punc_cache, false, audio_fs,
wav_format, (ASR_TYPE)asr_mode_,
hotwords_embedding, itn, decoder_handle);
hotwords_embedding, itn, decoder_handle,
svs_lang, sys_itn);
} else {
scoped_lock guard(thread_lock);
@@ -177,7 +180,8 @@ void WebSocketServer::do_decoder(
buffer.data(), buffer.size(), punc_cache,
is_final, audio_fs,
wav_format, (ASR_TYPE)asr_mode_,
hotwords_embedding, itn, decoder_handle);
hotwords_embedding, itn, decoder_handle,
svs_lang, sys_itn);
} else {
scoped_lock guard(thread_lock);
msg["access_num"]=(int)msg["access_num"]-1;
@@ -250,6 +254,8 @@ void WebSocketServer::on_open(websocketpp::connection_hdl hdl) {
data_msg->msg["audio_fs"] = 16000; // default is 16k
data_msg->msg["access_num"] = 0; // the number of access for this object, when it is 0, we can free it saftly
data_msg->msg["is_eof"]=false; // if this connection is closed
data_msg->msg["svs_lang"]="auto";
data_msg->msg["svs_itn"]=true;
FUNASR_DEC_HANDLE decoder_handle =
FunASRWfstDecoderInit(tpass_handle, ASR_TWO_PASS, global_beam_, lattice_beam_, am_scale_);
data_msg->decoder_handle = decoder_handle;
@@ -475,6 +481,12 @@ void WebSocketServer::on_message(websocketpp::connection_hdl hdl,
if (jsonresult.contains("itn")) {
msg_data->msg["itn"] = jsonresult["itn"];
}
if (jsonresult.contains("svs_lang")) {
msg_data->msg["svs_lang"] = jsonresult["svs_lang"];
}
if (jsonresult.contains("svs_itn")) {
msg_data->msg["svs_itn"] = jsonresult["svs_itn"];
}
LOG(INFO) << "jsonresult=" << jsonresult
<< ", msg_data->msg=" << msg_data->msg;
if ((jsonresult["is_speaking"] == false ||
@@ -499,7 +511,9 @@ void WebSocketServer::on_message(websocketpp::connection_hdl hdl,
msg_data->msg["audio_fs"],
msg_data->msg["wav_format"],
std::ref(msg_data->tpass_online_handle),
std::ref(msg_data->decoder_handle)));
std::ref(msg_data->decoder_handle),
msg_data->msg["svs_lang"],
msg_data->msg["svs_itn"]));
msg_data->msg["access_num"]=(int)(msg_data->msg["access_num"])+1;
}
catch (std::exception const &e)
@@ -547,7 +561,9 @@ void WebSocketServer::on_message(websocketpp::connection_hdl hdl,
msg_data->msg["audio_fs"],
msg_data->msg["wav_format"],
std::ref(msg_data->tpass_online_handle),
std::ref(msg_data->decoder_handle)));
std::ref(msg_data->decoder_handle),
msg_data->msg["svs_lang"],
msg_data->msg["svs_itn"]));
msg_data->msg["access_num"]=(int)(msg_data->msg["access_num"])+1;
}
}
runtime/websocket/bin/websocket-server-2pass.h
+3 -1
View File
@@ -125,7 +125,9 @@ class WebSocketServer {
int audio_fs,
std::string wav_format,
FUNASR_HANDLE& tpass_online_handle,
FUNASR_DEC_HANDLE& decoder_handle);
FUNASR_DEC_HANDLE& decoder_handle,
std::string svs_lang,
bool sys_itn);
void initAsr(std::map<std::string, std::string>& model_path, int thread_num);
void on_message(websocketpp::connection_hdl hdl, message_ptr msg);