From 702ec03ad89d5c62e97eed770a6882d6412f8d58 Mon Sep 17 00:00:00 2001 From: Yabin Li Date: Tue, 7 Nov 2023 18:34:29 +0800 Subject: [PATCH] Dev new (#1065) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add hotword for deploy_tools * Support wfst decoder and contextual biasing (#1039) * Support wfst decoder and contextual biasing * Turn on fstbin compilation --------- Co-authored-by: gongbo.gb * mv funasr/runtime runtime * Fix crash caused by OOV in hotwords list * funasr infer * funasr infer * funasr infer * funasr infer * funasr infer * fix some bugs about fst hotword; support wfst for websocket server and clients; mv runtime out of funasr; modify relative docs * del onnxruntime/include/gflags * update tensor.h * update run_server.sh * update deploy tools * update deploy tools * update websocket-server * update funasr-wss-server * Remove self loop propagation * Update websocket_protocol_zh.md * Update websocket_protocol_zh.md * update hotword protocol * author zhaomingwork: change hotwords for h5 and java * update hotword protocol * catch exception for json_fst_hws * update hotword on message * update onnx benchmark for ngram&hotword * update docs * update funasr-wss-serve * add NONE for LM_DIR * update docs * update run_server.sh * add whats-new * modify whats-new * update whats-new * update whats-new * Support decoder option for beam searching * update benchmark_onnx_cpp * Support decoder option for websocket * fix bug of CompileHotwordEmbedding * update html client * update docs --------- Co-authored-by: gongbo.gb <35997837+aibulamusi@users.noreply.github.com> Co-authored-by: gongbo.gb Co-authored-by: 游雁 --- README_zh.md | 75 +- funasr/__init__.py | 124 ++ funasr/bin/asr_inference_launch.py | 46 +- funasr/runtime/onnxruntime/src/util.cpp | 231 -- funasr/runtime/onnxruntime/src/util.h | 38 - .../openfst/src/lib/CMakeLists.txt | 26 - funasr/runtime/python/grpc/.gitignore | 1 - funasr/runtime/websocket/hotwords.txt | 2 - funasr/text/cleaner.py | 5 +- {funasr/runtime => runtime}/__init__.py | 0 .../android/AndroidClient/app/build.gradle | 0 .../AndroidClient/app/proguard-rules.pro | 0 .../androidclient/ExampleInstrumentedTest.kt | 0 .../app/src/main/AndroidManifest.xml | 0 .../yeyupiaoling/androidclient/AudioView.java | 0 .../androidclient/MainActivity.java | 0 .../androidclient/SSLSocketClient.java | 0 .../src/main/res/drawable/edittext_border.xml | 0 .../res/drawable/ic_launcher_background.xml | 0 .../res/drawable/ic_launcher_foreground.xml | 0 .../app/src/main/res/drawable/logo.png | Bin .../app/src/main/res/layout/activity_main.xml | 0 .../main/res/layout/dialog_input_hotwords.xml | 0 .../src/main/res/layout/dialog_input_uri.xml | 0 .../app/src/main/res/menu/menu.xml | 0 .../res/mipmap-anydpi-v26/ic_launcher.xml | 0 .../mipmap-anydpi-v26/ic_launcher_round.xml | 0 .../src/main/res/mipmap-hdpi/ic_launcher.webp | Bin .../res/mipmap-hdpi/ic_launcher_round.webp | Bin .../src/main/res/mipmap-mdpi/ic_launcher.webp | Bin .../res/mipmap-mdpi/ic_launcher_round.webp | Bin .../main/res/mipmap-xhdpi/ic_launcher.webp | Bin .../res/mipmap-xhdpi/ic_launcher_round.webp | Bin .../main/res/mipmap-xxhdpi/ic_launcher.webp | Bin .../res/mipmap-xxhdpi/ic_launcher_round.webp | Bin .../main/res/mipmap-xxxhdpi/ic_launcher.webp | Bin .../res/mipmap-xxxhdpi/ic_launcher_round.webp | Bin .../app/src/main/res/values/colors.xml | 0 .../app/src/main/res/values/strings.xml | 0 .../app/src/main/res/values/themes.xml | 0 .../app/src/main/res/xml/backup_rules.xml | 0 .../main/res/xml/data_extraction_rules.xml | 0 .../androidclient/ExampleUnitTest.kt | 0 .../android/AndroidClient/build.gradle | 0 .../android/AndroidClient/gradle.properties | 0 .../gradle/wrapper/gradle-wrapper.jar | Bin .../gradle/wrapper/gradle-wrapper.properties | 0 .../android/AndroidClient/gradlew | 0 .../android/AndroidClient/gradlew.bat | 0 .../android/AndroidClient/settings.gradle | 0 .../android/images/QRcode.png | Bin .../android/images/image1.png | Bin .../android/images/image2.png | Bin .../android/images/image3.png | Bin {funasr/runtime => runtime}/android/readme.md | 0 .../AliFsmnVadSharp.Examples.csproj | 0 .../AliFsmnVadSharp.Examples/Program.cs | 0 .../csharp/AliFsmnVad/AliFsmnVadSharp.sln | 0 .../AliFsmnVad/AliFsmnVadSharp/AliFsmnVad.cs | 0 .../AliFsmnVadSharp/AliFsmnVadSharp.csproj | 0 .../AliFsmnVadSharp/DLL/KaldiNativeFbank.cs | 0 .../AliFsmnVadSharp/DLL/KnfOnlineFbank.cs | 0 .../AliFsmnVad/AliFsmnVadSharp/E2EVadModel.cs | 0 .../Lib/kaldi-native-fbank-dll.dll | Bin .../AliFsmnVadSharp/Model/CmvnEntity.cs | 0 .../Model/E2EVadFrameProbEntity.cs | 0 .../Model/E2EVadSpeechBufWithDoaEntity.cs | 0 .../Model/EncoderConfEntity.cs | 0 .../Model/FrontendConfEntity.cs | 0 .../AliFsmnVadSharp/Model/SegmentEntity.cs | 0 .../AliFsmnVadSharp/Model/VadInputEntity.cs | 0 .../AliFsmnVadSharp/Model/VadOutputEntity.cs | 0 .../Model/VadPostConfEntity.cs | 0 .../AliFsmnVadSharp/Model/VadYamlEntity.cs | 0 .../AliFsmnVadSharp/Struct/FbankData.cs | 0 .../AliFsmnVadSharp/Utils/YamlHelper.cs | 0 .../AliFsmnVad/AliFsmnVadSharp/WavFrontend.cs | 0 .../AliFsmnVadSharp/WindowDetector.cs | 0 .../csharp/AliFsmnVad/README.md | 0 .../App.xaml | 0 .../App.xaml.cs | 0 .../AppShell.xaml | 0 .../AppShell.xaml.cs | 0 .../MainPage.xaml | 0 .../MainPage.xaml.cs | 0 .../MauiApp1.csproj | 0 .../MauiApp1.csproj.user | 0 .../MauiProgram.cs | 0 .../MySplashPage.xaml | 0 .../MySplashPage.xaml.cs | 0 .../Platforms/Android/AndroidManifest.xml | 0 .../Platforms/Android/MainActivity.cs | 0 .../Platforms/Android/MainApplication.cs | 0 .../Android/Resources/values/colors.xml | 0 .../Platforms/MacCatalyst/AppDelegate.cs | 0 .../Platforms/MacCatalyst/Info.plist | 0 .../Platforms/MacCatalyst/Program.cs | 0 .../Platforms/Tizen/Main.cs | 0 .../Platforms/Tizen/tizen-manifest.xml | 0 .../Platforms/Windows/App.xaml | 0 .../Platforms/Windows/App.xaml.cs | 0 .../Platforms/Windows/Package.appxmanifest | 0 .../Platforms/Windows/app.manifest | 0 .../Platforms/iOS/AppDelegate.cs | 0 .../Platforms/iOS/Info.plist | 0 .../Platforms/iOS/Program.cs | 0 .../Properties/launchSettings.json | 0 .../RecognitionForFiles.xaml | 0 .../RecognitionForFiles.xaml.cs | 0 .../Resources/AppIcon/appicon.svg | 0 .../Resources/AppIcon/appiconfg.svg | 0 .../Resources/Fonts/OpenSans-Regular.ttf | Bin .../Resources/Fonts/OpenSans-Semibold.ttf | Bin .../Resources/Images/dotnet_bot.svg | 0 .../Resources/Raw/AboutAssets.txt | 0 .../Resources/Splash/splash.svg | 0 .../Resources/Styles/Colors.xaml | 0 .../Resources/Styles/Styles.xaml | 0 .../Utils/DownloadHelper.cs | 0 .../Utils/SysConf.cs | 0 .../AliParaformerAsr.Examples.csproj | 0 .../AliParaformerAsr.Examples/Program.cs | 0 .../AliParaformerAsr/AliParaformerAsr.sln | 0 .../AliParaformerAsr/AliParaformerAsr.csproj | 0 .../AliParaformerAsr/Model/CmvnEntity.cs | 0 .../Model/DecoderConfEntity.cs | 0 .../Model/EncoderConfEntity.cs | 0 .../Model/FrontendConfEntity.cs | 0 .../AliParaformerAsr/Model/ModelConfEntity.cs | 0 .../Model/OfflineInputEntity.cs | 0 .../Model/OfflineOutputEntity.cs | 0 .../Model/OfflineYamlEntity.cs | 0 .../Model/PostEncoderConfEntity.cs | 0 .../Model/PreEncoderConfEntity.cs | 0 .../Model/PredictorConfEntity.cs | 0 .../AliParaformerAsr/OfflineRecognizer.cs | 0 .../AliParaformerAsr/Utils/YamlHelper.cs | 0 .../AliParaformerAsr/WavFrontend.cs | 0 .../csharp/AliParaformerAsr/README.md | 0 .../csharp/ws-client/FunASRClient_CShape.sln | 0 .../csharp/ws-client/FunASRClient_CShape.suo | Bin .../FunASRWSClient_Offline.csproj | 0 .../FunASRWSClient_Offline/Program.cs | 0 .../FunASRWSClient_Offline/README.md | 0 .../FunASRWSClient_Offline/WebScoketClient.cs | 0 .../FunASRWSClient_Online.csproj | 0 .../FunASRWSClient_Online/Program.cs | 0 .../ws-client/FunASRWSClient_Online/README.md | 0 .../FunASRWSClient_Online/WaveCollect.cs | 0 .../FunASRWSClient_Online/WebScoketClient.cs | 0 .../csharp/ws-client/confg/config.ini | 0 .../csharp/ws-client/confg/hotword.txt | 0 .../csharp/ws-client/confg/tmp.wav | Bin .../funasr-runtime-deploy-offline-cpu-en.sh | 16 +- .../funasr-runtime-deploy-offline-cpu-zh.sh | 108 +- .../funasr-runtime-deploy-online-cpu-zh.sh | 16 +- runtime/deploy_tools/install_docker.sh | 82 + .../docs/SDK_advanced_guide_offline.md | 29 +- .../docs/SDK_advanced_guide_offline_en.md | 20 +- .../docs/SDK_advanced_guide_offline_en_zh.md | 33 +- .../docs/SDK_advanced_guide_offline_zh.md | 44 +- .../docs/SDK_advanced_guide_online.md | 8 +- .../docs/SDK_advanced_guide_online_zh.md | 15 +- .../runtime => runtime}/docs/SDK_tutorial.md | 10 +- .../docs/SDK_tutorial_en.md | 6 +- .../docs/SDK_tutorial_en_zh.md | 8 +- .../docs/SDK_tutorial_online.md | 8 +- .../docs/SDK_tutorial_online_zh.md | 10 +- .../docs/SDK_tutorial_zh.md | 11 +- .../docs/aliyun_server_tutorial.md | 0 .../docs/benchmark_libtorch.md | 0 .../docs/benchmark_onnx.md | 0 .../docs/benchmark_onnx_cpp.md | 86 +- .../docs/docker_offline_cpu_en_lists | 2 +- .../docs/docker_offline_cpu_zh_lists | 5 +- .../docs/docker_online_cpu_zh_lists | 2 +- .../runtime => runtime}/docs/images/2pass.jpg | Bin .../docs/images/aliyun1.png | Bin .../docs/images/aliyun10.png | Bin .../docs/images/aliyun11.png | Bin .../docs/images/aliyun12.png | Bin .../docs/images/aliyun2.png | Bin .../docs/images/aliyun3.png | Bin .../docs/images/aliyun4.png | Bin .../docs/images/aliyun5.png | Bin .../docs/images/aliyun6.png | Bin .../docs/images/aliyun7.png | Bin .../docs/images/aliyun8.png | Bin .../docs/images/aliyun9.png | Bin .../runtime => runtime}/docs/images/html.png | Bin runtime/docs/lm_train_tutorial.md | 40 + .../docs/websocket_protocol.md | 8 +- .../docs/websocket_protocol_zh.md | 11 +- .../runtime => runtime}/grpc/CMakeLists.txt | 0 {funasr/runtime => runtime}/grpc/Readme.md | 0 {funasr/runtime => runtime}/grpc/build.sh | 0 {funasr/runtime => runtime}/grpc/common.cmake | 0 .../grpc/paraformer-server.cc | 0 .../grpc/paraformer-server.h | 0 .../runtime => runtime}/grpc/run_server.sh | 0 {funasr/runtime => runtime}/html5/h5Server.py | 0 {funasr/runtime => runtime}/html5/readme.md | 0 .../runtime => runtime}/html5/readme_zh.md | 0 {funasr/runtime => runtime}/html5/ssl_key | 0 .../html5/static/index.html | 16 +- .../runtime => runtime}/html5/static/main.js | 34 +- .../runtime => runtime}/html5/static/pcm.js | 0 .../html5/static/recorder-core.js | 0 .../runtime => runtime}/html5/static/wav.js | 0 .../html5/static/wsconnecter.js | 1 + {funasr/runtime => runtime}/ios/Readme.md | 0 .../ios/paraformer_online/Podfile | 0 .../project.pbxproj | 0 .../contents.xcworkspacedata | 0 .../UserInterfaceState.xcuserstate | Bin .../xcdebugger/Breakpoints_v2.xcbkptlist | 0 .../xcschemes/xcschememanagement.plist | 0 .../paraformer_online/AppDelegate.h | 0 .../paraformer_online/AppDelegate.m | 0 .../AccentColor.colorset/Contents.json | 0 .../AppIcon.appiconset/Contents.json | 0 .../Assets.xcassets/Contents.json | 0 .../paraformer_online/AudioCapture.h | 0 .../paraformer_online/AudioCapture.mm | 0 .../paraformer_online/AudioRecorder.h | 0 .../paraformer_online/AudioRecorder.m | 0 .../Base.lproj/LaunchScreen.storyboard | 0 .../Base.lproj/Main.storyboard | 0 .../paraformer_online/Info.plist | 0 .../paraformer_online/SceneDelegate.h | 0 .../paraformer_online/SceneDelegate.m | 0 .../paraformer_online/ViewController.h | 0 .../paraformer_online/ViewController.mm | 0 .../paraformer_online/main.m | 0 .../java/FunasrWsClient.java | 32 +- {funasr/runtime => runtime}/java/Makefile | 0 {funasr/runtime => runtime}/java/readme.md | 0 .../onnxruntime/CMakeLists.txt | 4 +- .../onnxruntime/CMakeSettings.json | 0 .../onnxruntime/bin/CMakeLists.txt | 0 .../onnxruntime/bin/funasr-onnx-2pass-rtf.cpp | 35 +- .../onnxruntime/bin/funasr-onnx-2pass.cpp | 30 +- .../bin/funasr-onnx-offline-punc.cpp | 0 .../bin/funasr-onnx-offline-rtf.cpp | 76 +- .../bin/funasr-onnx-offline-vad.cpp | 0 .../onnxruntime/bin/funasr-onnx-offline.cpp | 68 +- .../bin/funasr-onnx-online-asr.cpp | 0 .../bin/funasr-onnx-online-punc.cpp | 0 .../bin/funasr-onnx-online-rtf.cpp | 0 .../bin/funasr-onnx-online-vad.cpp | 0 .../onnxruntime/include/audio.h | 0 .../onnxruntime/include/com-define.h | 10 + .../onnxruntime/include/funasrruntime.h | 15 +- .../onnxruntime/include/itn-model.h | 0 .../onnxruntime/include/model.h | 6 +- .../onnxruntime/include/offline-stream.h | 0 .../onnxruntime/include/punc-model.h | 0 .../onnxruntime/include/tclap/Arg.h | 0 .../onnxruntime/include/tclap/ArgException.h | 0 .../onnxruntime/include/tclap/ArgTraits.h | 0 .../onnxruntime/include/tclap/CmdLine.h | 0 .../include/tclap/CmdLineInterface.h | 0 .../onnxruntime/include/tclap/CmdLineOutput.h | 0 .../onnxruntime/include/tclap/Constraint.h | 0 .../onnxruntime/include/tclap/DocBookOutput.h | 0 .../onnxruntime/include/tclap/HelpVisitor.h | 0 .../include/tclap/IgnoreRestVisitor.h | 0 .../onnxruntime/include/tclap/MultiArg.h | 0 .../include/tclap/MultiSwitchArg.h | 0 .../include/tclap/OptionalUnlabeledTracker.h | 0 .../include/tclap/StandardTraits.h | 0 .../onnxruntime/include/tclap/StdOutput.h | 0 .../onnxruntime/include/tclap/SwitchArg.h | 0 .../include/tclap/UnlabeledMultiArg.h | 0 .../include/tclap/UnlabeledValueArg.h | 0 .../onnxruntime/include/tclap/ValueArg.h | 0 .../include/tclap/ValuesConstraint.h | 0 .../include/tclap/VersionVisitor.h | 0 .../onnxruntime/include/tclap/Visitor.h | 0 .../onnxruntime/include/tclap/XorHandler.h | 0 .../include/tclap/ZshCompletionOutput.h | 0 .../onnxruntime/include/tclap/sstream.h | 0 .../onnxruntime/include/tpass-online-stream.h | 0 .../onnxruntime/include/tpass-stream.h | 0 .../onnxruntime/include/vad-model.h | 0 .../runtime => runtime}/onnxruntime/readme.md | 4 +- .../onnxruntime/src/CMakeLists.txt | 5 +- .../onnxruntime/src/alignedmem.cpp | 0 .../onnxruntime/src/alignedmem.h | 0 .../onnxruntime/src/audio.cpp | 0 runtime/onnxruntime/src/bias-lm.cpp | 178 ++ runtime/onnxruntime/src/bias-lm.h | 137 ++ .../onnxruntime/src/common-struct.h | 0 .../onnxruntime/src/commonfunc.h | 0 .../onnxruntime/src/ct-transformer-online.cpp | 0 .../onnxruntime/src/ct-transformer-online.h | 0 .../onnxruntime/src/ct-transformer.cpp | 0 .../onnxruntime/src/ct-transformer.h | 0 .../onnxruntime/src/e2e-vad.h | 0 .../onnxruntime/src/encode_converter.cpp | 0 .../onnxruntime/src/encode_converter.h | 0 .../onnxruntime/src/fsmn-vad-online.cpp | 0 .../onnxruntime/src/fsmn-vad-online.h | 0 .../onnxruntime/src/fsmn-vad.cpp | 0 .../onnxruntime/src/fsmn-vad.h | 0 .../onnxruntime/src/funasrruntime.cpp | 71 +- .../onnxruntime/src/itn-processor.cpp | 0 .../onnxruntime/src/itn-processor.h | 0 .../onnxruntime/src/itn-token-parser.cpp | 0 .../onnxruntime/src/itn-token-parser.h | 0 .../onnxruntime/src/model.cpp | 0 .../onnxruntime/src/offline-stream.cpp | 7 + .../onnxruntime/src/paraformer-online.cpp | 0 .../onnxruntime/src/paraformer-online.h | 8 + .../onnxruntime/src/paraformer.cpp | 300 +-- .../onnxruntime/src/paraformer.h | 29 +- runtime/onnxruntime/src/phone-set.cpp | 76 + runtime/onnxruntime/src/phone-set.h | 34 + .../onnxruntime/src/precomp.h | 4 +- .../onnxruntime/src/predefine-coe.h | 0 .../onnxruntime/src/punc-model.cpp | 0 .../onnxruntime/src/resample.cpp | 0 .../onnxruntime/src/resample.h | 0 .../onnxruntime/src/seg_dict.cpp | 0 .../onnxruntime/src/seg_dict.h | 0 .../onnxruntime/src/tensor.h | 4 +- .../onnxruntime/src/tokenizer.cpp | 0 .../onnxruntime/src/tokenizer.h | 0 .../onnxruntime/src/tpass-online-stream.cpp | 0 .../onnxruntime/src/tpass-stream.cpp | 0 .../onnxruntime/src/utf8-string.cpp | 0 .../onnxruntime/src/utf8-string.h | 0 runtime/onnxruntime/src/util.cpp | 672 ++++++ runtime/onnxruntime/src/util.h | 61 + .../onnxruntime/src/vad-model.cpp | 0 .../onnxruntime/src/vocab.cpp | 22 +- .../onnxruntime/src/vocab.h | 4 +- runtime/onnxruntime/src/wfst-decoder.cpp | 114 + runtime/onnxruntime/src/wfst-decoder.h | 86 + .../third_party/download_ffmpeg.sh | 0 .../third_party/download_onnxruntime.sh | 0 .../third_party/gflags/.gitattributes | 0 .../third_party/gflags/.gitmodules | 0 .../third_party/gflags/.travis.yml | 0 .../third_party/gflags/AUTHORS.txt | 0 .../third_party/gflags/CMakeLists.txt | 0 .../third_party/gflags/COPYING.txt | 0 .../third_party/gflags/ChangeLog.txt | 0 .../onnxruntime/third_party/gflags/INSTALL.md | 0 .../onnxruntime/third_party/gflags/README.md | 0 .../onnxruntime/third_party/gflags/WORKSPACE | 0 .../third_party/gflags/appveyor.yml | 0 .../third_party/gflags/bazel/gflags.bzl | 0 .../gflags/cmake/README_runtime.txt | 0 .../gflags/cmake/cmake_uninstall.cmake.in | 0 .../third_party/gflags/cmake/config.cmake.in | 0 .../gflags/cmake/execute_test.cmake | 0 .../third_party/gflags/cmake/package.cmake.in | 0 .../third_party/gflags/cmake/package.pc.in | 0 .../third_party/gflags/cmake/utils.cmake | 0 .../third_party/gflags/cmake/version.cmake.in | 0 .../third_party/gflags/src/config.h | 0 .../third_party/gflags/src/defines.h.in | 0 .../third_party/gflags/src/gflags.cc | 0 .../third_party/gflags/src/gflags.h.in | 0 .../gflags/src/gflags_completions.cc | 0 .../gflags/src/gflags_completions.h.in | 0 .../gflags/src/gflags_completions.sh | 0 .../gflags/src/gflags_declare.h.in | 0 .../third_party/gflags/src/gflags_ns.h.in | 0 .../gflags/src/gflags_reporting.cc | 0 .../third_party/gflags/src/mutex.h | 0 .../onnxruntime/third_party/gflags/src/util.h | 0 .../third_party/gflags/src/windows_port.cc | 0 .../third_party/gflags/src/windows_port.h | 0 .../third_party/gflags/test/CMakeLists.txt | 0 .../gflags/test/config/CMakeLists.txt | 0 .../third_party/gflags/test/config/main.cc | 0 .../third_party/gflags/test/flagfile.1 | 0 .../third_party/gflags/test/flagfile.2 | 0 .../third_party/gflags/test/flagfile.3 | 0 .../gflags/test/gflags_build.py.in | 0 .../gflags/test/gflags_declare_flags.cc | 0 .../gflags/test/gflags_declare_test.cc | 0 .../gflags/test/gflags_strip_flags_test.cc | 0 .../gflags/test/gflags_strip_flags_test.cmake | 0 .../gflags/test/gflags_unittest.cc | 0 .../gflags/test/gflags_unittest_flagfile | 0 .../third_party/gflags/test/nc/CMakeLists.txt | 0 .../third_party/gflags/test/nc/gflags_nc.cc | 0 .../third_party/glog/CMakeLists.txt | 0 .../glog/cmake/DetermineGflagsNamespace.cmake | 0 .../third_party/glog/cmake/FindUnwind.cmake | 0 .../glog/cmake/GetCacheVariables.cmake | 0 .../glog/cmake/RunCleanerTest1.cmake | 0 .../glog/cmake/RunCleanerTest2.cmake | 0 .../glog/cmake/RunCleanerTest3.cmake | 0 .../glog/cmake/TestInitPackageConfig.cmake | 0 .../glog/cmake/TestPackageConfig.cmake | 0 .../third_party/glog/glog-config.cmake.in | 0 .../third_party/glog/glog-modules.cmake.in | 0 .../third_party/glog/libglog.pc.in | 0 .../glog/src/base/commandlineflags.h | 0 .../third_party/glog/src/base/googleinit.h | 0 .../third_party/glog/src/base/mutex.h | 0 .../onnxruntime/third_party/glog/src/config.h | 0 .../third_party/glog/src/config.h.cmake.in | 0 .../third_party/glog/src/demangle.cc | 0 .../third_party/glog/src/demangle.h | 0 .../third_party/glog/src/fuzz_demangle.cc | 0 .../third_party/glog/src/glog/export.h | 0 .../third_party/glog/src/glog/log_severity.h | 0 .../third_party/glog/src/glog/logging.h | 0 .../third_party/glog/src/glog/logging.h.in | 0 .../third_party/glog/src/glog/platform.h | 0 .../third_party/glog/src/glog/raw_logging.h | 0 .../glog/src/glog/raw_logging.h.in | 0 .../third_party/glog/src/glog/stl_logging.h | 0 .../glog/src/glog/stl_logging.h.in | 0 .../third_party/glog/src/glog/vlog_is_on.h | 0 .../third_party/glog/src/glog/vlog_is_on.h.in | 0 .../third_party/glog/src/logging.cc | 0 .../third_party/glog/src/mock-log.h | 0 .../third_party/glog/src/raw_logging.cc | 0 .../third_party/glog/src/signalhandler.cc | 0 .../third_party/glog/src/stacktrace.h | 0 .../glog/src/stacktrace_generic-inl.h | 0 .../glog/src/stacktrace_libunwind-inl.h | 0 .../glog/src/stacktrace_powerpc-inl.h | 0 .../glog/src/stacktrace_unittest.cc | 0 .../glog/src/stacktrace_unwind-inl.h | 0 .../glog/src/stacktrace_windows-inl.h | 0 .../third_party/glog/src/stacktrace_x86-inl.h | 0 .../third_party/glog/src/symbolize.cc | 0 .../third_party/glog/src/symbolize.h | 0 .../third_party/glog/src/utilities.cc | 0 .../third_party/glog/src/utilities.h | 0 .../third_party/glog/src/vlog_is_on.cc | 0 .../jieba/include/cppjieba/DictTrie.hpp | 0 .../jieba/include/cppjieba/FullSegment.hpp | 0 .../jieba/include/cppjieba/HMMModel.hpp | 0 .../jieba/include/cppjieba/HMMSegment.hpp | 0 .../jieba/include/cppjieba/Jieba.hpp | 0 .../include/cppjieba/KeywordExtractor.hpp | 0 .../jieba/include/cppjieba/MPSegment.hpp | 0 .../jieba/include/cppjieba/MixSegment.hpp | 0 .../jieba/include/cppjieba/PosTagger.hpp | 0 .../jieba/include/cppjieba/PreFilter.hpp | 0 .../jieba/include/cppjieba/QuerySegment.hpp | 0 .../jieba/include/cppjieba/SegmentBase.hpp | 0 .../jieba/include/cppjieba/SegmentTagged.hpp | 0 .../include/cppjieba/TextRankExtractor.hpp | 0 .../jieba/include/cppjieba/Trie.hpp | 0 .../jieba/include/cppjieba/Unicode.hpp | 0 .../jieba/include/limonp/ArgvContext.hpp | 0 .../jieba/include/limonp/BlockingQueue.hpp | 0 .../include/limonp/BoundedBlockingQueue.hpp | 0 .../jieba/include/limonp/BoundedQueue.hpp | 0 .../jieba/include/limonp/Closure.hpp | 0 .../jieba/include/limonp/Colors.hpp | 0 .../jieba/include/limonp/Condition.hpp | 0 .../jieba/include/limonp/Config.hpp | 0 .../jieba/include/limonp/FileLock.hpp | 0 .../jieba/include/limonp/ForcePublic.hpp | 0 .../jieba/include/limonp/LocalVector.hpp | 0 .../jieba/include/limonp/Logging.hpp | 0 .../third_party/jieba/include/limonp/Md5.hpp | 0 .../jieba/include/limonp/MutexLock.hpp | 0 .../jieba/include/limonp/NonCopyable.hpp | 0 .../jieba/include/limonp/StdExtension.hpp | 0 .../jieba/include/limonp/StringUtil.hpp | 0 .../jieba/include/limonp/Thread.hpp | 0 .../jieba/include/limonp/ThreadPool.hpp | 0 .../kaldi-native-fbank/CMakeLists.txt | 0 .../cmake/Modules/FetchContent.cmake | 0 .../Modules/FetchContent/CMakeLists.cmake.in | 0 .../cmake/Modules/README.md | 0 .../cmake/cmake_extension.py | 0 .../kaldi-native-fbank/cmake/googletest.cmake | 0 .../kaldi-native-fbank/cmake/pybind11.cmake | 0 .../kaldi-native-fbank/CMakeLists.txt | 0 .../kaldi-native-fbank/csrc/CMakeLists.txt | 0 .../csrc/CMakeLists.txt.bak | 0 .../kaldi-native-fbank/csrc/feature-fbank.cc | 0 .../kaldi-native-fbank/csrc/feature-fbank.h | 0 .../csrc/feature-functions.cc | 0 .../csrc/feature-functions.h | 0 .../kaldi-native-fbank/csrc/feature-window.cc | 0 .../kaldi-native-fbank/csrc/feature-window.h | 0 .../kaldi-native-fbank/csrc/fftsg.c | 0 .../kaldi-native-fbank/csrc/log.cc | 0 .../kaldi-native-fbank/csrc/log.h | 0 .../csrc/mel-computations.cc | 0 .../csrc/mel-computations.h | 0 .../kaldi-native-fbank/csrc/online-feature.cc | 0 .../kaldi-native-fbank/csrc/online-feature.h | 0 .../kaldi-native-fbank/csrc/rfft.cc | 0 .../kaldi-native-fbank/csrc/rfft.h | 0 .../kaldi-native-fbank/csrc/test-log.cc | 0 .../csrc/test-online-fbank.cc | 0 .../csrc/test-online-feature.cc | 0 .../kaldi-native-fbank/csrc/test-rfft.cc | 0 .../third_party/kaldi/CMakeLists.txt | 51 + .../third_party/kaldi/base/Makefile | 27 + .../third_party/kaldi/base/get_version.sh | 94 + .../third_party/kaldi/base/io-funcs-inl.h | 327 +++ .../third_party/kaldi/base/io-funcs-test.cc | 160 ++ .../third_party/kaldi/base/io-funcs.cc | 218 ++ .../third_party/kaldi/base/io-funcs.h | 245 +++ .../third_party/kaldi/base/kaldi-common.h | 41 + .../kaldi/base/kaldi-error-test.cc | 82 + .../third_party/kaldi/base/kaldi-error.cc | 248 +++ .../third_party/kaldi/base/kaldi-error.h | 231 ++ .../third_party/kaldi/base/kaldi-math-test.cc | 334 +++ .../third_party/kaldi/base/kaldi-math.cc | 162 ++ .../third_party/kaldi/base/kaldi-math.h | 363 ++++ .../third_party/kaldi/base/kaldi-types.h | 75 + .../third_party/kaldi/base/kaldi-utils.cc | 46 + .../third_party/kaldi/base/kaldi-utils.h | 162 ++ .../third_party/kaldi/base/timer-test.cc | 45 + .../third_party/kaldi/base/timer.cc | 85 + .../third_party/kaldi/base/timer.h | 115 + .../third_party/kaldi/decoder/Makefile | 20 + .../kaldi/decoder/biglm-faster-decoder.h | 502 +++++ .../kaldi/decoder/decodable-mapped.h | 69 + .../kaldi/decoder/decodable-matrix.cc | 112 + .../kaldi/decoder/decodable-matrix.h | 253 +++ .../third_party/kaldi/decoder/decodable-sum.h | 109 + .../kaldi/decoder/decoder-wrappers.cc | 665 ++++++ .../kaldi/decoder/decoder-wrappers.h | 221 ++ .../kaldi/decoder/faster-decoder.cc | 351 +++ .../kaldi/decoder/faster-decoder.h | 195 ++ .../decoder/lattice-biglm-faster-decoder.h | 886 ++++++++ .../kaldi/decoder/lattice-faster-decoder.cc | 1061 ++++++++++ .../kaldi/decoder/lattice-faster-decoder.h | 597 ++++++ .../decoder/lattice-faster-online-decoder.cc | 285 +++ .../decoder/lattice-faster-online-decoder.h | 146 ++ .../decoder/lattice-incremental-decoder.cc | 1730 +++++++++++++++ .../decoder/lattice-incremental-decoder.h | 734 +++++++ .../lattice-incremental-online-decoder.cc | 158 ++ .../lattice-incremental-online-decoder.h | 132 ++ .../kaldi/decoder/lattice-simple-decoder.cc | 666 ++++++ .../kaldi/decoder/lattice-simple-decoder.h | 318 +++ .../kaldi/decoder/simple-decoder.cc | 293 +++ .../kaldi/decoder/simple-decoder.h | 156 ++ .../kaldi/decoder/training-graph-compiler.cc | 182 ++ .../kaldi/decoder/training-graph-compiler.h | 117 + .../third_party/kaldi/fstbin/Makefile | 31 + .../kaldi/fstbin/fstaddselfloops.cc | 93 + .../kaldi/fstbin/fstaddsubsequentialloop.cc | 85 + .../kaldi/fstbin/fstcomposecontext.cc | 183 ++ .../third_party/kaldi/fstbin/fstcopy.cc | 78 + .../kaldi/fstbin/fstdeterminizelog.cc | 65 + .../kaldi/fstbin/fstdeterminizestar.cc | 119 ++ .../kaldi/fstbin/fstisstochastic.cc | 85 + .../kaldi/fstbin/fstmakecontextfst.cc | 143 ++ .../kaldi/fstbin/fstmakecontextsyms.cc | 122 ++ .../kaldi/fstbin/fstminimizeencoded.cc | 73 + .../third_party/kaldi/fstbin/fstphicompose.cc | 183 ++ .../kaldi/fstbin/fstpushspecial.cc | 72 + .../third_party/kaldi/fstbin/fstrand.cc | 67 + .../third_party/kaldi/fstbin/fstrmepslocal.cc | 101 + .../third_party/kaldi/fstbin/fstrmsymbols.cc | 193 ++ .../third_party/kaldi/fstbin/fsts-concat.cc | 112 + .../third_party/kaldi/fstbin/fsts-project.cc | 82 + .../kaldi/fstbin/fsts-to-transcripts.cc | 103 + .../third_party/kaldi/fstbin/fsts-union.cc | 100 + .../kaldi/fstbin/fsttablecompose.cc | 202 ++ .../kaldi/fstbin/make-grammar-fst.cc | 167 ++ .../third_party/kaldi/fstext/Makefile | 30 + .../kaldi/fstext/context-fst-test.cc | 256 +++ .../third_party/kaldi/fstext/context-fst.cc | 401 ++++ .../third_party/kaldi/fstext/context-fst.h | 340 +++ .../kaldi/fstext/deterministic-fst-inl.h | 512 +++++ .../kaldi/fstext/deterministic-fst-test.cc | 232 ++ .../kaldi/fstext/deterministic-fst.h | 335 +++ .../kaldi/fstext/determinize-lattice-inl.h | 1296 ++++++++++++ .../kaldi/fstext/determinize-lattice-test.cc | 174 ++ .../kaldi/fstext/determinize-lattice.h | 150 ++ .../kaldi/fstext/determinize-star-inl.h | 1134 ++++++++++ .../kaldi/fstext/determinize-star-test.cc | 508 +++++ .../kaldi/fstext/determinize-star.h | 123 ++ .../kaldi/fstext/epsilon-property-inl.h | 126 ++ .../kaldi/fstext/epsilon-property-test.cc | 58 + .../kaldi/fstext/epsilon-property.h | 60 + .../third_party/kaldi/fstext/factor-inl.h | 284 +++ .../third_party/kaldi/fstext/factor-test.cc | 191 ++ .../third_party/kaldi/fstext/factor.h | 158 ++ .../third_party/kaldi/fstext/fst-test-utils.h | 33 + .../third_party/kaldi/fstext/fstext-lib.h | 36 + .../kaldi/fstext/fstext-utils-inl.h | 1284 +++++++++++ .../kaldi/fstext/fstext-utils-test.cc | 434 ++++ .../third_party/kaldi/fstext/fstext-utils.h | 427 ++++ .../kaldi/fstext/grammar-context-fst.cc | 228 ++ .../kaldi/fstext/grammar-context-fst.h | 287 +++ .../kaldi/fstext/kaldi-fst-io-inl.h | 211 ++ .../third_party/kaldi/fstext/kaldi-fst-io.cc | 145 ++ .../third_party/kaldi/fstext/kaldi-fst-io.h | 158 ++ .../kaldi/fstext/lattice-utils-inl.h | 282 +++ .../kaldi/fstext/lattice-utils-test.cc | 331 +++ .../third_party/kaldi/fstext/lattice-utils.h | 256 +++ .../kaldi/fstext/lattice-weight-test.cc | 197 ++ .../third_party/kaldi/fstext/lattice-weight.h | 862 ++++++++ .../kaldi/fstext/pre-determinize-inl.h | 728 +++++++ .../kaldi/fstext/pre-determinize-test.cc | 220 ++ .../kaldi/fstext/pre-determinize.h | 96 + .../kaldi/fstext/prune-special-inl.h | 173 ++ .../kaldi/fstext/prune-special-test.cc | 79 + .../third_party/kaldi/fstext/prune-special.h | 60 + .../kaldi/fstext/push-special-test.cc | 81 + .../third_party/kaldi/fstext/push-special.cc | 269 +++ .../third_party/kaldi/fstext/push-special.h | 42 + .../third_party/kaldi/fstext/rand-fst.h | 162 ++ .../kaldi/fstext/remove-eps-local-inl.h | 321 +++ .../kaldi/fstext/remove-eps-local-test.cc | 178 ++ .../kaldi/fstext/remove-eps-local.h | 60 + .../kaldi/fstext/table-matcher-test.cc | 257 +++ .../third_party/kaldi/fstext/table-matcher.h | 376 ++++ .../fstext/trivial-factor-weight-test.cc | 214 ++ .../kaldi/fstext/trivial-factor-weight.h | 401 ++++ .../third_party/kaldi/itf/clusterable-itf.h | 97 + .../third_party/kaldi/itf/context-dep-itf.h | 114 + .../third_party/kaldi/itf/decodable-itf.h | 122 ++ .../kaldi/itf/online-feature-itf.h | 125 ++ .../third_party/kaldi/itf/optimizable-itf.h | 51 + .../third_party/kaldi/itf/options-itf.h | 49 + .../kaldi/itf/transition-information.h | 108 + .../third_party/kaldi/lat/Makefile | 24 + .../third_party/kaldi/lat/arctic-weight.h | 153 ++ .../kaldi/lat/compose-lattice-pruned.cc | 955 +++++++++ .../kaldi/lat/compose-lattice-pruned.h | 179 ++ .../third_party/kaldi/lat/confidence.cc | 117 + .../third_party/kaldi/lat/confidence.h | 77 + .../lat/determinize-lattice-pruned-test.cc | 147 ++ .../kaldi/lat/determinize-lattice-pruned.cc | 1541 ++++++++++++++ .../kaldi/lat/determinize-lattice-pruned.h | 296 +++ .../kaldi/lat/kaldi-lattice-test.cc | 173 ++ .../third_party/kaldi/lat/kaldi-lattice.cc | 506 +++++ .../third_party/kaldi/lat/kaldi-lattice.h | 156 ++ .../lat/lattice-functions-transition-model.cc | 262 +++ .../lat/lattice-functions-transition-model.h | 104 + .../kaldi/lat/lattice-functions.cc | 1880 +++++++++++++++++ .../third_party/kaldi/lat/lattice-functions.h | 455 ++++ .../kaldi/lat/phone-align-lattice.cc | 421 ++++ .../kaldi/lat/phone-align-lattice.h | 68 + .../kaldi/lat/push-lattice-test.cc | 120 ++ .../third_party/kaldi/lat/push-lattice.cc | 290 +++ .../third_party/kaldi/lat/push-lattice.h | 61 + .../third_party/kaldi/lat/sausages.cc | 432 ++++ .../third_party/kaldi/lat/sausages.h | 270 +++ .../lat/word-align-lattice-lexicon-test.cc | 250 +++ .../kaldi/lat/word-align-lattice-lexicon.cc | 935 ++++++++ .../kaldi/lat/word-align-lattice-lexicon.h | 160 ++ .../kaldi/lat/word-align-lattice.cc | 927 ++++++++ .../kaldi/lat/word-align-lattice.h | 211 ++ .../onnxruntime/third_party/kaldi/lm/Makefile | 25 + .../onnxruntime/third_party/kaldi/lm/README | 108 + .../kaldi/lm/arpa-file-parser-test.cc | 373 ++++ .../third_party/kaldi/lm/arpa-file-parser.cc | 281 +++ .../third_party/kaldi/lm/arpa-file-parser.h | 146 ++ .../kaldi/lm/arpa-lm-compiler-test.cc | 250 +++ .../third_party/kaldi/lm/arpa-lm-compiler.cc | 377 ++++ .../third_party/kaldi/lm/arpa-lm-compiler.h | 65 + .../third_party/kaldi/lm/const-arpa-lm.cc | 1085 ++++++++++ .../third_party/kaldi/lm/const-arpa-lm.h | 430 ++++ .../kaldi/lm/examples/CREATURE.txt | 17 + .../kaldi/lm/examples/DAYOFMONTH.txt | 5 + .../third_party/kaldi/lm/examples/MONTH.txt | 5 + .../third_party/kaldi/lm/examples/YEAR.txt | 4 + .../kaldi/lm/examples/YEARDATE.txt | 1 + .../third_party/kaldi/lm/examples/input3.fst | 0 .../third_party/kaldi/lm/examples/input3.txt | 1 + .../third_party/kaldi/lm/examples/input4.fst | 0 .../third_party/kaldi/lm/examples/input4.txt | 1 + .../third_party/kaldi/lm/irstlm.mk | 7 + .../third_party/kaldi/lm/kaldi-rnnlm.cc | 140 ++ .../third_party/kaldi/lm/kaldi-rnnlm.h | 104 + .../third_party/kaldi/lm/kenlm-test.cc | 84 + .../onnxruntime/third_party/kaldi/lm/kenlm.cc | 99 + .../onnxruntime/third_party/kaldi/lm/kenlm.h | 207 ++ .../third_party/kaldi/lm/mikolov-rnnlm-lib.cc | 1232 +++++++++++ .../third_party/kaldi/lm/mikolov-rnnlm-lib.h | 239 +++ .../third_party/kaldi/lm/test_data/input.arpa | 21 + .../kaldi/lm/test_data/missing_backoffs.arpa | 20 + .../kaldi/lm/test_data/missing_bos.arpa | 18 + .../kaldi/lm/test_data/unused_backoffs.arpa | 26 + .../third_party/kaldi/lmbin/Makefile | 16 + .../kaldi/lmbin/arpa-to-const-arpa.cc | 88 + .../third_party/kaldi/lmbin/arpa2fst.cc | 149 ++ .../third_party/kaldi/util/Makefile | 20 + .../third_party/kaldi/util/basic-filebuf.h | 994 +++++++++ .../kaldi/util/const-integer-set-inl.h | 91 + .../kaldi/util/const-integer-set-test.cc | 145 ++ .../kaldi/util/const-integer-set.h | 96 + .../kaldi/util/edit-distance-inl.h | 200 ++ .../kaldi/util/edit-distance-test.cc | 267 +++ .../third_party/kaldi/util/edit-distance.h | 64 + .../third_party/kaldi/util/hash-list-inl.h | 194 ++ .../third_party/kaldi/util/hash-list-test.cc | 107 + .../third_party/kaldi/util/hash-list.h | 147 ++ .../kaldi/util/kaldi-cygwin-io-inl.h | 129 ++ .../third_party/kaldi/util/kaldi-io-inl.h | 46 + .../third_party/kaldi/util/kaldi-io-test.cc | 370 ++++ .../third_party/kaldi/util/kaldi-io.cc | 880 ++++++++ .../third_party/kaldi/util/kaldi-io.h | 280 +++ .../third_party/kaldi/util/kaldi-pipebuf.h | 87 + .../third_party/kaldi/util/kaldi-semaphore.cc | 57 + .../third_party/kaldi/util/kaldi-semaphore.h | 50 + .../kaldi/util/kaldi-thread-test.cc | 133 ++ .../third_party/kaldi/util/kaldi-thread.cc | 33 + .../third_party/kaldi/util/kaldi-thread.h | 290 +++ .../kaldi/util/parse-options-test.cc | 319 +++ .../third_party/kaldi/util/parse-options.cc | 669 ++++++ .../third_party/kaldi/util/parse-options.h | 264 +++ .../third_party/kaldi/util/simple-io-funcs.cc | 81 + .../third_party/kaldi/util/simple-io-funcs.h | 63 + .../kaldi/util/simple-options-test.cc | 86 + .../third_party/kaldi/util/simple-options.cc | 184 ++ .../third_party/kaldi/util/simple-options.h | 113 + .../third_party/kaldi/util/stl-utils-test.cc | 275 +++ .../third_party/kaldi/util/stl-utils.h | 317 +++ .../third_party/kaldi/util/table-types.h | 207 ++ .../third_party/kaldi/util/text-utils-test.cc | 538 +++++ .../third_party/kaldi/util/text-utils.cc | 591 ++++++ .../third_party/kaldi/util/text-utils.h | 281 +++ .../third_party/openfst/.editorconfig | 0 .../third_party/openfst/.gitignore | 0 .../third_party/openfst/BUILD.bazel | 0 .../third_party/openfst/CMakeLists.txt | 4 +- .../third_party/openfst/Makefile.am | 0 .../third_party/openfst/Makefile.in | 0 .../onnxruntime/third_party/openfst/README.md | 0 .../third_party/openfst/_import_release.sh | 0 .../third_party/openfst/_package_release.cmd | 0 .../third_party/openfst/aclocal.m4 | 0 .../third_party/openfst/config.guess | 0 .../third_party/openfst/config.h.in | 0 .../third_party/openfst/config.sub | 0 .../third_party/openfst/configure.ac | 0 .../onnxruntime/third_party/openfst/ltmain.sh | 0 .../third_party/openfst/m4/ac_python_devel.m4 | 0 .../third_party/openfst/m4/libtool.m4 | 0 .../third_party/openfst/m4/ltoptions.m4 | 0 .../third_party/openfst/m4/ltsugar.m4 | 0 .../third_party/openfst/m4/ltversion.m4 | 0 .../third_party/openfst/m4/lt~obsolete.m4 | 0 .../third_party/openfst/openfst.sln | 0 .../third_party/openfst/src/CMakeLists.txt | 0 .../third_party/openfst/src/Makefile.am | 0 .../third_party/openfst/src/Makefile.in | 0 .../openfst/src/bin/CMakeLists.txt | 0 .../third_party/openfst/src/bin/Makefile.am | 0 .../third_party/openfst/src/bin/Makefile.in | 0 .../third_party/openfst/src/bin/bin.vcxproj | 0 .../openfst/src/bin/fstarcsort-main.cc | 0 .../third_party/openfst/src/bin/fstarcsort.cc | 0 .../openfst/src/bin/fstclosure-main.cc | 0 .../third_party/openfst/src/bin/fstclosure.cc | 0 .../openfst/src/bin/fstcompile-main.cc | 0 .../third_party/openfst/src/bin/fstcompile.cc | 0 .../openfst/src/bin/fstcompose-main.cc | 0 .../third_party/openfst/src/bin/fstcompose.cc | 0 .../openfst/src/bin/fstconcat-main.cc | 0 .../third_party/openfst/src/bin/fstconcat.cc | 0 .../openfst/src/bin/fstconnect-main.cc | 0 .../third_party/openfst/src/bin/fstconnect.cc | 0 .../openfst/src/bin/fstconvert-main.cc | 0 .../third_party/openfst/src/bin/fstconvert.cc | 0 .../openfst/src/bin/fstdeterminize-main.cc | 0 .../openfst/src/bin/fstdeterminize.cc | 0 .../openfst/src/bin/fstdifference-main.cc | 0 .../openfst/src/bin/fstdifference.cc | 0 .../openfst/src/bin/fstdisambiguate-main.cc | 0 .../openfst/src/bin/fstdisambiguate.cc | 0 .../openfst/src/bin/fstdraw-main.cc | 0 .../third_party/openfst/src/bin/fstdraw.cc | 0 .../openfst/src/bin/fstencode-main.cc | 0 .../third_party/openfst/src/bin/fstencode.cc | 0 .../openfst/src/bin/fstepsnormalize-main.cc | 0 .../openfst/src/bin/fstepsnormalize.cc | 0 .../openfst/src/bin/fstequal-main.cc | 0 .../third_party/openfst/src/bin/fstequal.cc | 0 .../openfst/src/bin/fstequivalent-main.cc | 0 .../openfst/src/bin/fstequivalent.cc | 0 .../openfst/src/bin/fstinfo-main.cc | 0 .../third_party/openfst/src/bin/fstinfo.cc | 0 .../openfst/src/bin/fstintersect-main.cc | 0 .../openfst/src/bin/fstintersect.cc | 0 .../openfst/src/bin/fstinvert-main.cc | 0 .../third_party/openfst/src/bin/fstinvert.cc | 0 .../openfst/src/bin/fstisomorphic-main.cc | 0 .../openfst/src/bin/fstisomorphic.cc | 0 .../openfst/src/bin/fstmap-main.cc | 0 .../third_party/openfst/src/bin/fstmap.cc | 0 .../openfst/src/bin/fstminimize-main.cc | 0 .../openfst/src/bin/fstminimize.cc | 0 .../openfst/src/bin/fstprint-main.cc | 0 .../third_party/openfst/src/bin/fstprint.cc | 0 .../openfst/src/bin/fstproject-main.cc | 0 .../third_party/openfst/src/bin/fstproject.cc | 0 .../openfst/src/bin/fstprune-main.cc | 0 .../third_party/openfst/src/bin/fstprune.cc | 0 .../openfst/src/bin/fstpush-main.cc | 0 .../third_party/openfst/src/bin/fstpush.cc | 0 .../openfst/src/bin/fstrandgen-main.cc | 0 .../third_party/openfst/src/bin/fstrandgen.cc | 0 .../openfst/src/bin/fstrelabel-main.cc | 0 .../third_party/openfst/src/bin/fstrelabel.cc | 0 .../openfst/src/bin/fstreplace-main.cc | 0 .../third_party/openfst/src/bin/fstreplace.cc | 0 .../openfst/src/bin/fstreverse-main.cc | 0 .../third_party/openfst/src/bin/fstreverse.cc | 0 .../openfst/src/bin/fstreweight-main.cc | 0 .../openfst/src/bin/fstreweight.cc | 0 .../openfst/src/bin/fstrmepsilon-main.cc | 0 .../openfst/src/bin/fstrmepsilon.cc | 0 .../src/bin/fstshortestdistance-main.cc | 0 .../openfst/src/bin/fstshortestdistance.cc | 0 .../openfst/src/bin/fstshortestpath-main.cc | 0 .../openfst/src/bin/fstshortestpath.cc | 0 .../openfst/src/bin/fstsymbols-main.cc | 0 .../third_party/openfst/src/bin/fstsymbols.cc | 0 .../openfst/src/bin/fstsynchronize-main.cc | 0 .../openfst/src/bin/fstsynchronize.cc | 0 .../openfst/src/bin/fsttopsort-main.cc | 0 .../third_party/openfst/src/bin/fsttopsort.cc | 0 .../openfst/src/bin/fstunion-main.cc | 0 .../third_party/openfst/src/bin/fstunion.cc | 0 .../openfst/src/extensions/CMakeLists.txt | 0 .../openfst/src/extensions/Makefile.am | 0 .../openfst/src/extensions/Makefile.in | 0 .../src/extensions/compact/CMakeLists.txt | 0 .../src/extensions/compact/Makefile.am | 0 .../src/extensions/compact/Makefile.in | 0 .../compact/compact16_acceptor-fst.cc | 0 .../compact/compact16_string-fst.cc | 0 .../compact/compact16_unweighted-fst.cc | 0 .../compact16_unweighted_acceptor-fst.cc | 0 .../compact/compact16_weighted_string-fst.cc | 0 .../compact/compact64_acceptor-fst.cc | 0 .../compact/compact64_string-fst.cc | 0 .../compact/compact64_unweighted-fst.cc | 0 .../compact64_unweighted_acceptor-fst.cc | 0 .../compact/compact64_weighted_string-fst.cc | 0 .../compact/compact8_acceptor-fst.cc | 0 .../extensions/compact/compact8_string-fst.cc | 0 .../compact/compact8_unweighted-fst.cc | 0 .../compact8_unweighted_acceptor-fst.cc | 0 .../compact/compact8_weighted_string-fst.cc | 0 .../src/extensions/compress/CMakeLists.txt | 0 .../src/extensions/compress/Makefile.am | 0 .../src/extensions/compress/Makefile.in | 0 .../extensions/compress/compress-script.cc | 0 .../src/extensions/compress/fstcompress.cc | 0 .../src/extensions/compress/fstrandmod.cc | 0 .../src/extensions/const/CMakeLists.txt | 0 .../openfst/src/extensions/const/Makefile.am | 0 .../openfst/src/extensions/const/Makefile.in | 0 .../src/extensions/const/const16-fst.cc | 0 .../src/extensions/const/const64-fst.cc | 0 .../src/extensions/const/const8-fst.cc | 0 .../openfst/src/extensions/far/CMakeLists.txt | 0 .../openfst/src/extensions/far/Makefile.am | 0 .../openfst/src/extensions/far/Makefile.in | 0 .../openfst/src/extensions/far/far-class.cc | 0 .../src/extensions/far/farcompilestrings.cc | 0 .../openfst/src/extensions/far/farcreate.cc | 0 .../openfst/src/extensions/far/farequal.cc | 0 .../openfst/src/extensions/far/farextract.cc | 0 .../openfst/src/extensions/far/farinfo.cc | 0 .../src/extensions/far/farisomorphic.cc | 0 .../src/extensions/far/farprintstrings.cc | 0 .../openfst/src/extensions/far/farscript.cc | 0 .../openfst/src/extensions/far/getters.cc | 0 .../openfst/src/extensions/far/script-impl.cc | 0 .../openfst/src/extensions/far/stlist.cc | 0 .../openfst/src/extensions/far/strings.cc | 0 .../openfst/src/extensions/far/sttable.cc | 0 .../src/extensions/linear/CMakeLists.txt | 0 .../openfst/src/extensions/linear/Makefile.am | 0 .../openfst/src/extensions/linear/Makefile.in | 0 .../src/extensions/linear/fstlinear.cc | 0 .../extensions/linear/fstloglinearapply.cc | 0 .../linear/linear-classifier-fst.cc | 0 .../extensions/linear/linear-tagger-fst.cc | 0 .../src/extensions/linear/linearscript.cc | 0 .../src/extensions/lookahead/CMakeLists.txt | 0 .../src/extensions/lookahead/Makefile.am | 0 .../src/extensions/lookahead/Makefile.in | 0 .../extensions/lookahead/arc_lookahead-fst.cc | 0 .../lookahead/ilabel_lookahead-fst.cc | 0 .../lookahead/olabel_lookahead-fst.cc | 0 .../src/extensions/mpdt/CMakeLists.txt | 0 .../openfst/src/extensions/mpdt/Makefile.am | 0 .../openfst/src/extensions/mpdt/Makefile.in | 0 .../src/extensions/mpdt/mpdtcompose.cc | 0 .../openfst/src/extensions/mpdt/mpdtexpand.cc | 0 .../openfst/src/extensions/mpdt/mpdtinfo.cc | 0 .../src/extensions/mpdt/mpdtreverse.cc | 0 .../openfst/src/extensions/mpdt/mpdtscript.cc | 0 .../src/extensions/ngram/CMakeLists.txt | 0 .../openfst/src/extensions/ngram/Makefile.am | 0 .../openfst/src/extensions/ngram/Makefile.in | 0 .../src/extensions/ngram/bitmap-index.cc | 0 .../openfst/src/extensions/ngram/ngram-fst.cc | 0 .../openfst/src/extensions/ngram/nthbit.cc | 0 .../openfst/src/extensions/pdt/CMakeLists.txt | 0 .../openfst/src/extensions/pdt/Makefile.am | 0 .../openfst/src/extensions/pdt/Makefile.in | 0 .../openfst/src/extensions/pdt/getters.cc | 0 .../openfst/src/extensions/pdt/pdtcompose.cc | 0 .../openfst/src/extensions/pdt/pdtexpand.cc | 0 .../openfst/src/extensions/pdt/pdtinfo.cc | 0 .../openfst/src/extensions/pdt/pdtreplace.cc | 0 .../openfst/src/extensions/pdt/pdtreverse.cc | 0 .../openfst/src/extensions/pdt/pdtscript.cc | 0 .../src/extensions/pdt/pdtshortestpath.cc | 0 .../openfst/src/extensions/python/Makefile.am | 0 .../openfst/src/extensions/python/Makefile.in | 0 .../src/extensions/python/basictypes.pxd | 0 .../openfst/src/extensions/python/fst.pxd | 0 .../openfst/src/extensions/python/ios.pxd | 0 .../openfst/src/extensions/python/memory.pxd | 0 .../src/extensions/python/pywrapfst.cc | 0 .../src/extensions/python/pywrapfst.pxd | 0 .../src/extensions/python/pywrapfst.pyx | 0 .../src/extensions/special/CMakeLists.txt | 0 .../src/extensions/special/Makefile.am | 0 .../src/extensions/special/Makefile.in | 0 .../src/extensions/special/fstspecial.cc | 0 .../openfst/src/extensions/special/phi-fst.cc | 0 .../openfst/src/extensions/special/rho-fst.cc | 0 .../src/extensions/special/sigma-fst.cc | 0 .../openfst/src/include/Makefile.am | 0 .../openfst/src/include/Makefile.in | 0 .../openfst/src/include/fst/accumulator.h | 0 .../openfst/src/include/fst/add-on.h | 0 .../openfst/src/include/fst/arc-arena.h | 0 .../openfst/src/include/fst/arc-map.h | 0 .../third_party/openfst/src/include/fst/arc.h | 0 .../openfst/src/include/fst/arcfilter.h | 0 .../openfst/src/include/fst/arcsort.h | 0 .../openfst/src/include/fst/bi-table.h | 0 .../openfst/src/include/fst/cache.h | 0 .../openfst/src/include/fst/closure.h | 0 .../openfst/src/include/fst/compact-fst.h | 0 .../openfst/src/include/fst/compat.h | 0 .../openfst/src/include/fst/complement.h | 0 .../openfst/src/include/fst/compose-filter.h | 0 .../openfst/src/include/fst/compose.h | 0 .../openfst/src/include/fst/concat.h | 0 .../openfst/src/include/fst/config.h | 0 .../openfst/src/include/fst/config.h.in | 0 .../openfst/src/include/fst/connect.h | 0 .../openfst/src/include/fst/const-fst.h | 0 .../openfst/src/include/fst/determinize.h | 0 .../openfst/src/include/fst/dfs-visit.h | 0 .../openfst/src/include/fst/difference.h | 0 .../openfst/src/include/fst/disambiguate.h | 0 .../openfst/src/include/fst/edit-fst.h | 0 .../openfst/src/include/fst/encode.h | 0 .../openfst/src/include/fst/epsnormalize.h | 0 .../openfst/src/include/fst/equal.h | 0 .../openfst/src/include/fst/equivalent.h | 0 .../openfst/src/include/fst/expanded-fst.h | 0 .../src/include/fst/expectation-weight.h | 0 .../fst/extensions/compress/compress-script.h | 0 .../fst/extensions/compress/compress.h | 0 .../include/fst/extensions/compress/elias.h | 0 .../include/fst/extensions/compress/gzfile.h | 0 .../include/fst/extensions/compress/randmod.h | 0 .../fst/extensions/far/compile-strings.h | 0 .../src/include/fst/extensions/far/create.h | 0 .../src/include/fst/extensions/far/equal.h | 0 .../src/include/fst/extensions/far/extract.h | 0 .../include/fst/extensions/far/far-class.h | 0 .../src/include/fst/extensions/far/far.h | 0 .../src/include/fst/extensions/far/farlib.h | 0 .../include/fst/extensions/far/farscript.h | 0 .../src/include/fst/extensions/far/getters.h | 0 .../src/include/fst/extensions/far/info.h | 0 .../include/fst/extensions/far/isomorphic.h | 0 .../fst/extensions/far/print-strings.h | 0 .../include/fst/extensions/far/script-impl.h | 0 .../src/include/fst/extensions/far/stlist.h | 0 .../src/include/fst/extensions/far/sttable.h | 0 .../linear/linear-fst-data-builder.h | 0 .../fst/extensions/linear/linear-fst-data.h | 0 .../fst/extensions/linear/linear-fst.h | 0 .../fst/extensions/linear/linearscript.h | 0 .../fst/extensions/linear/loglinear-apply.h | 0 .../src/include/fst/extensions/linear/trie.h | 0 .../src/include/fst/extensions/mpdt/compose.h | 0 .../src/include/fst/extensions/mpdt/expand.h | 0 .../src/include/fst/extensions/mpdt/info.h | 0 .../src/include/fst/extensions/mpdt/mpdt.h | 0 .../src/include/fst/extensions/mpdt/mpdtlib.h | 0 .../include/fst/extensions/mpdt/mpdtscript.h | 0 .../fst/extensions/mpdt/read_write_utils.h | 0 .../src/include/fst/extensions/mpdt/reverse.h | 0 .../fst/extensions/ngram/bitmap-index.h | 0 .../include/fst/extensions/ngram/ngram-fst.h | 0 .../src/include/fst/extensions/ngram/nthbit.h | 0 .../include/fst/extensions/pdt/collection.h | 0 .../src/include/fst/extensions/pdt/compose.h | 0 .../src/include/fst/extensions/pdt/expand.h | 0 .../src/include/fst/extensions/pdt/getters.h | 0 .../src/include/fst/extensions/pdt/info.h | 0 .../src/include/fst/extensions/pdt/paren.h | 0 .../src/include/fst/extensions/pdt/pdt.h | 0 .../src/include/fst/extensions/pdt/pdtlib.h | 0 .../include/fst/extensions/pdt/pdtscript.h | 0 .../src/include/fst/extensions/pdt/replace.h | 0 .../src/include/fst/extensions/pdt/reverse.h | 0 .../fst/extensions/pdt/shortest-path.h | 0 .../include/fst/extensions/special/phi-fst.h | 0 .../include/fst/extensions/special/rho-fst.h | 0 .../fst/extensions/special/sigma-fst.h | 0 .../openfst/src/include/fst/factor-weight.h | 0 .../openfst/src/include/fst/filter-state.h | 0 .../openfst/src/include/fst/flags.h | 0 .../openfst/src/include/fst/float-weight.h | 0 .../openfst/src/include/fst/fst-decl.h | 0 .../third_party/openfst/src/include/fst/fst.h | 0 .../openfst/src/include/fst/fstlib.h | 0 .../src/include/fst/generic-register.h | 0 .../openfst/src/include/fst/heap.h | 0 .../third_party/openfst/src/include/fst/icu.h | 0 .../openfst/src/include/fst/intersect.h | 0 .../openfst/src/include/fst/interval-set.h | 0 .../openfst/src/include/fst/invert.h | 0 .../openfst/src/include/fst/isomorphic.h | 0 .../openfst/src/include/fst/label-reachable.h | 0 .../src/include/fst/lexicographic-weight.h | 0 .../openfst/src/include/fst/lock.h | 0 .../third_party/openfst/src/include/fst/log.h | 0 .../src/include/fst/lookahead-filter.h | 0 .../src/include/fst/lookahead-matcher.h | 0 .../third_party/openfst/src/include/fst/map.h | 0 .../openfst/src/include/fst/mapped-file.h | 0 .../openfst/src/include/fst/matcher-fst.h | 0 .../openfst/src/include/fst/matcher.h | 0 .../openfst/src/include/fst/memory.h | 0 .../openfst/src/include/fst/minimize.h | 0 .../openfst/src/include/fst/mutable-fst.h | 0 .../openfst/src/include/fst/pair-weight.h | 0 .../openfst/src/include/fst/partition.h | 0 .../openfst/src/include/fst/power-weight.h | 0 .../openfst/src/include/fst/product-weight.h | 0 .../openfst/src/include/fst/project.h | 0 .../openfst/src/include/fst/properties.h | 0 .../openfst/src/include/fst/prune.h | 0 .../openfst/src/include/fst/push.h | 0 .../openfst/src/include/fst/queue.h | 0 .../openfst/src/include/fst/randequivalent.h | 0 .../openfst/src/include/fst/randgen.h | 0 .../openfst/src/include/fst/rational.h | 0 .../openfst/src/include/fst/register.h | 0 .../openfst/src/include/fst/relabel.h | 0 .../openfst/src/include/fst/replace-util.h | 0 .../openfst/src/include/fst/replace.h | 0 .../openfst/src/include/fst/reverse.h | 0 .../openfst/src/include/fst/reweight.h | 0 .../openfst/src/include/fst/rmepsilon.h | 0 .../openfst/src/include/fst/rmfinalepsilon.h | 0 .../src/include/fst/script/arc-class.h | 0 .../include/fst/script/arciterator-class.h | 0 .../openfst/src/include/fst/script/arcsort.h | 0 .../src/include/fst/script/arg-packs.h | 0 .../openfst/src/include/fst/script/closure.h | 0 .../src/include/fst/script/compile-impl.h | 0 .../openfst/src/include/fst/script/compile.h | 0 .../openfst/src/include/fst/script/compose.h | 0 .../openfst/src/include/fst/script/concat.h | 0 .../openfst/src/include/fst/script/connect.h | 0 .../openfst/src/include/fst/script/convert.h | 0 .../openfst/src/include/fst/script/decode.h | 0 .../src/include/fst/script/determinize.h | 0 .../src/include/fst/script/difference.h | 0 .../src/include/fst/script/disambiguate.h | 0 .../src/include/fst/script/draw-impl.h | 0 .../openfst/src/include/fst/script/draw.h | 0 .../openfst/src/include/fst/script/encode.h | 0 .../include/fst/script/encodemapper-class.h | 0 .../src/include/fst/script/epsnormalize.h | 0 .../openfst/src/include/fst/script/equal.h | 0 .../src/include/fst/script/equivalent.h | 0 .../src/include/fst/script/fst-class.h | 0 .../src/include/fst/script/fstscript-decl.h | 0 .../src/include/fst/script/fstscript.h | 0 .../openfst/src/include/fst/script/getters.h | 0 .../src/include/fst/script/info-impl.h | 0 .../openfst/src/include/fst/script/info.h | 0 .../src/include/fst/script/intersect.h | 0 .../openfst/src/include/fst/script/invert.h | 0 .../src/include/fst/script/isomorphic.h | 0 .../openfst/src/include/fst/script/map.h | 0 .../openfst/src/include/fst/script/minimize.h | 0 .../src/include/fst/script/print-impl.h | 0 .../openfst/src/include/fst/script/print.h | 0 .../openfst/src/include/fst/script/project.h | 0 .../openfst/src/include/fst/script/prune.h | 0 .../openfst/src/include/fst/script/push.h | 0 .../src/include/fst/script/randequivalent.h | 0 .../openfst/src/include/fst/script/randgen.h | 0 .../openfst/src/include/fst/script/register.h | 0 .../openfst/src/include/fst/script/relabel.h | 0 .../openfst/src/include/fst/script/replace.h | 0 .../openfst/src/include/fst/script/reverse.h | 0 .../openfst/src/include/fst/script/reweight.h | 0 .../src/include/fst/script/rmepsilon.h | 0 .../src/include/fst/script/script-impl.h | 0 .../include/fst/script/shortest-distance.h | 0 .../src/include/fst/script/shortest-path.h | 0 .../include/fst/script/stateiterator-class.h | 0 .../src/include/fst/script/synchronize.h | 0 .../openfst/src/include/fst/script/text-io.h | 0 .../openfst/src/include/fst/script/topsort.h | 0 .../openfst/src/include/fst/script/union.h | 0 .../openfst/src/include/fst/script/verify.h | 0 .../src/include/fst/script/weight-class.h | 0 .../openfst/src/include/fst/set-weight.h | 0 .../src/include/fst/shortest-distance.h | 0 .../openfst/src/include/fst/shortest-path.h | 0 .../src/include/fst/signed-log-weight.h | 0 .../src/include/fst/sparse-power-weight.h | 0 .../src/include/fst/sparse-tuple-weight.h | 0 .../openfst/src/include/fst/state-map.h | 0 .../openfst/src/include/fst/state-reachable.h | 0 .../openfst/src/include/fst/state-table.h | 0 .../openfst/src/include/fst/statesort.h | 0 .../openfst/src/include/fst/string-weight.h | 0 .../openfst/src/include/fst/string.h | 0 .../src/include/fst/symbol-table-ops.h | 0 .../openfst/src/include/fst/symbol-table.h | 0 .../openfst/src/include/fst/synchronize.h | 0 .../openfst/src/include/fst/test-properties.h | 0 .../openfst/src/include/fst/test/algo_test.h | 0 .../openfst/src/include/fst/test/fst_test.h | 0 .../openfst/src/include/fst/test/rand-fst.h | 0 .../src/include/fst/test/weight-tester.h | 0 .../openfst/src/include/fst/topsort.h | 0 .../openfst/src/include/fst/tuple-weight.h | 0 .../openfst/src/include/fst/types.h | 0 .../openfst/src/include/fst/union-find.h | 0 .../openfst/src/include/fst/union-weight.h | 0 .../openfst/src/include/fst/union.h | 0 .../openfst/src/include/fst/util.h | 0 .../openfst/src/include/fst/vector-fst.h | 0 .../openfst/src/include/fst/verify.h | 0 .../openfst/src/include/fst/visit.h | 0 .../openfst/src/include/fst/weight.h | 0 .../openfst/src/lib/CMakeLists.txt | 34 + .../third_party/openfst/src/lib/Makefile.am | 0 .../third_party/openfst/src/lib/Makefile.in | 0 .../third_party/openfst/src/lib/compat.cc | 0 .../third_party/openfst/src/lib/flags.cc | 0 .../third_party/openfst/src/lib/fst-types.cc | 0 .../third_party/openfst/src/lib/fst.cc | 0 .../openfst/src/lib/libfst.vcxproj | 0 .../openfst/src/lib/libfst.vcxproj.filters | 0 .../openfst/src/lib/mapped-file.cc | 0 .../third_party/openfst/src/lib/properties.cc | 0 .../openfst/src/lib/symbol-table-ops.cc | 0 .../openfst/src/lib/symbol-table.cc | 0 .../third_party/openfst/src/lib/util.cc | 0 .../third_party/openfst/src/lib/weight.cc | 0 .../openfst/src/openfst-multibin.targets | 0 .../third_party/openfst/src/openfst.props | 0 .../third_party/openfst/src/openfst.targets | 0 .../openfst/src/openfst.user.props | 0 .../openfst/src/script/CMakeLists.txt | 0 .../openfst/src/script/Makefile.am | 0 .../openfst/src/script/Makefile.in | 0 .../openfst/src/script/arciterator-class.cc | 0 .../third_party/openfst/src/script/arcsort.cc | 0 .../third_party/openfst/src/script/closure.cc | 0 .../third_party/openfst/src/script/compile.cc | 0 .../third_party/openfst/src/script/compose.cc | 0 .../third_party/openfst/src/script/concat.cc | 0 .../third_party/openfst/src/script/connect.cc | 0 .../third_party/openfst/src/script/convert.cc | 0 .../third_party/openfst/src/script/decode.cc | 0 .../openfst/src/script/determinize.cc | 0 .../openfst/src/script/difference.cc | 0 .../openfst/src/script/disambiguate.cc | 0 .../third_party/openfst/src/script/draw.cc | 0 .../third_party/openfst/src/script/encode.cc | 0 .../openfst/src/script/encodemapper-class.cc | 0 .../openfst/src/script/epsnormalize.cc | 0 .../third_party/openfst/src/script/equal.cc | 0 .../openfst/src/script/equivalent.cc | 0 .../openfst/src/script/fst-class.cc | 0 .../third_party/openfst/src/script/getters.cc | 0 .../openfst/src/script/info-impl.cc | 0 .../third_party/openfst/src/script/info.cc | 0 .../openfst/src/script/intersect.cc | 0 .../third_party/openfst/src/script/invert.cc | 0 .../openfst/src/script/isomorphic.cc | 0 .../openfst/src/script/libfstscript.vcxproj | 0 .../src/script/libfstscript.vcxproj.filters | 0 .../third_party/openfst/src/script/map.cc | 0 .../openfst/src/script/minimize.cc | 0 .../third_party/openfst/src/script/print.cc | 0 .../third_party/openfst/src/script/project.cc | 0 .../third_party/openfst/src/script/prune.cc | 0 .../third_party/openfst/src/script/push.cc | 0 .../openfst/src/script/randequivalent.cc | 0 .../third_party/openfst/src/script/randgen.cc | 0 .../third_party/openfst/src/script/relabel.cc | 0 .../third_party/openfst/src/script/replace.cc | 0 .../third_party/openfst/src/script/reverse.cc | 0 .../openfst/src/script/reweight.cc | 0 .../openfst/src/script/rmepsilon.cc | 0 .../openfst/src/script/shortest-distance.cc | 0 .../openfst/src/script/shortest-path.cc | 0 .../openfst/src/script/stateiterator-class.cc | 0 .../openfst/src/script/synchronize.cc | 0 .../third_party/openfst/src/script/text-io.cc | 0 .../third_party/openfst/src/script/topsort.cc | 0 .../third_party/openfst/src/script/union.cc | 0 .../third_party/openfst/src/script/verify.cc | 0 .../openfst/src/script/weight-class.cc | 0 .../openfst/src/test/CMakeLists.txt | 0 .../third_party/openfst/src/test/Makefile.am | 0 .../third_party/openfst/src/test/Makefile.in | 0 .../third_party/openfst/src/test/algo_test.cc | 0 .../third_party/openfst/src/test/fst_test.cc | 0 .../openfst/src/test/weight_test.cc | 0 .../third_party/yaml-cpp/CMakeLists.txt | 0 .../yaml-cpp/include/yaml-cpp/anchor.h | 0 .../yaml-cpp/include/yaml-cpp/binary.h | 0 .../include/yaml-cpp/contrib/anchordict.h | 0 .../include/yaml-cpp/contrib/graphbuilder.h | 0 .../yaml-cpp/include/yaml-cpp/dll.h | 0 .../include/yaml-cpp/emitfromevents.h | 0 .../yaml-cpp/include/yaml-cpp/emitter.h | 0 .../yaml-cpp/include/yaml-cpp/emitterdef.h | 0 .../yaml-cpp/include/yaml-cpp/emittermanip.h | 0 .../yaml-cpp/include/yaml-cpp/emitterstyle.h | 0 .../yaml-cpp/include/yaml-cpp/eventhandler.h | 0 .../yaml-cpp/include/yaml-cpp/exceptions.h | 0 .../yaml-cpp/include/yaml-cpp/mark.h | 0 .../yaml-cpp/include/yaml-cpp/node/convert.h | 0 .../include/yaml-cpp/node/detail/bool_type.h | 0 .../include/yaml-cpp/node/detail/impl.h | 0 .../include/yaml-cpp/node/detail/iterator.h | 0 .../yaml-cpp/node/detail/iterator_fwd.h | 0 .../include/yaml-cpp/node/detail/memory.h | 0 .../include/yaml-cpp/node/detail/node.h | 0 .../include/yaml-cpp/node/detail/node_data.h | 0 .../yaml-cpp/node/detail/node_iterator.h | 0 .../include/yaml-cpp/node/detail/node_ref.h | 0 .../yaml-cpp/include/yaml-cpp/node/emit.h | 0 .../yaml-cpp/include/yaml-cpp/node/impl.h | 0 .../yaml-cpp/include/yaml-cpp/node/iterator.h | 0 .../yaml-cpp/include/yaml-cpp/node/node.h | 0 .../yaml-cpp/include/yaml-cpp/node/parse.h | 0 .../yaml-cpp/include/yaml-cpp/node/ptr.h | 0 .../yaml-cpp/include/yaml-cpp/node/type.h | 0 .../yaml-cpp/include/yaml-cpp/noncopyable.h | 0 .../yaml-cpp/include/yaml-cpp/null.h | 0 .../include/yaml-cpp/ostream_wrapper.h | 0 .../yaml-cpp/include/yaml-cpp/parser.h | 0 .../yaml-cpp/include/yaml-cpp/stlemitter.h | 0 .../yaml-cpp/include/yaml-cpp/traits.h | 0 .../yaml-cpp/include/yaml-cpp/yaml.h | 0 .../third_party/yaml-cpp/install.txt | 0 .../third_party/yaml-cpp/src/binary.cpp | 0 .../yaml-cpp/src/collectionstack.h | 0 .../yaml-cpp/src/contrib/graphbuilder.cpp | 0 .../src/contrib/graphbuilderadapter.cpp | 0 .../src/contrib/graphbuilderadapter.h | 0 .../third_party/yaml-cpp/src/convert.cpp | 0 .../third_party/yaml-cpp/src/directives.cpp | 0 .../third_party/yaml-cpp/src/directives.h | 0 .../third_party/yaml-cpp/src/emit.cpp | 0 .../yaml-cpp/src/emitfromevents.cpp | 0 .../third_party/yaml-cpp/src/emitter.cpp | 0 .../third_party/yaml-cpp/src/emitterstate.cpp | 0 .../third_party/yaml-cpp/src/emitterstate.h | 0 .../third_party/yaml-cpp/src/emitterutils.cpp | 0 .../third_party/yaml-cpp/src/emitterutils.h | 0 .../third_party/yaml-cpp/src/exceptions.cpp | 0 .../third_party/yaml-cpp/src/exp.cpp | 0 .../third_party/yaml-cpp/src/exp.h | 0 .../third_party/yaml-cpp/src/indentation.h | 0 .../third_party/yaml-cpp/src/memory.cpp | 0 .../third_party/yaml-cpp/src/node.cpp | 0 .../third_party/yaml-cpp/src/node_data.cpp | 0 .../third_party/yaml-cpp/src/nodebuilder.cpp | 0 .../third_party/yaml-cpp/src/nodebuilder.h | 0 .../third_party/yaml-cpp/src/nodeevents.cpp | 0 .../third_party/yaml-cpp/src/nodeevents.h | 0 .../third_party/yaml-cpp/src/null.cpp | 0 .../yaml-cpp/src/ostream_wrapper.cpp | 0 .../third_party/yaml-cpp/src/parse.cpp | 0 .../third_party/yaml-cpp/src/parser.cpp | 0 .../third_party/yaml-cpp/src/ptr_vector.h | 0 .../third_party/yaml-cpp/src/regex_yaml.cpp | 0 .../third_party/yaml-cpp/src/regex_yaml.h | 0 .../third_party/yaml-cpp/src/regeximpl.h | 0 .../third_party/yaml-cpp/src/scanner.cpp | 0 .../third_party/yaml-cpp/src/scanner.h | 0 .../third_party/yaml-cpp/src/scanscalar.cpp | 0 .../third_party/yaml-cpp/src/scanscalar.h | 0 .../third_party/yaml-cpp/src/scantag.cpp | 0 .../third_party/yaml-cpp/src/scantag.h | 0 .../third_party/yaml-cpp/src/scantoken.cpp | 0 .../third_party/yaml-cpp/src/setting.h | 0 .../third_party/yaml-cpp/src/simplekey.cpp | 0 .../yaml-cpp/src/singledocparser.cpp | 0 .../yaml-cpp/src/singledocparser.h | 0 .../third_party/yaml-cpp/src/stream.cpp | 0 .../third_party/yaml-cpp/src/stream.h | 0 .../yaml-cpp/src/streamcharsource.h | 0 .../third_party/yaml-cpp/src/stringsource.h | 0 .../third_party/yaml-cpp/src/tag.cpp | 0 .../third_party/yaml-cpp/src/tag.h | 0 .../third_party/yaml-cpp/src/token.h | 0 .../yaml-cpp/yaml-cpp-config-version.cmake.in | 0 .../yaml-cpp/yaml-cpp-config.cmake.in | 0 .../third_party/yaml-cpp/yaml-cpp.pc.cmake | 0 .../utils => runtime/python}/__init__.py | 0 .../runtime => runtime}/python/grpc/Readme.md | 0 .../python/grpc/grpc_main_client.py | 0 .../python/grpc/proto/Readme.md | 0 .../python/grpc/proto/paraformer.proto | 0 .../python/grpc/proto/workflow.png | Bin .../python/grpc/requirements.txt | 0 .../runtime => runtime}/python/http/README.md | 0 .../python/http/asr_nginx.conf | 0 .../runtime => runtime}/python/http/client.py | 0 .../python/http/requirements.txt | 0 .../runtime => runtime}/python/http/server.py | 0 .../python/http/start_server.sh | 0 .../python/libtorch/README.md | 0 .../python/libtorch}/__init__.py | 0 .../python/libtorch/demo.py | 0 .../python/libtorch/funasr_torch/__init__.py | 0 .../libtorch/funasr_torch/paraformer_bin.py | 0 .../libtorch/funasr_torch}/utils/__init__.py | 0 .../funasr_torch/utils/compute_wer.py | 0 .../libtorch/funasr_torch/utils/frontend.py | 0 .../funasr_torch/utils/postprocess_utils.py | 0 .../funasr_torch/utils/timestamp_utils.py | 0 .../libtorch/funasr_torch/utils/utils.py | 0 .../python/libtorch/setup.py | 0 .../python/onnxruntime/README.md | 0 .../python/onnxruntime/__init__.py | 0 .../onnxruntime/demo_contextual_paraformer.py | 0 .../onnxruntime/demo_paraformer_offline.py | 0 .../onnxruntime/demo_paraformer_online.py | 0 .../python/onnxruntime/demo_punc_offline.py | 0 .../python/onnxruntime/demo_punc_online.py | 0 .../python/onnxruntime/demo_vad_offline.py | 0 .../python/onnxruntime/demo_vad_online.py | 0 .../onnxruntime/funasr_onnx/__init__.py | 0 .../onnxruntime/funasr_onnx/paraformer_bin.py | 0 .../funasr_onnx/paraformer_online_bin.py | 0 .../onnxruntime/funasr_onnx/punc_bin.py | 0 .../onnxruntime/funasr_onnx/utils/__init__.py | 0 .../onnxruntime/funasr_onnx/utils/e2e_vad.py | 0 .../onnxruntime/funasr_onnx/utils/frontend.py | 0 .../funasr_onnx/utils/postprocess_utils.py | 0 .../funasr_onnx/utils/timestamp_utils.py | 0 .../onnxruntime/funasr_onnx/utils/utils.py | 0 .../python/onnxruntime/funasr_onnx/vad_bin.py | 0 .../python/onnxruntime/setup.py | 0 .../python/utils/compute_wer.py | 0 .../python/utils/proce_text.py | 0 .../python/utils/requirements.txt | 0 .../python/utils/split_scp.pl | 0 .../python/utils/test_cer.py | 0 .../python/utils/test_cer.sh | 0 .../python/utils/test_rtf.py | 0 .../python/utils/test_rtf.sh | 0 .../python/utils/test_rtf_gpu.py | 0 .../python/websocket/README.md | 0 .../python/websocket/funasr_client_api.py | 0 .../python/websocket/funasr_wss_client.py | 0 .../python/websocket/funasr_wss_server.py | 0 .../python/websocket/requirements_client.txt | 0 .../python/websocket/requirements_server.txt | 0 .../python/websocket/ssl_key | 0 {funasr/runtime => runtime}/readme.md | 30 +- {funasr/runtime => runtime}/readme_cn.md | 34 +- {funasr/runtime => runtime}/run_server.sh | 35 +- .../runtime => runtime}/run_server_2pass.sh | 36 +- {funasr/runtime => runtime}/ssl_key/readme.md | 0 .../runtime => runtime}/ssl_key/readme_cn.md | 0 .../runtime => runtime}/ssl_key/server.crt | 0 .../runtime => runtime}/ssl_key/server.key | 0 runtime/tools/fst/add_lex_disambig.pl | 118 ++ runtime/tools/fst/collect_resource_file.sh | 30 + runtime/tools/fst/compile_dict_token.sh | 84 + runtime/tools/fst/ctc_token_fst.py | 24 + runtime/tools/fst/eps2disambig.pl | 23 + runtime/tools/fst/generate_lexicon.py | 35 + runtime/tools/fst/make_decode_graph.sh | 34 + runtime/tools/fst/make_lexicon_fst.pl | 148 ++ runtime/tools/fst/parse_options.sh | 97 + runtime/tools/fst/s2eps.pl | 27 + runtime/tools/fst/train_lms.sh | 25 + runtime/tools/path.sh | 3 + runtime/tools/train_compile_ngram.sh | 21 + .../triton_gpu/Dockerfile/Dockerfile.server | 0 .../runtime => runtime}/triton_gpu/README.md | 0 .../triton_gpu/client/aishell_test.txt | 0 .../triton_gpu/client/client.py | 0 .../client/decode_manifest_triton.py | 0 .../client/decode_manifest_triton_wo_cuts.py | 0 .../triton_gpu/client/speech_client.py | 0 .../triton_gpu/client/test_wavs/long.wav | Bin .../triton_gpu/client/test_wavs/mid.wav | Bin .../triton_gpu/client/utils.py | 0 .../encoder/1/.gitkeep | 0 .../encoder/config.pbtxt | 0 .../feature_extractor/1/model.py | 0 .../feature_extractor/config.pbtxt | 0 .../infer_pipeline/1/.gitkeep | 0 .../infer_pipeline/config.pbtxt | 0 .../scoring/1/model.py | 0 .../scoring/config.pbtxt | 0 .../websocket/CMakeLists.txt | 4 + .../websocket/bin/CMakeLists.txt | 0 .../websocket/bin/funasr-wss-client-2pass.cpp | 89 +- .../websocket/bin/funasr-wss-client.cpp | 69 +- .../websocket/bin/funasr-wss-server-2pass.cpp | 44 +- .../websocket/bin/funasr-wss-server.cpp | 201 +- .../websocket/bin/microphone.cpp | 0 .../websocket/bin/microphone.h | 0 .../websocket/bin/websocket-server-2pass.cpp | 75 +- .../websocket/bin/websocket-server-2pass.h | 1 + .../websocket/bin/websocket-server.cpp | 95 +- .../websocket/bin/websocket-server.h | 16 +- runtime/websocket/hotwords.txt | 2 + .../runtime => runtime}/websocket/readme.md | 2 +- .../websocket/readme_zh.md | 2 +- .../websocket/requirements_install.md | 0 1440 files changed, 60488 insertions(+), 1135 deletions(-) delete mode 100644 funasr/runtime/onnxruntime/src/util.cpp delete mode 100644 funasr/runtime/onnxruntime/src/util.h delete mode 100644 funasr/runtime/onnxruntime/third_party/openfst/src/lib/CMakeLists.txt delete mode 100644 funasr/runtime/python/grpc/.gitignore delete mode 100644 funasr/runtime/websocket/hotwords.txt rename {funasr/runtime => runtime}/__init__.py (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/build.gradle (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/proguard-rules.pro (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/androidTest/java/com/yeyupiaoling/androidclient/ExampleInstrumentedTest.kt (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/AndroidManifest.xml (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/java/com/yeyupiaoling/androidclient/AudioView.java (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/java/com/yeyupiaoling/androidclient/MainActivity.java (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/java/com/yeyupiaoling/androidclient/SSLSocketClient.java (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/res/drawable/edittext_border.xml (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/res/drawable/ic_launcher_background.xml (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/res/drawable/ic_launcher_foreground.xml (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/res/drawable/logo.png (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/res/layout/activity_main.xml (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/res/layout/dialog_input_hotwords.xml (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/res/layout/dialog_input_uri.xml (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/res/menu/menu.xml (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/res/mipmap-hdpi/ic_launcher.webp (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/res/mipmap-mdpi/ic_launcher.webp (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/res/mipmap-mdpi/ic_launcher_round.webp (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/res/mipmap-xhdpi/ic_launcher.webp (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/res/mipmap-xxhdpi/ic_launcher.webp (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/res/values/colors.xml (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/res/values/strings.xml (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/res/values/themes.xml (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/res/xml/backup_rules.xml (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/main/res/xml/data_extraction_rules.xml (100%) rename {funasr/runtime => runtime}/android/AndroidClient/app/src/test/java/com/yeyupiaoling/androidclient/ExampleUnitTest.kt (100%) rename {funasr/runtime => runtime}/android/AndroidClient/build.gradle (100%) rename {funasr/runtime => runtime}/android/AndroidClient/gradle.properties (100%) rename {funasr/runtime => runtime}/android/AndroidClient/gradle/wrapper/gradle-wrapper.jar (100%) rename {funasr/runtime => runtime}/android/AndroidClient/gradle/wrapper/gradle-wrapper.properties (100%) rename {funasr/runtime => runtime}/android/AndroidClient/gradlew (100%) rename {funasr/runtime => runtime}/android/AndroidClient/gradlew.bat (100%) rename {funasr/runtime => runtime}/android/AndroidClient/settings.gradle (100%) rename {funasr/runtime => runtime}/android/images/QRcode.png (100%) rename {funasr/runtime => runtime}/android/images/image1.png (100%) rename {funasr/runtime => runtime}/android/images/image2.png (100%) rename {funasr/runtime => runtime}/android/images/image3.png (100%) rename {funasr/runtime => runtime}/android/readme.md (100%) rename {funasr/runtime => runtime}/csharp/AliFsmnVad/AliFsmnVadSharp.Examples/AliFsmnVadSharp.Examples.csproj (100%) rename {funasr/runtime => runtime}/csharp/AliFsmnVad/AliFsmnVadSharp.Examples/Program.cs (100%) rename {funasr/runtime => runtime}/csharp/AliFsmnVad/AliFsmnVadSharp.sln (100%) rename {funasr/runtime => runtime}/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVad.cs (100%) rename {funasr/runtime => runtime}/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVadSharp.csproj (100%) rename {funasr/runtime => runtime}/csharp/AliFsmnVad/AliFsmnVadSharp/DLL/KaldiNativeFbank.cs (100%) rename {funasr/runtime => runtime}/csharp/AliFsmnVad/AliFsmnVadSharp/DLL/KnfOnlineFbank.cs (100%) rename {funasr/runtime => runtime}/csharp/AliFsmnVad/AliFsmnVadSharp/E2EVadModel.cs (100%) rename {funasr/runtime => runtime}/csharp/AliFsmnVad/AliFsmnVadSharp/Lib/kaldi-native-fbank-dll.dll (100%) rename {funasr/runtime => runtime}/csharp/AliFsmnVad/AliFsmnVadSharp/Model/CmvnEntity.cs (100%) rename {funasr/runtime => runtime}/csharp/AliFsmnVad/AliFsmnVadSharp/Model/E2EVadFrameProbEntity.cs (100%) rename {funasr/runtime => runtime}/csharp/AliFsmnVad/AliFsmnVadSharp/Model/E2EVadSpeechBufWithDoaEntity.cs (100%) rename {funasr/runtime => runtime}/csharp/AliFsmnVad/AliFsmnVadSharp/Model/EncoderConfEntity.cs (100%) rename {funasr/runtime => runtime}/csharp/AliFsmnVad/AliFsmnVadSharp/Model/FrontendConfEntity.cs (100%) rename {funasr/runtime => runtime}/csharp/AliFsmnVad/AliFsmnVadSharp/Model/SegmentEntity.cs (100%) rename {funasr/runtime => runtime}/csharp/AliFsmnVad/AliFsmnVadSharp/Model/VadInputEntity.cs (100%) rename {funasr/runtime => runtime}/csharp/AliFsmnVad/AliFsmnVadSharp/Model/VadOutputEntity.cs (100%) rename {funasr/runtime => runtime}/csharp/AliFsmnVad/AliFsmnVadSharp/Model/VadPostConfEntity.cs (100%) rename {funasr/runtime => runtime}/csharp/AliFsmnVad/AliFsmnVadSharp/Model/VadYamlEntity.cs (100%) rename {funasr/runtime => runtime}/csharp/AliFsmnVad/AliFsmnVadSharp/Struct/FbankData.cs (100%) rename {funasr/runtime => runtime}/csharp/AliFsmnVad/AliFsmnVadSharp/Utils/YamlHelper.cs (100%) rename {funasr/runtime => runtime}/csharp/AliFsmnVad/AliFsmnVadSharp/WavFrontend.cs (100%) rename {funasr/runtime => runtime}/csharp/AliFsmnVad/AliFsmnVadSharp/WindowDetector.cs (100%) rename {funasr/runtime => runtime}/csharp/AliFsmnVad/README.md (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/App.xaml (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/App.xaml.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/AppShell.xaml (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/AppShell.xaml.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MainPage.xaml (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MainPage.xaml.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MauiApp1.csproj (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MauiApp1.csproj.user (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MauiProgram.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MySplashPage.xaml (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MySplashPage.xaml.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Android/AndroidManifest.xml (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Android/MainActivity.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Android/MainApplication.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Android/Resources/values/colors.xml (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/MacCatalyst/AppDelegate.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/MacCatalyst/Info.plist (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/MacCatalyst/Program.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Tizen/Main.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Tizen/tizen-manifest.xml (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Windows/App.xaml (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Windows/App.xaml.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Windows/Package.appxmanifest (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Windows/app.manifest (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/iOS/AppDelegate.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/iOS/Info.plist (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/iOS/Program.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Properties/launchSettings.json (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/RecognitionForFiles.xaml (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/RecognitionForFiles.xaml.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/AppIcon/appicon.svg (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/AppIcon/appiconfg.svg (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Fonts/OpenSans-Regular.ttf (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Fonts/OpenSans-Semibold.ttf (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Images/dotnet_bot.svg (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Raw/AboutAssets.txt (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Splash/splash.svg (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Styles/Colors.xaml (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Styles/Styles.xaml (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Utils/DownloadHelper.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Utils/SysConf.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples/AliParaformerAsr.Examples.csproj (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.Examples/Program.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr.sln (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr/AliParaformerAsr.csproj (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr/Model/CmvnEntity.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr/Model/DecoderConfEntity.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr/Model/EncoderConfEntity.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr/Model/FrontendConfEntity.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr/Model/ModelConfEntity.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr/Model/OfflineInputEntity.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr/Model/OfflineOutputEntity.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr/Model/OfflineYamlEntity.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr/Model/PostEncoderConfEntity.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr/Model/PreEncoderConfEntity.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr/Model/PredictorConfEntity.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr/OfflineRecognizer.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr/Utils/YamlHelper.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/AliParaformerAsr/WavFrontend.cs (100%) rename {funasr/runtime => runtime}/csharp/AliParaformerAsr/README.md (100%) rename {funasr/runtime => runtime}/csharp/ws-client/FunASRClient_CShape.sln (100%) rename {funasr/runtime => runtime}/csharp/ws-client/FunASRClient_CShape.suo (100%) rename {funasr/runtime => runtime}/csharp/ws-client/FunASRWSClient_Offline/FunASRWSClient_Offline.csproj (100%) rename {funasr/runtime => runtime}/csharp/ws-client/FunASRWSClient_Offline/Program.cs (100%) rename {funasr/runtime => runtime}/csharp/ws-client/FunASRWSClient_Offline/README.md (100%) rename {funasr/runtime => runtime}/csharp/ws-client/FunASRWSClient_Offline/WebScoketClient.cs (100%) rename {funasr/runtime => runtime}/csharp/ws-client/FunASRWSClient_Online/FunASRWSClient_Online.csproj (100%) rename {funasr/runtime => runtime}/csharp/ws-client/FunASRWSClient_Online/Program.cs (100%) rename {funasr/runtime => runtime}/csharp/ws-client/FunASRWSClient_Online/README.md (100%) rename {funasr/runtime => runtime}/csharp/ws-client/FunASRWSClient_Online/WaveCollect.cs (100%) rename {funasr/runtime => runtime}/csharp/ws-client/FunASRWSClient_Online/WebScoketClient.cs (100%) rename {funasr/runtime => runtime}/csharp/ws-client/confg/config.ini (100%) rename {funasr/runtime => runtime}/csharp/ws-client/confg/hotword.txt (100%) rename {funasr/runtime => runtime}/csharp/ws-client/confg/tmp.wav (100%) rename {funasr/runtime => runtime}/deploy_tools/funasr-runtime-deploy-offline-cpu-en.sh (99%) rename {funasr/runtime => runtime}/deploy_tools/funasr-runtime-deploy-offline-cpu-zh.sh (93%) rename {funasr/runtime => runtime}/deploy_tools/funasr-runtime-deploy-online-cpu-zh.sh (99%) create mode 100644 runtime/deploy_tools/install_docker.sh rename {funasr/runtime => runtime}/docs/SDK_advanced_guide_offline.md (90%) rename {funasr/runtime => runtime}/docs/SDK_advanced_guide_offline_en.md (92%) rename {funasr/runtime => runtime}/docs/SDK_advanced_guide_offline_en_zh.md (91%) rename {funasr/runtime => runtime}/docs/SDK_advanced_guide_offline_zh.md (87%) rename {funasr/runtime => runtime}/docs/SDK_advanced_guide_online.md (95%) rename {funasr/runtime => runtime}/docs/SDK_advanced_guide_online_zh.md (93%) rename {funasr/runtime => runtime}/docs/SDK_tutorial.md (95%) rename {funasr/runtime => runtime}/docs/SDK_tutorial_en.md (96%) rename {funasr/runtime => runtime}/docs/SDK_tutorial_en_zh.md (94%) rename {funasr/runtime => runtime}/docs/SDK_tutorial_online.md (96%) rename {funasr/runtime => runtime}/docs/SDK_tutorial_online_zh.md (94%) rename {funasr/runtime => runtime}/docs/SDK_tutorial_zh.md (94%) rename {funasr/runtime => runtime}/docs/aliyun_server_tutorial.md (100%) rename {funasr/runtime => runtime}/docs/benchmark_libtorch.md (100%) rename {funasr/runtime => runtime}/docs/benchmark_onnx.md (100%) rename {funasr/runtime => runtime}/docs/benchmark_onnx_cpp.md (62%) rename {funasr/runtime => runtime}/docs/docker_offline_cpu_en_lists (87%) rename {funasr/runtime => runtime}/docs/docker_offline_cpu_zh_lists (82%) rename {funasr/runtime => runtime}/docs/docker_online_cpu_zh_lists (93%) rename {funasr/runtime => runtime}/docs/images/2pass.jpg (100%) rename {funasr/runtime => runtime}/docs/images/aliyun1.png (100%) rename {funasr/runtime => runtime}/docs/images/aliyun10.png (100%) rename {funasr/runtime => runtime}/docs/images/aliyun11.png (100%) rename {funasr/runtime => runtime}/docs/images/aliyun12.png (100%) rename {funasr/runtime => runtime}/docs/images/aliyun2.png (100%) rename {funasr/runtime => runtime}/docs/images/aliyun3.png (100%) rename {funasr/runtime => runtime}/docs/images/aliyun4.png (100%) rename {funasr/runtime => runtime}/docs/images/aliyun5.png (100%) rename {funasr/runtime => runtime}/docs/images/aliyun6.png (100%) rename {funasr/runtime => runtime}/docs/images/aliyun7.png (100%) rename {funasr/runtime => runtime}/docs/images/aliyun8.png (100%) rename {funasr/runtime => runtime}/docs/images/aliyun9.png (100%) rename {funasr/runtime => runtime}/docs/images/html.png (100%) create mode 100644 runtime/docs/lm_train_tutorial.md rename {funasr/runtime => runtime}/docs/websocket_protocol.md (87%) rename {funasr/runtime => runtime}/docs/websocket_protocol_zh.md (85%) rename {funasr/runtime => runtime}/grpc/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/grpc/Readme.md (100%) rename {funasr/runtime => runtime}/grpc/build.sh (100%) rename {funasr/runtime => runtime}/grpc/common.cmake (100%) rename {funasr/runtime => runtime}/grpc/paraformer-server.cc (100%) rename {funasr/runtime => runtime}/grpc/paraformer-server.h (100%) rename {funasr/runtime => runtime}/grpc/run_server.sh (100%) rename {funasr/runtime => runtime}/html5/h5Server.py (100%) rename {funasr/runtime => runtime}/html5/readme.md (100%) rename {funasr/runtime => runtime}/html5/readme_zh.md (100%) rename {funasr/runtime => runtime}/html5/ssl_key (100%) rename {funasr/runtime => runtime}/html5/static/index.html (88%) rename {funasr/runtime => runtime}/html5/static/main.js (91%) rename {funasr/runtime => runtime}/html5/static/pcm.js (100%) rename {funasr/runtime => runtime}/html5/static/recorder-core.js (100%) rename {funasr/runtime => runtime}/html5/static/wav.js (100%) rename {funasr/runtime => runtime}/html5/static/wsconnecter.js (95%) rename {funasr/runtime => runtime}/ios/Readme.md (100%) rename {funasr/runtime => runtime}/ios/paraformer_online/Podfile (100%) rename {funasr/runtime => runtime}/ios/paraformer_online/paraformer_online.xcodeproj/project.pbxproj (100%) rename {funasr/runtime => runtime}/ios/paraformer_online/paraformer_online.xcodeproj/project.xcworkspace/contents.xcworkspacedata (100%) rename {funasr/runtime => runtime}/ios/paraformer_online/paraformer_online.xcodeproj/project.xcworkspace/xcuserdata/qiuwei.xcuserdatad/UserInterfaceState.xcuserstate (100%) rename {funasr/runtime => runtime}/ios/paraformer_online/paraformer_online.xcodeproj/xcuserdata/qiuwei.xcuserdatad/xcdebugger/Breakpoints_v2.xcbkptlist (100%) rename {funasr/runtime => runtime}/ios/paraformer_online/paraformer_online.xcodeproj/xcuserdata/qiuwei.xcuserdatad/xcschemes/xcschememanagement.plist (100%) rename {funasr/runtime => runtime}/ios/paraformer_online/paraformer_online/AppDelegate.h (100%) rename {funasr/runtime => runtime}/ios/paraformer_online/paraformer_online/AppDelegate.m (100%) rename {funasr/runtime => runtime}/ios/paraformer_online/paraformer_online/Assets.xcassets/AccentColor.colorset/Contents.json (100%) rename {funasr/runtime => runtime}/ios/paraformer_online/paraformer_online/Assets.xcassets/AppIcon.appiconset/Contents.json (100%) rename {funasr/runtime => runtime}/ios/paraformer_online/paraformer_online/Assets.xcassets/Contents.json (100%) rename {funasr/runtime => runtime}/ios/paraformer_online/paraformer_online/AudioCapture.h (100%) rename {funasr/runtime => runtime}/ios/paraformer_online/paraformer_online/AudioCapture.mm (100%) rename {funasr/runtime => runtime}/ios/paraformer_online/paraformer_online/AudioRecorder.h (100%) rename {funasr/runtime => runtime}/ios/paraformer_online/paraformer_online/AudioRecorder.m (100%) rename {funasr/runtime => runtime}/ios/paraformer_online/paraformer_online/Base.lproj/LaunchScreen.storyboard (100%) rename {funasr/runtime => runtime}/ios/paraformer_online/paraformer_online/Base.lproj/Main.storyboard (100%) rename {funasr/runtime => runtime}/ios/paraformer_online/paraformer_online/Info.plist (100%) rename {funasr/runtime => runtime}/ios/paraformer_online/paraformer_online/SceneDelegate.h (100%) rename {funasr/runtime => runtime}/ios/paraformer_online/paraformer_online/SceneDelegate.m (100%) rename {funasr/runtime => runtime}/ios/paraformer_online/paraformer_online/ViewController.h (100%) rename {funasr/runtime => runtime}/ios/paraformer_online/paraformer_online/ViewController.mm (100%) rename {funasr/runtime => runtime}/ios/paraformer_online/paraformer_online/main.m (100%) rename {funasr/runtime => runtime}/java/FunasrWsClient.java (90%) rename {funasr/runtime => runtime}/java/Makefile (100%) rename {funasr/runtime => runtime}/java/readme.md (100%) rename {funasr/runtime => runtime}/onnxruntime/CMakeLists.txt (93%) rename {funasr/runtime => runtime}/onnxruntime/CMakeSettings.json (100%) rename {funasr/runtime => runtime}/onnxruntime/bin/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/bin/funasr-onnx-2pass-rtf.cpp (94%) rename {funasr/runtime => runtime}/onnxruntime/bin/funasr-onnx-2pass.cpp (93%) rename {funasr/runtime => runtime}/onnxruntime/bin/funasr-onnx-offline-punc.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/bin/funasr-onnx-offline-rtf.cpp (75%) rename {funasr/runtime => runtime}/onnxruntime/bin/funasr-onnx-offline-vad.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/bin/funasr-onnx-offline.cpp (71%) rename {funasr/runtime => runtime}/onnxruntime/bin/funasr-onnx-online-asr.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/bin/funasr-onnx-online-punc.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/bin/funasr-onnx-online-rtf.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/bin/funasr-onnx-online-vad.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/include/audio.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/com-define.h (89%) rename {funasr/runtime => runtime}/onnxruntime/include/funasrruntime.h (86%) rename {funasr/runtime => runtime}/onnxruntime/include/itn-model.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/model.h (81%) rename {funasr/runtime => runtime}/onnxruntime/include/offline-stream.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/punc-model.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tclap/Arg.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tclap/ArgException.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tclap/ArgTraits.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tclap/CmdLine.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tclap/CmdLineInterface.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tclap/CmdLineOutput.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tclap/Constraint.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tclap/DocBookOutput.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tclap/HelpVisitor.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tclap/IgnoreRestVisitor.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tclap/MultiArg.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tclap/MultiSwitchArg.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tclap/OptionalUnlabeledTracker.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tclap/StandardTraits.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tclap/StdOutput.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tclap/SwitchArg.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tclap/UnlabeledMultiArg.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tclap/UnlabeledValueArg.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tclap/ValueArg.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tclap/ValuesConstraint.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tclap/VersionVisitor.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tclap/Visitor.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tclap/XorHandler.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tclap/ZshCompletionOutput.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tclap/sstream.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tpass-online-stream.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/tpass-stream.h (100%) rename {funasr/runtime => runtime}/onnxruntime/include/vad-model.h (100%) rename {funasr/runtime => runtime}/onnxruntime/readme.md (92%) rename {funasr/runtime => runtime}/onnxruntime/src/CMakeLists.txt (76%) rename {funasr/runtime => runtime}/onnxruntime/src/alignedmem.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/src/alignedmem.h (100%) rename {funasr/runtime => runtime}/onnxruntime/src/audio.cpp (100%) create mode 100644 runtime/onnxruntime/src/bias-lm.cpp create mode 100644 runtime/onnxruntime/src/bias-lm.h rename {funasr/runtime => runtime}/onnxruntime/src/common-struct.h (100%) rename {funasr/runtime => runtime}/onnxruntime/src/commonfunc.h (100%) rename {funasr/runtime => runtime}/onnxruntime/src/ct-transformer-online.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/src/ct-transformer-online.h (100%) rename {funasr/runtime => runtime}/onnxruntime/src/ct-transformer.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/src/ct-transformer.h (100%) rename {funasr/runtime => runtime}/onnxruntime/src/e2e-vad.h (100%) rename {funasr/runtime => runtime}/onnxruntime/src/encode_converter.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/src/encode_converter.h (100%) rename {funasr/runtime => runtime}/onnxruntime/src/fsmn-vad-online.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/src/fsmn-vad-online.h (100%) rename {funasr/runtime => runtime}/onnxruntime/src/fsmn-vad.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/src/fsmn-vad.h (100%) rename {funasr/runtime => runtime}/onnxruntime/src/funasrruntime.cpp (89%) rename {funasr/runtime => runtime}/onnxruntime/src/itn-processor.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/src/itn-processor.h (100%) rename {funasr/runtime => runtime}/onnxruntime/src/itn-token-parser.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/src/itn-token-parser.h (100%) rename {funasr/runtime => runtime}/onnxruntime/src/model.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/src/offline-stream.cpp (92%) rename {funasr/runtime => runtime}/onnxruntime/src/paraformer-online.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/src/paraformer-online.h (97%) rename {funasr/runtime => runtime}/onnxruntime/src/paraformer.cpp (72%) rename {funasr/runtime => runtime}/onnxruntime/src/paraformer.h (79%) create mode 100644 runtime/onnxruntime/src/phone-set.cpp create mode 100644 runtime/onnxruntime/src/phone-set.h rename {funasr/runtime => runtime}/onnxruntime/src/precomp.h (93%) rename {funasr/runtime => runtime}/onnxruntime/src/predefine-coe.h (100%) rename {funasr/runtime => runtime}/onnxruntime/src/punc-model.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/src/resample.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/src/resample.h (100%) rename {funasr/runtime => runtime}/onnxruntime/src/seg_dict.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/src/seg_dict.h (100%) rename {funasr/runtime => runtime}/onnxruntime/src/tensor.h (98%) rename {funasr/runtime => runtime}/onnxruntime/src/tokenizer.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/src/tokenizer.h (100%) rename {funasr/runtime => runtime}/onnxruntime/src/tpass-online-stream.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/src/tpass-stream.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/src/utf8-string.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/src/utf8-string.h (100%) create mode 100644 runtime/onnxruntime/src/util.cpp create mode 100644 runtime/onnxruntime/src/util.h rename {funasr/runtime => runtime}/onnxruntime/src/vad-model.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/src/vocab.cpp (92%) rename {funasr/runtime => runtime}/onnxruntime/src/vocab.h (86%) create mode 100644 runtime/onnxruntime/src/wfst-decoder.cpp create mode 100644 runtime/onnxruntime/src/wfst-decoder.h rename {funasr/runtime => runtime}/onnxruntime/third_party/download_ffmpeg.sh (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/download_onnxruntime.sh (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/.gitattributes (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/.gitmodules (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/.travis.yml (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/AUTHORS.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/COPYING.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/ChangeLog.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/INSTALL.md (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/README.md (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/WORKSPACE (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/appveyor.yml (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/bazel/gflags.bzl (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/cmake/README_runtime.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/cmake/cmake_uninstall.cmake.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/cmake/config.cmake.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/cmake/execute_test.cmake (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/cmake/package.cmake.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/cmake/package.pc.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/cmake/utils.cmake (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/cmake/version.cmake.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/src/config.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/src/defines.h.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/src/gflags.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/src/gflags.h.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/src/gflags_completions.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/src/gflags_completions.h.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/src/gflags_completions.sh (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/src/gflags_declare.h.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/src/gflags_ns.h.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/src/gflags_reporting.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/src/mutex.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/src/util.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/src/windows_port.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/src/windows_port.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/test/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/test/config/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/test/config/main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/test/flagfile.1 (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/test/flagfile.2 (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/test/flagfile.3 (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/test/gflags_build.py.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/test/gflags_declare_flags.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/test/gflags_declare_test.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/test/gflags_strip_flags_test.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/test/gflags_strip_flags_test.cmake (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/test/gflags_unittest.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/test/gflags_unittest_flagfile (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/test/nc/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/gflags/test/nc/gflags_nc.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/cmake/DetermineGflagsNamespace.cmake (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/cmake/FindUnwind.cmake (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/cmake/GetCacheVariables.cmake (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/cmake/RunCleanerTest1.cmake (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/cmake/RunCleanerTest2.cmake (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/cmake/RunCleanerTest3.cmake (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/cmake/TestInitPackageConfig.cmake (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/cmake/TestPackageConfig.cmake (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/glog-config.cmake.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/glog-modules.cmake.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/libglog.pc.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/base/commandlineflags.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/base/googleinit.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/base/mutex.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/config.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/config.h.cmake.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/demangle.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/demangle.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/fuzz_demangle.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/glog/export.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/glog/log_severity.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/glog/logging.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/glog/logging.h.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/glog/platform.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/glog/raw_logging.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/glog/raw_logging.h.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/glog/stl_logging.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/glog/stl_logging.h.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/glog/vlog_is_on.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/glog/vlog_is_on.h.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/logging.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/mock-log.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/raw_logging.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/signalhandler.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/stacktrace.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/stacktrace_generic-inl.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/stacktrace_libunwind-inl.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/stacktrace_powerpc-inl.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/stacktrace_unittest.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/stacktrace_unwind-inl.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/stacktrace_windows-inl.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/stacktrace_x86-inl.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/symbolize.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/symbolize.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/utilities.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/utilities.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/glog/src/vlog_is_on.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/cppjieba/DictTrie.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/cppjieba/FullSegment.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/cppjieba/HMMModel.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/cppjieba/HMMSegment.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/cppjieba/Jieba.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/cppjieba/KeywordExtractor.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/cppjieba/MPSegment.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/cppjieba/MixSegment.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/cppjieba/PosTagger.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/cppjieba/PreFilter.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/cppjieba/QuerySegment.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/cppjieba/SegmentBase.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/cppjieba/SegmentTagged.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/cppjieba/TextRankExtractor.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/cppjieba/Trie.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/cppjieba/Unicode.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/limonp/ArgvContext.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/limonp/BlockingQueue.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/limonp/BoundedBlockingQueue.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/limonp/BoundedQueue.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/limonp/Closure.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/limonp/Colors.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/limonp/Condition.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/limonp/Config.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/limonp/FileLock.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/limonp/ForcePublic.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/limonp/LocalVector.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/limonp/Logging.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/limonp/Md5.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/limonp/MutexLock.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/limonp/NonCopyable.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/limonp/StdExtension.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/limonp/StringUtil.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/limonp/Thread.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/jieba/include/limonp/ThreadPool.hpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/cmake/Modules/FetchContent.cmake (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/cmake/Modules/FetchContent/CMakeLists.cmake.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/cmake/Modules/README.md (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/cmake/cmake_extension.py (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/cmake/googletest.cmake (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/cmake/pybind11.cmake (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/CMakeLists.txt.bak (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-fbank.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-fbank.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-functions.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-functions.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-window.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-window.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/fftsg.c (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/log.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/log.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/mel-computations.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/mel-computations.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/online-feature.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/online-feature.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/rfft.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/rfft.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/test-log.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/test-online-fbank.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/test-online-feature.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/test-rfft.cc (100%) create mode 100644 runtime/onnxruntime/third_party/kaldi/CMakeLists.txt create mode 100644 runtime/onnxruntime/third_party/kaldi/base/Makefile create mode 100755 runtime/onnxruntime/third_party/kaldi/base/get_version.sh create mode 100644 runtime/onnxruntime/third_party/kaldi/base/io-funcs-inl.h create mode 100644 runtime/onnxruntime/third_party/kaldi/base/io-funcs-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/base/io-funcs.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/base/io-funcs.h create mode 100644 runtime/onnxruntime/third_party/kaldi/base/kaldi-common.h create mode 100644 runtime/onnxruntime/third_party/kaldi/base/kaldi-error-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/base/kaldi-error.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/base/kaldi-error.h create mode 100644 runtime/onnxruntime/third_party/kaldi/base/kaldi-math-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/base/kaldi-math.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/base/kaldi-math.h create mode 100644 runtime/onnxruntime/third_party/kaldi/base/kaldi-types.h create mode 100644 runtime/onnxruntime/third_party/kaldi/base/kaldi-utils.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/base/kaldi-utils.h create mode 100644 runtime/onnxruntime/third_party/kaldi/base/timer-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/base/timer.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/base/timer.h create mode 100644 runtime/onnxruntime/third_party/kaldi/decoder/Makefile create mode 100644 runtime/onnxruntime/third_party/kaldi/decoder/biglm-faster-decoder.h create mode 100644 runtime/onnxruntime/third_party/kaldi/decoder/decodable-mapped.h create mode 100644 runtime/onnxruntime/third_party/kaldi/decoder/decodable-matrix.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/decoder/decodable-matrix.h create mode 100644 runtime/onnxruntime/third_party/kaldi/decoder/decodable-sum.h create mode 100644 runtime/onnxruntime/third_party/kaldi/decoder/decoder-wrappers.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/decoder/decoder-wrappers.h create mode 100644 runtime/onnxruntime/third_party/kaldi/decoder/faster-decoder.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/decoder/faster-decoder.h create mode 100644 runtime/onnxruntime/third_party/kaldi/decoder/lattice-biglm-faster-decoder.h create mode 100644 runtime/onnxruntime/third_party/kaldi/decoder/lattice-faster-decoder.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/decoder/lattice-faster-decoder.h create mode 100644 runtime/onnxruntime/third_party/kaldi/decoder/lattice-faster-online-decoder.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/decoder/lattice-faster-online-decoder.h create mode 100644 runtime/onnxruntime/third_party/kaldi/decoder/lattice-incremental-decoder.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/decoder/lattice-incremental-decoder.h create mode 100644 runtime/onnxruntime/third_party/kaldi/decoder/lattice-incremental-online-decoder.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/decoder/lattice-incremental-online-decoder.h create mode 100644 runtime/onnxruntime/third_party/kaldi/decoder/lattice-simple-decoder.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/decoder/lattice-simple-decoder.h create mode 100644 runtime/onnxruntime/third_party/kaldi/decoder/simple-decoder.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/decoder/simple-decoder.h create mode 100644 runtime/onnxruntime/third_party/kaldi/decoder/training-graph-compiler.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/decoder/training-graph-compiler.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstbin/Makefile create mode 100644 runtime/onnxruntime/third_party/kaldi/fstbin/fstaddselfloops.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstbin/fstaddsubsequentialloop.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstbin/fstcomposecontext.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstbin/fstcopy.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstbin/fstdeterminizelog.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstbin/fstdeterminizestar.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstbin/fstisstochastic.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstbin/fstmakecontextfst.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstbin/fstmakecontextsyms.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstbin/fstminimizeencoded.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstbin/fstphicompose.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstbin/fstpushspecial.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstbin/fstrand.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstbin/fstrmepslocal.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstbin/fstrmsymbols.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstbin/fsts-concat.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstbin/fsts-project.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstbin/fsts-to-transcripts.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstbin/fsts-union.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstbin/fsttablecompose.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstbin/make-grammar-fst.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/Makefile create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/context-fst-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/context-fst.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/context-fst.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/deterministic-fst-inl.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/deterministic-fst-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/deterministic-fst.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/determinize-lattice-inl.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/determinize-lattice-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/determinize-lattice.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/determinize-star-inl.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/determinize-star-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/determinize-star.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/epsilon-property-inl.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/epsilon-property-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/epsilon-property.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/factor-inl.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/factor-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/factor.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/fst-test-utils.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/fstext-lib.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/fstext-utils-inl.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/fstext-utils-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/fstext-utils.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/grammar-context-fst.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/grammar-context-fst.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/kaldi-fst-io-inl.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/kaldi-fst-io.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/kaldi-fst-io.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/lattice-utils-inl.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/lattice-utils-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/lattice-utils.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/lattice-weight-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/lattice-weight.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/pre-determinize-inl.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/pre-determinize-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/pre-determinize.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/prune-special-inl.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/prune-special-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/prune-special.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/push-special-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/push-special.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/push-special.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/rand-fst.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/remove-eps-local-inl.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/remove-eps-local-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/remove-eps-local.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/table-matcher-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/table-matcher.h create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/trivial-factor-weight-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/fstext/trivial-factor-weight.h create mode 100644 runtime/onnxruntime/third_party/kaldi/itf/clusterable-itf.h create mode 100644 runtime/onnxruntime/third_party/kaldi/itf/context-dep-itf.h create mode 100644 runtime/onnxruntime/third_party/kaldi/itf/decodable-itf.h create mode 100644 runtime/onnxruntime/third_party/kaldi/itf/online-feature-itf.h create mode 100644 runtime/onnxruntime/third_party/kaldi/itf/optimizable-itf.h create mode 100644 runtime/onnxruntime/third_party/kaldi/itf/options-itf.h create mode 100644 runtime/onnxruntime/third_party/kaldi/itf/transition-information.h create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/Makefile create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/arctic-weight.h create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/compose-lattice-pruned.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/compose-lattice-pruned.h create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/confidence.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/confidence.h create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/determinize-lattice-pruned-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/determinize-lattice-pruned.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/determinize-lattice-pruned.h create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/kaldi-lattice-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/kaldi-lattice.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/kaldi-lattice.h create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/lattice-functions-transition-model.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/lattice-functions-transition-model.h create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/lattice-functions.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/lattice-functions.h create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/phone-align-lattice.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/phone-align-lattice.h create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/push-lattice-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/push-lattice.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/push-lattice.h create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/sausages.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/sausages.h create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/word-align-lattice-lexicon-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/word-align-lattice-lexicon.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/word-align-lattice-lexicon.h create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/word-align-lattice.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/lat/word-align-lattice.h create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/Makefile create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/README create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/arpa-file-parser-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/arpa-file-parser.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/arpa-file-parser.h create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/arpa-lm-compiler-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/arpa-lm-compiler.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/arpa-lm-compiler.h create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/const-arpa-lm.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/const-arpa-lm.h create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/examples/CREATURE.txt create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/examples/DAYOFMONTH.txt create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/examples/MONTH.txt create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/examples/YEAR.txt create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/examples/YEARDATE.txt rename funasr/runtime/python/__init__.py => runtime/onnxruntime/third_party/kaldi/lm/examples/input3.fst (100%) create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/examples/input3.txt rename funasr/runtime/python/libtorch/__init__.py => runtime/onnxruntime/third_party/kaldi/lm/examples/input4.fst (100%) create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/examples/input4.txt create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/irstlm.mk create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/kaldi-rnnlm.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/kaldi-rnnlm.h create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/kenlm-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/kenlm.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/kenlm.h create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/mikolov-rnnlm-lib.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/mikolov-rnnlm-lib.h create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/test_data/input.arpa create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/test_data/missing_backoffs.arpa create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/test_data/missing_bos.arpa create mode 100644 runtime/onnxruntime/third_party/kaldi/lm/test_data/unused_backoffs.arpa create mode 100644 runtime/onnxruntime/third_party/kaldi/lmbin/Makefile create mode 100644 runtime/onnxruntime/third_party/kaldi/lmbin/arpa-to-const-arpa.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/lmbin/arpa2fst.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/util/Makefile create mode 100644 runtime/onnxruntime/third_party/kaldi/util/basic-filebuf.h create mode 100644 runtime/onnxruntime/third_party/kaldi/util/const-integer-set-inl.h create mode 100644 runtime/onnxruntime/third_party/kaldi/util/const-integer-set-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/util/const-integer-set.h create mode 100644 runtime/onnxruntime/third_party/kaldi/util/edit-distance-inl.h create mode 100644 runtime/onnxruntime/third_party/kaldi/util/edit-distance-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/util/edit-distance.h create mode 100644 runtime/onnxruntime/third_party/kaldi/util/hash-list-inl.h create mode 100644 runtime/onnxruntime/third_party/kaldi/util/hash-list-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/util/hash-list.h create mode 100644 runtime/onnxruntime/third_party/kaldi/util/kaldi-cygwin-io-inl.h create mode 100644 runtime/onnxruntime/third_party/kaldi/util/kaldi-io-inl.h create mode 100644 runtime/onnxruntime/third_party/kaldi/util/kaldi-io-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/util/kaldi-io.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/util/kaldi-io.h create mode 100644 runtime/onnxruntime/third_party/kaldi/util/kaldi-pipebuf.h create mode 100644 runtime/onnxruntime/third_party/kaldi/util/kaldi-semaphore.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/util/kaldi-semaphore.h create mode 100644 runtime/onnxruntime/third_party/kaldi/util/kaldi-thread-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/util/kaldi-thread.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/util/kaldi-thread.h create mode 100644 runtime/onnxruntime/third_party/kaldi/util/parse-options-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/util/parse-options.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/util/parse-options.h create mode 100644 runtime/onnxruntime/third_party/kaldi/util/simple-io-funcs.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/util/simple-io-funcs.h create mode 100644 runtime/onnxruntime/third_party/kaldi/util/simple-options-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/util/simple-options.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/util/simple-options.h create mode 100644 runtime/onnxruntime/third_party/kaldi/util/stl-utils-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/util/stl-utils.h create mode 100644 runtime/onnxruntime/third_party/kaldi/util/table-types.h create mode 100644 runtime/onnxruntime/third_party/kaldi/util/text-utils-test.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/util/text-utils.cc create mode 100644 runtime/onnxruntime/third_party/kaldi/util/text-utils.h rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/.editorconfig (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/.gitignore (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/BUILD.bazel (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/CMakeLists.txt (94%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/Makefile.am (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/Makefile.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/README.md (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/_import_release.sh (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/_package_release.cmd (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/aclocal.m4 (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/config.guess (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/config.h.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/config.sub (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/configure.ac (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/ltmain.sh (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/m4/ac_python_devel.m4 (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/m4/libtool.m4 (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/m4/ltoptions.m4 (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/m4/ltsugar.m4 (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/m4/ltversion.m4 (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/m4/lt~obsolete.m4 (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/openfst.sln (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/Makefile.am (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/Makefile.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/Makefile.am (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/Makefile.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/bin.vcxproj (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstarcsort-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstarcsort.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstclosure-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstclosure.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstcompile-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstcompile.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstcompose-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstcompose.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstconcat-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstconcat.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstconnect-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstconnect.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstconvert-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstconvert.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstdeterminize-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstdeterminize.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstdifference-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstdifference.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstdisambiguate-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstdisambiguate.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstdraw-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstdraw.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstencode-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstencode.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstepsnormalize-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstepsnormalize.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstequal-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstequal.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstequivalent-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstequivalent.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstinfo-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstinfo.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstintersect-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstintersect.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstinvert-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstinvert.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstisomorphic-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstisomorphic.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstmap-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstmap.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstminimize-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstminimize.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstprint-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstprint.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstproject-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstproject.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstprune-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstprune.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstpush-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstpush.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstrandgen-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstrandgen.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstrelabel-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstrelabel.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstreplace-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstreplace.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstreverse-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstreverse.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstreweight-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstreweight.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstrmepsilon-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstrmepsilon.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstshortestdistance-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstshortestdistance.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstshortestpath-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstshortestpath.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstsymbols-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstsymbols.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstsynchronize-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstsynchronize.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fsttopsort-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fsttopsort.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstunion-main.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/bin/fstunion.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/Makefile.am (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/Makefile.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/compact/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/compact/Makefile.am (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/compact/Makefile.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/compact/compact16_acceptor-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/compact/compact16_string-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/compact/compact16_unweighted-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/compact/compact16_unweighted_acceptor-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/compact/compact16_weighted_string-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/compact/compact64_acceptor-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/compact/compact64_string-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/compact/compact64_unweighted-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/compact/compact64_unweighted_acceptor-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/compact/compact64_weighted_string-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/compact/compact8_acceptor-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/compact/compact8_string-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/compact/compact8_unweighted-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/compact/compact8_unweighted_acceptor-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/compact/compact8_weighted_string-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/compress/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/compress/Makefile.am (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/compress/Makefile.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/compress/compress-script.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/compress/fstcompress.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/compress/fstrandmod.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/const/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/const/Makefile.am (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/const/Makefile.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/const/const16-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/const/const64-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/const/const8-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/far/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/far/Makefile.am (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/far/Makefile.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/far/far-class.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/far/farcompilestrings.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/far/farcreate.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/far/farequal.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/far/farextract.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/far/farinfo.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/far/farisomorphic.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/far/farprintstrings.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/far/farscript.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/far/getters.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/far/script-impl.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/far/stlist.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/far/strings.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/far/sttable.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/linear/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/linear/Makefile.am (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/linear/Makefile.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/linear/fstlinear.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/linear/fstloglinearapply.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/linear/linear-classifier-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/linear/linear-tagger-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/linear/linearscript.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/lookahead/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/lookahead/Makefile.am (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/lookahead/Makefile.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/lookahead/arc_lookahead-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/lookahead/ilabel_lookahead-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/lookahead/olabel_lookahead-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/mpdt/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/mpdt/Makefile.am (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/mpdt/Makefile.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/mpdt/mpdtcompose.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/mpdt/mpdtexpand.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/mpdt/mpdtinfo.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/mpdt/mpdtreverse.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/mpdt/mpdtscript.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/ngram/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/ngram/Makefile.am (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/ngram/Makefile.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/ngram/bitmap-index.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/ngram/ngram-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/ngram/nthbit.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/pdt/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/pdt/Makefile.am (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/pdt/Makefile.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/pdt/getters.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/pdt/pdtcompose.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/pdt/pdtexpand.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/pdt/pdtinfo.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/pdt/pdtreplace.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/pdt/pdtreverse.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/pdt/pdtscript.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/pdt/pdtshortestpath.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/python/Makefile.am (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/python/Makefile.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/python/basictypes.pxd (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/python/fst.pxd (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/python/ios.pxd (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/python/memory.pxd (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/python/pywrapfst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/python/pywrapfst.pxd (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/python/pywrapfst.pyx (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/special/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/special/Makefile.am (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/special/Makefile.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/special/fstspecial.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/special/phi-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/special/rho-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/extensions/special/sigma-fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/Makefile.am (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/Makefile.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/accumulator.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/add-on.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/arc-arena.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/arc-map.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/arc.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/arcfilter.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/arcsort.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/bi-table.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/cache.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/closure.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/compact-fst.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/compat.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/complement.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/compose-filter.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/compose.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/concat.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/config.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/config.h.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/connect.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/const-fst.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/determinize.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/dfs-visit.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/difference.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/disambiguate.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/edit-fst.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/encode.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/epsnormalize.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/equal.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/equivalent.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/expanded-fst.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/expectation-weight.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/compress/compress-script.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/compress/compress.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/compress/elias.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/compress/gzfile.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/compress/randmod.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/far/compile-strings.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/far/create.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/far/equal.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/far/extract.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/far/far-class.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/far/far.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/far/farlib.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/far/farscript.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/far/getters.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/far/info.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/far/isomorphic.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/far/print-strings.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/far/script-impl.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/far/stlist.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/far/sttable.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/linear/linear-fst-data-builder.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/linear/linear-fst-data.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/linear/linear-fst.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/linear/linearscript.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/linear/loglinear-apply.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/linear/trie.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/mpdt/compose.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/mpdt/expand.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/mpdt/info.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/mpdt/mpdt.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/mpdt/mpdtlib.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/mpdt/mpdtscript.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/mpdt/read_write_utils.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/mpdt/reverse.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/ngram/bitmap-index.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/ngram/ngram-fst.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/ngram/nthbit.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/pdt/collection.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/pdt/compose.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/pdt/expand.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/pdt/getters.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/pdt/info.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/pdt/paren.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/pdt/pdt.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/pdt/pdtlib.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/pdt/pdtscript.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/pdt/replace.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/pdt/reverse.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/pdt/shortest-path.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/special/phi-fst.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/special/rho-fst.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/extensions/special/sigma-fst.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/factor-weight.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/filter-state.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/flags.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/float-weight.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/fst-decl.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/fst.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/fstlib.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/generic-register.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/heap.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/icu.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/intersect.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/interval-set.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/invert.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/isomorphic.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/label-reachable.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/lexicographic-weight.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/lock.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/log.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/lookahead-filter.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/lookahead-matcher.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/map.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/mapped-file.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/matcher-fst.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/matcher.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/memory.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/minimize.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/mutable-fst.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/pair-weight.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/partition.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/power-weight.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/product-weight.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/project.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/properties.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/prune.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/push.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/queue.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/randequivalent.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/randgen.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/rational.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/register.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/relabel.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/replace-util.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/replace.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/reverse.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/reweight.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/rmepsilon.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/rmfinalepsilon.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/arc-class.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/arciterator-class.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/arcsort.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/arg-packs.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/closure.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/compile-impl.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/compile.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/compose.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/concat.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/connect.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/convert.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/decode.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/determinize.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/difference.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/disambiguate.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/draw-impl.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/draw.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/encode.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/encodemapper-class.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/epsnormalize.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/equal.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/equivalent.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/fst-class.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/fstscript-decl.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/fstscript.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/getters.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/info-impl.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/info.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/intersect.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/invert.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/isomorphic.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/map.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/minimize.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/print-impl.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/print.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/project.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/prune.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/push.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/randequivalent.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/randgen.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/register.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/relabel.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/replace.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/reverse.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/reweight.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/rmepsilon.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/script-impl.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/shortest-distance.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/shortest-path.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/stateiterator-class.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/synchronize.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/text-io.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/topsort.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/union.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/verify.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/script/weight-class.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/set-weight.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/shortest-distance.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/shortest-path.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/signed-log-weight.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/sparse-power-weight.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/sparse-tuple-weight.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/state-map.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/state-reachable.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/state-table.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/statesort.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/string-weight.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/string.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/symbol-table-ops.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/symbol-table.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/synchronize.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/test-properties.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/test/algo_test.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/test/fst_test.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/test/rand-fst.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/test/weight-tester.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/topsort.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/tuple-weight.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/types.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/union-find.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/union-weight.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/union.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/util.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/vector-fst.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/verify.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/visit.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/include/fst/weight.h (100%) create mode 100644 runtime/onnxruntime/third_party/openfst/src/lib/CMakeLists.txt rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/lib/Makefile.am (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/lib/Makefile.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/lib/compat.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/lib/flags.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/lib/fst-types.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/lib/fst.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/lib/libfst.vcxproj (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/lib/libfst.vcxproj.filters (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/lib/mapped-file.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/lib/properties.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/lib/symbol-table-ops.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/lib/symbol-table.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/lib/util.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/lib/weight.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/openfst-multibin.targets (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/openfst.props (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/openfst.targets (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/openfst.user.props (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/Makefile.am (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/Makefile.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/arciterator-class.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/arcsort.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/closure.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/compile.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/compose.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/concat.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/connect.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/convert.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/decode.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/determinize.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/difference.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/disambiguate.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/draw.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/encode.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/encodemapper-class.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/epsnormalize.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/equal.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/equivalent.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/fst-class.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/getters.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/info-impl.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/info.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/intersect.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/invert.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/isomorphic.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/libfstscript.vcxproj (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/libfstscript.vcxproj.filters (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/map.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/minimize.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/print.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/project.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/prune.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/push.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/randequivalent.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/randgen.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/relabel.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/replace.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/reverse.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/reweight.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/rmepsilon.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/shortest-distance.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/shortest-path.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/stateiterator-class.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/synchronize.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/text-io.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/topsort.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/union.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/verify.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/script/weight-class.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/test/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/test/Makefile.am (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/test/Makefile.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/test/algo_test.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/test/fst_test.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/openfst/src/test/weight_test.cc (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/anchor.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/binary.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/contrib/anchordict.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/contrib/graphbuilder.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/dll.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/emitfromevents.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/emitter.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/emitterdef.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/emittermanip.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/emitterstyle.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/eventhandler.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/exceptions.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/mark.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/node/convert.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/node/detail/bool_type.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/node/detail/impl.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/node/detail/iterator.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/node/detail/iterator_fwd.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/node/detail/memory.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/node/detail/node.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/node/detail/node_data.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/node/detail/node_iterator.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/node/detail/node_ref.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/node/emit.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/node/impl.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/node/iterator.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/node/node.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/node/parse.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/node/ptr.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/node/type.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/noncopyable.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/null.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/ostream_wrapper.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/parser.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/stlemitter.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/traits.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/include/yaml-cpp/yaml.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/install.txt (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/binary.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/collectionstack.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/contrib/graphbuilder.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/contrib/graphbuilderadapter.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/contrib/graphbuilderadapter.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/convert.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/directives.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/directives.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/emit.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/emitfromevents.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/emitter.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/emitterstate.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/emitterstate.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/emitterutils.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/emitterutils.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/exceptions.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/exp.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/exp.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/indentation.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/memory.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/node.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/node_data.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/nodebuilder.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/nodebuilder.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/nodeevents.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/nodeevents.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/null.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/ostream_wrapper.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/parse.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/parser.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/ptr_vector.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/regex_yaml.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/regex_yaml.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/regeximpl.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/scanner.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/scanner.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/scanscalar.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/scanscalar.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/scantag.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/scantag.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/scantoken.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/setting.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/simplekey.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/singledocparser.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/singledocparser.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/stream.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/stream.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/streamcharsource.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/stringsource.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/tag.cpp (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/tag.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/src/token.h (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/yaml-cpp-config-version.cmake.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/yaml-cpp-config.cmake.in (100%) rename {funasr/runtime => runtime}/onnxruntime/third_party/yaml-cpp/yaml-cpp.pc.cmake (100%) rename {funasr/runtime/python/libtorch/funasr_torch/utils => runtime/python}/__init__.py (100%) rename {funasr/runtime => runtime}/python/grpc/Readme.md (100%) rename {funasr/runtime => runtime}/python/grpc/grpc_main_client.py (100%) rename {funasr/runtime => runtime}/python/grpc/proto/Readme.md (100%) rename {funasr/runtime => runtime}/python/grpc/proto/paraformer.proto (100%) rename {funasr/runtime => runtime}/python/grpc/proto/workflow.png (100%) rename {funasr/runtime => runtime}/python/grpc/requirements.txt (100%) rename {funasr/runtime => runtime}/python/http/README.md (100%) rename {funasr/runtime => runtime}/python/http/asr_nginx.conf (100%) rename {funasr/runtime => runtime}/python/http/client.py (100%) rename {funasr/runtime => runtime}/python/http/requirements.txt (100%) rename {funasr/runtime => runtime}/python/http/server.py (100%) rename {funasr/runtime => runtime}/python/http/start_server.sh (100%) rename {funasr/runtime => runtime}/python/libtorch/README.md (100%) rename {funasr/runtime/python/onnxruntime => runtime/python/libtorch}/__init__.py (100%) rename {funasr/runtime => runtime}/python/libtorch/demo.py (100%) rename {funasr/runtime => runtime}/python/libtorch/funasr_torch/__init__.py (100%) rename {funasr/runtime => runtime}/python/libtorch/funasr_torch/paraformer_bin.py (100%) rename {funasr/runtime/python/onnxruntime/funasr_onnx => runtime/python/libtorch/funasr_torch}/utils/__init__.py (100%) rename {funasr/runtime => runtime}/python/libtorch/funasr_torch/utils/compute_wer.py (100%) rename {funasr/runtime => runtime}/python/libtorch/funasr_torch/utils/frontend.py (100%) rename {funasr/runtime => runtime}/python/libtorch/funasr_torch/utils/postprocess_utils.py (100%) rename {funasr/runtime => runtime}/python/libtorch/funasr_torch/utils/timestamp_utils.py (100%) rename {funasr/runtime => runtime}/python/libtorch/funasr_torch/utils/utils.py (100%) rename {funasr/runtime => runtime}/python/libtorch/setup.py (100%) rename {funasr/runtime => runtime}/python/onnxruntime/README.md (100%) rename funasr/runtime/triton_gpu/model_repo_paraformer_large_offline/encoder/1/.gitkeep => runtime/python/onnxruntime/__init__.py (100%) rename {funasr/runtime => runtime}/python/onnxruntime/demo_contextual_paraformer.py (100%) rename {funasr/runtime => runtime}/python/onnxruntime/demo_paraformer_offline.py (100%) rename {funasr/runtime => runtime}/python/onnxruntime/demo_paraformer_online.py (100%) rename {funasr/runtime => runtime}/python/onnxruntime/demo_punc_offline.py (100%) rename {funasr/runtime => runtime}/python/onnxruntime/demo_punc_online.py (100%) rename {funasr/runtime => runtime}/python/onnxruntime/demo_vad_offline.py (100%) rename {funasr/runtime => runtime}/python/onnxruntime/demo_vad_online.py (100%) rename {funasr/runtime => runtime}/python/onnxruntime/funasr_onnx/__init__.py (100%) rename {funasr/runtime => runtime}/python/onnxruntime/funasr_onnx/paraformer_bin.py (100%) rename {funasr/runtime => runtime}/python/onnxruntime/funasr_onnx/paraformer_online_bin.py (100%) rename {funasr/runtime => runtime}/python/onnxruntime/funasr_onnx/punc_bin.py (100%) rename funasr/runtime/triton_gpu/model_repo_paraformer_large_offline/infer_pipeline/1/.gitkeep => runtime/python/onnxruntime/funasr_onnx/utils/__init__.py (100%) rename {funasr/runtime => runtime}/python/onnxruntime/funasr_onnx/utils/e2e_vad.py (100%) rename {funasr/runtime => runtime}/python/onnxruntime/funasr_onnx/utils/frontend.py (100%) rename {funasr/runtime => runtime}/python/onnxruntime/funasr_onnx/utils/postprocess_utils.py (100%) rename {funasr/runtime => runtime}/python/onnxruntime/funasr_onnx/utils/timestamp_utils.py (100%) rename {funasr/runtime => runtime}/python/onnxruntime/funasr_onnx/utils/utils.py (100%) rename {funasr/runtime => runtime}/python/onnxruntime/funasr_onnx/vad_bin.py (100%) rename {funasr/runtime => runtime}/python/onnxruntime/setup.py (100%) rename {funasr/runtime => runtime}/python/utils/compute_wer.py (100%) rename {funasr/runtime => runtime}/python/utils/proce_text.py (100%) rename {funasr/runtime => runtime}/python/utils/requirements.txt (100%) rename {funasr/runtime => runtime}/python/utils/split_scp.pl (100%) rename {funasr/runtime => runtime}/python/utils/test_cer.py (100%) rename {funasr/runtime => runtime}/python/utils/test_cer.sh (100%) rename {funasr/runtime => runtime}/python/utils/test_rtf.py (100%) rename {funasr/runtime => runtime}/python/utils/test_rtf.sh (100%) rename {funasr/runtime => runtime}/python/utils/test_rtf_gpu.py (100%) rename {funasr/runtime => runtime}/python/websocket/README.md (100%) rename {funasr/runtime => runtime}/python/websocket/funasr_client_api.py (100%) rename {funasr/runtime => runtime}/python/websocket/funasr_wss_client.py (100%) rename {funasr/runtime => runtime}/python/websocket/funasr_wss_server.py (100%) rename {funasr/runtime => runtime}/python/websocket/requirements_client.txt (100%) rename {funasr/runtime => runtime}/python/websocket/requirements_server.txt (100%) rename {funasr/runtime => runtime}/python/websocket/ssl_key (100%) rename {funasr/runtime => runtime}/readme.md (69%) rename {funasr/runtime => runtime}/readme_cn.md (62%) rename {funasr/runtime => runtime}/run_server.sh (56%) rename {funasr/runtime => runtime}/run_server_2pass.sh (58%) rename {funasr/runtime => runtime}/ssl_key/readme.md (100%) rename {funasr/runtime => runtime}/ssl_key/readme_cn.md (100%) rename {funasr/runtime => runtime}/ssl_key/server.crt (100%) rename {funasr/runtime => runtime}/ssl_key/server.key (100%) create mode 100755 runtime/tools/fst/add_lex_disambig.pl create mode 100755 runtime/tools/fst/collect_resource_file.sh create mode 100755 runtime/tools/fst/compile_dict_token.sh create mode 100755 runtime/tools/fst/ctc_token_fst.py create mode 100755 runtime/tools/fst/eps2disambig.pl create mode 100755 runtime/tools/fst/generate_lexicon.py create mode 100755 runtime/tools/fst/make_decode_graph.sh create mode 100755 runtime/tools/fst/make_lexicon_fst.pl create mode 100755 runtime/tools/fst/parse_options.sh create mode 100755 runtime/tools/fst/s2eps.pl create mode 100755 runtime/tools/fst/train_lms.sh create mode 100644 runtime/tools/path.sh create mode 100644 runtime/tools/train_compile_ngram.sh rename {funasr/runtime => runtime}/triton_gpu/Dockerfile/Dockerfile.server (100%) rename {funasr/runtime => runtime}/triton_gpu/README.md (100%) rename {funasr/runtime => runtime}/triton_gpu/client/aishell_test.txt (100%) rename {funasr/runtime => runtime}/triton_gpu/client/client.py (100%) rename {funasr/runtime => runtime}/triton_gpu/client/decode_manifest_triton.py (100%) rename {funasr/runtime => runtime}/triton_gpu/client/decode_manifest_triton_wo_cuts.py (100%) rename {funasr/runtime => runtime}/triton_gpu/client/speech_client.py (100%) rename {funasr/runtime => runtime}/triton_gpu/client/test_wavs/long.wav (100%) rename {funasr/runtime => runtime}/triton_gpu/client/test_wavs/mid.wav (100%) rename {funasr/runtime => runtime}/triton_gpu/client/utils.py (100%) create mode 100644 runtime/triton_gpu/model_repo_paraformer_large_offline/encoder/1/.gitkeep rename {funasr/runtime => runtime}/triton_gpu/model_repo_paraformer_large_offline/encoder/config.pbtxt (100%) rename {funasr/runtime => runtime}/triton_gpu/model_repo_paraformer_large_offline/feature_extractor/1/model.py (100%) rename {funasr/runtime => runtime}/triton_gpu/model_repo_paraformer_large_offline/feature_extractor/config.pbtxt (100%) create mode 100644 runtime/triton_gpu/model_repo_paraformer_large_offline/infer_pipeline/1/.gitkeep rename {funasr/runtime => runtime}/triton_gpu/model_repo_paraformer_large_offline/infer_pipeline/config.pbtxt (100%) rename {funasr/runtime => runtime}/triton_gpu/model_repo_paraformer_large_offline/scoring/1/model.py (100%) rename {funasr/runtime => runtime}/triton_gpu/model_repo_paraformer_large_offline/scoring/config.pbtxt (100%) rename {funasr/runtime => runtime}/websocket/CMakeLists.txt (94%) rename {funasr/runtime => runtime}/websocket/bin/CMakeLists.txt (100%) rename {funasr/runtime => runtime}/websocket/bin/funasr-wss-client-2pass.cpp (90%) rename {funasr/runtime => runtime}/websocket/bin/funasr-wss-client.cpp (90%) rename {funasr/runtime => runtime}/websocket/bin/funasr-wss-server-2pass.cpp (95%) rename {funasr/runtime => runtime}/websocket/bin/funasr-wss-server.cpp (72%) rename {funasr/runtime => runtime}/websocket/bin/microphone.cpp (100%) rename {funasr/runtime => runtime}/websocket/bin/microphone.h (100%) rename {funasr/runtime => runtime}/websocket/bin/websocket-server-2pass.cpp (90%) rename {funasr/runtime => runtime}/websocket/bin/websocket-server-2pass.h (99%) rename {funasr/runtime => runtime}/websocket/bin/websocket-server.cpp (78%) rename {funasr/runtime => runtime}/websocket/bin/websocket-server.h (91%) create mode 100644 runtime/websocket/hotwords.txt rename {funasr/runtime => runtime}/websocket/readme.md (94%) rename {funasr/runtime => runtime}/websocket/readme_zh.md (94%) rename {funasr/runtime => runtime}/websocket/requirements_install.md (100%) diff --git a/README_zh.md b/README_zh.md index f58d243e3..051a4d274 100644 --- a/README_zh.md +++ b/README_zh.md @@ -44,15 +44,80 @@ FunASR希望在语音识别的学术研究和工业应用之间架起一座桥 ## 安装教程 FunASR安装教程请阅读（[Installation](https://alibaba-damo-academy.github.io/FunASR/en/installation/installation.html)） - -## 服务部署 -FunASR支持预训练或者进一步微调的模型进行服务部署。目前中文离线文件转写服务一键部署的CPU版本已经发布，详细信息参阅([一键部署文档](funasr/runtime/docs/SDK_tutorial_zh.md)。更多服务部署详细信息可以参阅([服务部署文档](funasr/runtime/readme_cn.md))。 +## 模型仓库 + +FunASR开源了大量在工业数据上预训练模型，您可以在[模型许可协议](./MODEL_LICENSE)下自由使用、复制、修改和分享FunASR模型，下面列举代表性的模型，更多模型请参考[模型仓库]()。 + +（注：[🤗]()表示Huggingface模型仓库链接，[⭐]()表示ModelScope模型仓库链接） + + +| 模型名字 | 任务详情 | 训练数据 | 参数量 | +|:------------------------------------------------------------------------------------------------------------------------------------------------------:|:------------------:|:------------:|:----:| +| paraformer-zh ([🤗]() [⭐](https://www.modelscope.cn/models/damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch/summary) ) | 语音识别，带时间戳输出，非实时 | 60000小时，中文 | 220M | +| paraformer-zh-spk ([🤗]() [⭐](https://modelscope.cn/models/damo/speech_paraformer-large-vad-punc-spk_asr_nat-zh-cn/summary) ) | 分角色语音识别，带时间戳输出，非实时 | 60000小时，中文 | 220M | +| paraformer-zh-online ([🤗]() [⭐](https://modelscope.cn/models/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online/summary) ) | 语音识别，实时 | 60000小时，中文 | 220M | +| paraformer-en ([🤗]() [⭐](https://www.modelscope.cn/models/damo/speech_paraformer-large-vad-punc_asr_nat-en-16k-common-vocab10020/summary) ) | 分角色语音识别，带时间戳输出，非实时 | 50000小时，英文 | 220M | +| paraformer-en-spk ([🤗]() [⭐]() ) | 语音识别，非实时 | 50000小时，英文 | 220M | +| conformer-en ([🤗]() [⭐](https://modelscope.cn/models/damo/speech_conformer_asr-en-16k-vocab4199-pytorch/summary) ) | 语音识别，非实时 | 50000小时，英文 | 220M | +| ct-punc ([🤗]() [⭐](https://modelscope.cn/models/damo/punc_ct-transformer_cn-en-common-vocab471067-large/summary) ) | 标点恢复，非实时 | 100M，中文与英文 | 1.1G | +| fsmn-vad ([🤗]() [⭐](https://modelscope.cn/models/damo/speech_fsmn_vad_zh-cn-16k-common-pytorch/summary) ) | 语音端点检测，实时 | 5000小时，中文与英文 | 0.4M | +| fa-zh ([🤗]() [⭐](https://modelscope.cn/models/damo/speech_timestamp_prediction-v1-16k-offline/summary) ) | 字级别时间戳预测 | 50000小时，中文 | 38M | + ## 快速开始 -快速使用教程（[新人文档](https://alibaba-damo-academy.github.io/FunASR/en/funasr/quick_start_zh.html)） +FunASR支持数万小时工业数据训练的模型的推理和微调，详细信息可以参阅（[modelscope_egs](https://alibaba-damo-academy.github.io/FunASR/en/modelscope_pipeline/quick_start.html)）；也支持学术标准数据集模型的训练和微调，详细信息可以参阅（[egs](https://alibaba-damo-academy.github.io/FunASR/en/academic_recipe/asr_recipe.html)）。 + +下面为快速上手教程，测试音频（[中文](https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/vad_example.wav)，[英文]()） +### 非实时语音识别 +```python +from funasr import infer + +p = infer(model="paraformer-zh", vad_model="fsmn-vad", punc_model="ct-punc", model_hub="ms") + +res = p("asr_example_zh.wav", batch_size_token=5000) +print(res) +``` +注：`model_hub`：表示模型仓库，`ms`为选择modelscope下载，`hf`为选择huggingface下载。 + +### 实时语音识别 +```python +from funasr import infer + +p = infer(model="paraformer-zh-streaming", model_hub="ms") + +chunk_size = [0, 10, 5] #[0, 10, 5] 600ms, [0, 8, 4] 480ms +param_dict = {"cache": dict(), "is_final": False, "chunk_size": chunk_size, "encoder_chunk_look_back": 4, "decoder_chunk_look_back": 1} + +import torchaudio +speech = torchaudio.load("asr_example_zh.wav")[0][0] +speech_length = speech.shape[0] + +stride_size = chunk_size[1] * 960 +sample_offset = 0 +for sample_offset in range(0, speech_length, min(stride_size, speech_length - sample_offset)): + param_dict["is_final"] = True if sample_offset + stride_size >= speech_length - 1 else False + input = speech[sample_offset: sample_offset + stride_size] + rec_result = p(input=input, param_dict=param_dict) + print(rec_result) +``` +注：`chunk_size`为流式延时配置，`[0,10,5]`表示上屏实时出字粒度为`10*60=600ms`，未来信息为`5*60=300ms`。每次推理输入为`600ms`（采样点数为`16000*0.6=960`），输出为对应文字，最后一个语音片段输入需要设置`is_final=True`来强制输出最后一个字。 + +更多详细用法（[新人文档](https://alibaba-damo-academy.github.io/FunASR/en/funasr/quick_start_zh.html)） + + + +## 服务部署 +FunASR支持预训练或者进一步微调的模型进行服务部署。目前支持以下几种服务部署： + +- 中文离线文件转写服务（CPU版本），已完成 +- 中文流式语音识别服务（CPU版本），已完成 +- 英文离线文件转写服务（CPU版本），已完成 +- 中文离线文件转写服务（GPU版本），进行中 +- 更多支持中 + +详细信息可以参阅([服务部署文档](funasr/runtime/readme_cn.md))。 -FunASR支持数万小时工业数据训练的模型的推理和微调，详细信息可以参阅（[modelscope_egs](https://alibaba-damo-academy.github.io/FunASR/en/modelscope_pipeline/quick_start.html)）；也支持学术标准数据集模型的训练和微调，详细信息可以参阅（[egs](https://alibaba-damo-academy.github.io/FunASR/en/academic_recipe/asr_recipe.html)）。 模型包含语音识别（ASR）、语音活动检测（VAD）、标点恢复、语言模型、说话人验证、说话人分离和多人对话语音识别等，详细模型列表可以参阅[模型仓库](https://github.com/alibaba-damo-academy/FunASR/blob/main/docs/model_zoo/modelscope_models.md)： ## 联系我们 diff --git a/funasr/__init__.py b/funasr/__init__.py index f297bc3e6..1f31505ac 100644 --- a/funasr/__init__.py +++ b/funasr/__init__.py @@ -1,8 +1,132 @@ """Initialize funasr package.""" import os +from pathlib import Path +import torch +import numpy as np dirname = os.path.dirname(__file__) version_file = os.path.join(dirname, "version.txt") with open(version_file, "r") as f: __version__ = f.read().strip() + + +def prepare_model( + model: str = None, + # mode: str = None, + vad_model: str = None, + punc_model: str = None, + model_hub: str = "ms", + cache_dir: str = None, + **kwargs, +): + if not Path(model).exists(): + if model_hub == "ms" or model_hub == "modelscope": + try: + from modelscope.hub.snapshot_download import snapshot_download as download_tool + model = name_maps_ms[model] if model is not None else None + vad_model = name_maps_ms[vad_model] if vad_model is not None else None + punc_model = name_maps_ms[punc_model] if punc_model is not None else None + except: + raise "You are exporting model from modelscope, please install modelscope and try it again. To install modelscope, you could:\n" \ + "\npip3 install -U modelscope\n" \ + "For the users in China, you could install with the command:\n" \ + "\npip3 install -U modelscope -i https://mirror.sjtu.edu.cn/pypi/web/simple" + elif model_hub == "hf" or model_hub == "huggingface": + download_tool = 0 + else: + raise "model_hub must be on of ms or hf, but get {}".format(model_hub) + try: + model = download_tool(model, cache_dir=cache_dir, revision=kwargs.get("revision", None)) + print("model have been downloaded to: {}".format(model)) + except: + raise "model_dir must be model_name in modelscope or local path downloaded from modelscope, but is {}".format( + model) + + if vad_model is not None and not Path(vad_model).exists(): + vad_model = download_tool(vad_model, cache_dir=cache_dir) + print("model have been downloaded to: {}".format(vad_model)) + if punc_model is not None and not Path(punc_model).exists(): + punc_model = download_tool(punc_model, cache_dir=cache_dir) + print("model have been downloaded to: {}".format(punc_model)) + + # asr + kwargs.update({"cmvn_file": None if model is None else os.path.join(model, "am.mvn"), + "asr_model_file": None if model is None else os.path.join(model, "model.pb"), + "asr_train_config": None if model is None else os.path.join(model, "config.yaml"), + }) + mode = kwargs.get("mode", None) + if mode is None: + import json + json_file = os.path.join(model, 'configuration.json') + with open(json_file, 'r') as f: + config_data = json.load(f) + if config_data['task'] == "punctuation": + mode = config_data['model']['punc_model_config']['mode'] + else: + mode = config_data['model']['model_config']['mode'] + if vad_model is not None and "vad" not in mode: + mode = "paraformer_vad" + kwargs["mode"] = mode + # vad + kwargs.update({"vad_cmvn_file": None if vad_model is None else os.path.join(vad_model, "vad.mvn"), + "vad_model_file": None if vad_model is None else os.path.join(vad_model, "vad.pb"), + "vad_infer_config": None if vad_model is None else os.path.join(vad_model, "vad.yaml"), + }) + # punc + kwargs.update({ + "punc_model_file": None if punc_model is None else os.path.join(punc_model, "punc.pb"), + "punc_infer_config": None if punc_model is None else os.path.join(punc_model, "punc.yaml"), + }) + + + return model, vad_model, punc_model, kwargs + +name_maps_ms = { + "paraformer-zh": "damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch", + "paraformer-zh-spk": "damo/speech_paraformer-large-vad-punc-spk_asr_nat-zh-cn", + "paraformer-en": "damo/speech_paraformer-large-vad-punc_asr_nat-en-16k-common-vocab10020", + "paraformer-en-spk": "damo/speech_paraformer-large-vad-punc_asr_nat-en-16k-common-vocab10020", + "paraformer-zh-streaming": "damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online", + "fsmn-vad": "damo/speech_fsmn_vad_zh-cn-16k-common-pytorch", + "ct-punc": "damo/punc_ct-transformer_cn-en-common-vocab471067-large", + "fa-zh": "damo/speech_timestamp_prediction-v1-16k-offline", +} + +def infer(task_name: str = "asr", + model: str = None, + # mode: str = None, + vad_model: str = None, + punc_model: str = None, + model_hub: str = "ms", + cache_dir: str = None, + **kwargs, + ): + + model, vad_model, punc_model, kwargs = prepare_model(model, vad_model, punc_model, model_hub, cache_dir, **kwargs) + if task_name == "asr": + from funasr.bin.asr_inference_launch import inference_launch + + inference_pipeline = inference_launch(**kwargs) + elif task_name == "": + pipeline = 1 + elif task_name == "": + pipeline = 2 + elif task_name == "": + pipeline = 2 + + def _infer_fn(input, **kwargs): + data_type = kwargs.get('data_type', 'sound') + data_path_and_name_and_type = [input, 'speech', data_type] + raw_inputs = None + if isinstance(input, torch.Tensor): + input = input.numpy() + if isinstance(input, np.ndarray): + data_path_and_name_and_type = None + raw_inputs = input + + + + return inference_pipeline(data_path_and_name_and_type, raw_inputs=raw_inputs, **kwargs) + + return _infer_fn \ No newline at end of file diff --git a/funasr/bin/asr_inference_launch.py b/funasr/bin/asr_inference_launch.py index 04ddd925a..e93d74037 100644 --- a/funasr/bin/asr_inference_launch.py +++ b/funasr/bin/asr_inference_launch.py @@ -462,18 +462,18 @@ def inference_paraformer( def inference_paraformer_vad_punc( - maxlenratio: float, - minlenratio: float, - batch_size: int, - beam_size: int, - ngpu: int, - ctc_weight: float, - lm_weight: float, - penalty: float, - log_level: Union[int, str], + maxlenratio: float=0.0, + minlenratio: float=0.0, + batch_size: int=1, + beam_size: int=1, + ngpu: int=1, + ctc_weight: float=0.0, + lm_weight: float=0.0, + penalty: float=0.0, + log_level: Union[int, str]=logging.ERROR, # data_path_and_name_and_type, - asr_train_config: Optional[str], - asr_model_file: Optional[str], + asr_train_config: Optional[str]=None, + asr_model_file: Optional[str]=None, cmvn_file: Optional[str] = None, lm_train_config: Optional[str] = None, lm_file: Optional[str] = None, @@ -487,7 +487,7 @@ def inference_paraformer_vad_punc( seed: int = 0, ngram_weight: float = 0.9, nbest: int = 1, - num_workers: int = 1, + num_workers: int = 0, vad_infer_config: Optional[str] = None, vad_model_file: Optional[str] = None, vad_cmvn_file: Optional[str] = None, @@ -1098,18 +1098,18 @@ def inference_paraformer_vad_speaker( def inference_paraformer_online( - maxlenratio: float, - minlenratio: float, - batch_size: int, - beam_size: int, - ngpu: int, - ctc_weight: float, - lm_weight: float, - penalty: float, - log_level: Union[int, str], + maxlenratio: float=0.0, + minlenratio: float=0.0, + batch_size: int=1, + beam_size: int=1, + ngpu: int=1, + ctc_weight: float=0.0, + lm_weight: float=0.0, + penalty: float=0.0, + log_level: Union[int, str]=logging.ERROR, # data_path_and_name_and_type, - asr_train_config: Optional[str], - asr_model_file: Optional[str], + asr_train_config: Optional[str]=None, + asr_model_file: Optional[str]=None, cmvn_file: Optional[str] = None, lm_train_config: Optional[str] = None, lm_file: Optional[str] = None, diff --git a/funasr/runtime/onnxruntime/src/util.cpp b/funasr/runtime/onnxruntime/src/util.cpp deleted file mode 100644 index 70059ca21..000000000 --- a/funasr/runtime/onnxruntime/src/util.cpp +++ /dev/null @@ -1,231 +0,0 @@ - -#include "precomp.h" - -namespace funasr { -float *LoadParams(const char *filename) -{ - - FILE *fp; - fp = fopen(filename, "rb"); - fseek(fp, 0, SEEK_END); - uint32_t nFileLen = ftell(fp); - fseek(fp, 0, SEEK_SET); - - float *params_addr = (float *)AlignedMalloc(32, nFileLen); - int n = fread(params_addr, 1, nFileLen, fp); - fclose(fp); - - return params_addr; -} - -int ValAlign(int val, int align) -{ - float tmp = ceil((float)val / (float)align) * (float)align; - return (int)tmp; -} - -void DispParams(float *din, int size) -{ - int i; - for (i = 0; i < size; i++) { - printf("%f ", din[i]); - } - printf("\n"); -} -void SaveDataFile(const char *filename, void *data, uint32_t len) -{ - FILE *fp; - fp = fopen(filename, "wb+"); - fwrite(data, 1, len, fp); - fclose(fp); -} - -void BasicNorm(Tensor *&din, float norm) -{ - - int Tmax = din->size[2]; - - int i, j; - for (i = 0; i < Tmax; i++) { - float sum = 0; - for (j = 0; j < 512; j++) { - int ii = i * 512 + j; - sum += din->buff[ii] * din->buff[ii]; - } - float mean = sqrt(sum / 512 + norm); - for (j = 0; j < 512; j++) { - int ii = i * 512 + j; - din->buff[ii] = din->buff[ii] / mean; - } - } -} - -void FindMax(float *din, int len, float &max_val, int &max_idx) -{ - int i; - max_val = -INFINITY; - max_idx = -1; - for (i = 0; i < len; i++) { - if (din[i] > max_val) { - max_val = din[i]; - max_idx = i; - } - } -} - -string PathAppend(const string &p1, const string &p2) -{ - - char sep = '/'; - string tmp = p1; - -#ifdef _WIN32 - sep = '\\'; -#endif - - if (p1[p1.length()-1] != sep) { // Need to add a - tmp += sep; // path separator - return (tmp + p2); - } else - return (p1 + p2); -} - -void Relu(Tensor *din) -{ - int i; - for (i = 0; i < din->buff_size; i++) { - float val = din->buff[i]; - din->buff[i] = val < 0 ? 0 : val; - } -} - -void Swish(Tensor *din) -{ - int i; - for (i = 0; i < din->buff_size; i++) { - float val = din->buff[i]; - din->buff[i] = val / (1 + exp(-val)); - } -} - -void Sigmoid(Tensor *din) -{ - int i; - for (i = 0; i < din->buff_size; i++) { - float val = din->buff[i]; - din->buff[i] = 1 / (1 + exp(-val)); - } -} - -void DoubleSwish(Tensor *din) -{ - int i; - for (i = 0; i < din->buff_size; i++) { - float val = din->buff[i]; - din->buff[i] = val / (1 + exp(-val + 1)); - } -} - -void Softmax(float *din, int mask, int len) -{ - float *tmp = (float *)malloc(mask * sizeof(float)); - int i; - float sum = 0; - float max = -INFINITY; - - for (i = 0; i < mask; i++) { - max = max < din[i] ? din[i] : max; - } - - for (i = 0; i < mask; i++) { - tmp[i] = exp(din[i] - max); - sum += tmp[i]; - } - for (i = 0; i < mask; i++) { - din[i] = tmp[i] / sum; - } - free(tmp); - for (i = mask; i < len; i++) { - din[i] = 0; - } -} - -void LogSoftmax(float *din, int len) -{ - float *tmp = (float *)malloc(len * sizeof(float)); - int i; - float sum = 0; - for (i = 0; i < len; i++) { - tmp[i] = exp(din[i]); - sum += tmp[i]; - } - for (i = 0; i < len; i++) { - din[i] = log(tmp[i] / sum); - } - free(tmp); -} - -void Glu(Tensor *din, Tensor *dout) -{ - int mm = din->buff_size / 1024; - int i, j; - for (i = 0; i < mm; i++) { - for (j = 0; j < 512; j++) { - int in_off = i * 1024 + j; - int out_off = i * 512 + j; - float a = din->buff[in_off]; - float b = din->buff[in_off + 512]; - dout->buff[out_off] = a / (1 + exp(-b)); - } - } -} - -bool is_target_file(const std::string& filename, const std::string target) { - std::size_t pos = filename.find_last_of("."); - if (pos == std::string::npos) { - return false; - } - std::string extension = filename.substr(pos + 1); - return (extension == target); -} - -void KeepChineseCharacterAndSplit(const std::string &input_str, - std::vector &chinese_characters) { - chinese_characters.resize(0); - std::vector u16_buf; - u16_buf.resize(std::max(u16_buf.size(), input_str.size() + 1)); - U16CHAR_T* pu16 = u16_buf.data(); - U8CHAR_T * pu8 = (U8CHAR_T*)input_str.data(); - size_t ilen = input_str.size(); - size_t len = EncodeConverter::Utf8ToUtf16(pu8, ilen, pu16, ilen + 1); - for (size_t i = 0; i < len; i++) { - if (EncodeConverter::IsChineseCharacter(pu16[i])) { - U8CHAR_T u8buf[4]; - size_t n = EncodeConverter::Utf16ToUtf8(pu16 + i, u8buf); - u8buf[n] = '\0'; - chinese_characters.push_back((const char*)u8buf); - } - } -} - -std::vector split(const std::string &s, char delim) { - std::vector elems; - std::stringstream ss(s); - std::string item; - while(std::getline(ss, item, delim)) { - elems.push_back(item); - } - return elems; -} - -template -void PrintMat(const std::vector> &mat, const std::string &name) { - std::cout << name << ":" << std::endl; - for (auto item : mat) { - for (auto item_ : item) { - std::cout << item_ << " "; - } - std::cout << std::endl; - } -} -} // namespace funasr diff --git a/funasr/runtime/onnxruntime/src/util.h b/funasr/runtime/onnxruntime/src/util.h deleted file mode 100644 index df9abd1e0..000000000 --- a/funasr/runtime/onnxruntime/src/util.h +++ /dev/null @@ -1,38 +0,0 @@ -#ifndef UTIL_H -#define UTIL_H - -using namespace std; - -namespace funasr { -extern float *LoadParams(const char *filename); - -extern void SaveDataFile(const char *filename, void *data, uint32_t len); -extern void Relu(Tensor *din); -extern void Swish(Tensor *din); -extern void Sigmoid(Tensor *din); -extern void DoubleSwish(Tensor *din); - -extern void Softmax(float *din, int mask, int len); - -extern void LogSoftmax(float *din, int len); -extern int ValAlign(int val, int align); -extern void DispParams(float *din, int size); - -extern void BasicNorm(Tensor *&din, float norm); - -extern void FindMax(float *din, int len, float &max_val, int &max_idx); - -extern void Glu(Tensor *din, Tensor *dout); - -string PathAppend(const string &p1, const string &p2); -bool is_target_file(const std::string& filename, const std::string target); - -void KeepChineseCharacterAndSplit(const std::string &input_str, - std::vector &chinese_characters); - -std::vector split(const std::string &s, char delim); - -template -void PrintMat(const std::vector> &mat, const std::string &name); -} // namespace funasr -#endif diff --git a/funasr/runtime/onnxruntime/third_party/openfst/src/lib/CMakeLists.txt b/funasr/runtime/onnxruntime/third_party/openfst/src/lib/CMakeLists.txt deleted file mode 100644 index c004aee10..000000000 --- a/funasr/runtime/onnxruntime/third_party/openfst/src/lib/CMakeLists.txt +++ /dev/null @@ -1,26 +0,0 @@ -FILE(GLOB HEADER_FILES ../include/fst/*.h) - - -add_library(fst - compat.cc - flags.cc - fst-types.cc - fst.cc - mapped-file.cc - properties.cc - symbol-table.cc - symbol-table-ops.cc - util.cc - weight.cc - ${HEADER_FILES} -) -set_target_properties(fst PROPERTIES - SOVERSION "${SOVERSION}" -) - - - -install(TARGETS fst - LIBRARY DESTINATION lib - ARCHIVE DESTINATION lib - RUNTIME DESTINATION lib) diff --git a/funasr/runtime/python/grpc/.gitignore b/funasr/runtime/python/grpc/.gitignore deleted file mode 100644 index eeb8a6ec4..000000000 --- a/funasr/runtime/python/grpc/.gitignore +++ /dev/null @@ -1 +0,0 @@ -**/__pycache__ diff --git a/funasr/runtime/websocket/hotwords.txt b/funasr/runtime/websocket/hotwords.txt deleted file mode 100644 index 6179cbc6a..000000000 --- a/funasr/runtime/websocket/hotwords.txt +++ /dev/null @@ -1,2 +0,0 @@ -阿里巴巴 -通义实验室 diff --git a/funasr/text/cleaner.py b/funasr/text/cleaner.py index 632267296..1044f504f 100644 --- a/funasr/text/cleaner.py +++ b/funasr/text/cleaner.py @@ -1,7 +1,7 @@ from typing import Collection from jaconv import jaconv -import tacotron_cleaner.cleaners +# import tacotron_cleaner.cleaners try: from vietnamese_cleaner import vietnamese_cleaners @@ -31,7 +31,8 @@ class TextCleaner: def __call__(self, text: str) -> str: for t in self.cleaner_types: if t == "tacotron": - text = tacotron_cleaner.cleaners.custom_english_cleaners(text) + # text = tacotron_cleaner.cleaners.custom_english_cleaners(text) + pass elif t == "jaconv": text = jaconv.normalize(text) elif t == "vietnamese": diff --git a/funasr/runtime/__init__.py b/runtime/__init__.py similarity index 100% rename from funasr/runtime/__init__.py rename to runtime/__init__.py diff --git a/funasr/runtime/android/AndroidClient/app/build.gradle b/runtime/android/AndroidClient/app/build.gradle similarity index 100% rename from funasr/runtime/android/AndroidClient/app/build.gradle rename to runtime/android/AndroidClient/app/build.gradle diff --git a/funasr/runtime/android/AndroidClient/app/proguard-rules.pro b/runtime/android/AndroidClient/app/proguard-rules.pro similarity index 100% rename from funasr/runtime/android/AndroidClient/app/proguard-rules.pro rename to runtime/android/AndroidClient/app/proguard-rules.pro diff --git a/funasr/runtime/android/AndroidClient/app/src/androidTest/java/com/yeyupiaoling/androidclient/ExampleInstrumentedTest.kt b/runtime/android/AndroidClient/app/src/androidTest/java/com/yeyupiaoling/androidclient/ExampleInstrumentedTest.kt similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/androidTest/java/com/yeyupiaoling/androidclient/ExampleInstrumentedTest.kt rename to runtime/android/AndroidClient/app/src/androidTest/java/com/yeyupiaoling/androidclient/ExampleInstrumentedTest.kt diff --git a/funasr/runtime/android/AndroidClient/app/src/main/AndroidManifest.xml b/runtime/android/AndroidClient/app/src/main/AndroidManifest.xml similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/AndroidManifest.xml rename to runtime/android/AndroidClient/app/src/main/AndroidManifest.xml diff --git a/funasr/runtime/android/AndroidClient/app/src/main/java/com/yeyupiaoling/androidclient/AudioView.java b/runtime/android/AndroidClient/app/src/main/java/com/yeyupiaoling/androidclient/AudioView.java similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/java/com/yeyupiaoling/androidclient/AudioView.java rename to runtime/android/AndroidClient/app/src/main/java/com/yeyupiaoling/androidclient/AudioView.java diff --git a/funasr/runtime/android/AndroidClient/app/src/main/java/com/yeyupiaoling/androidclient/MainActivity.java b/runtime/android/AndroidClient/app/src/main/java/com/yeyupiaoling/androidclient/MainActivity.java similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/java/com/yeyupiaoling/androidclient/MainActivity.java rename to runtime/android/AndroidClient/app/src/main/java/com/yeyupiaoling/androidclient/MainActivity.java diff --git a/funasr/runtime/android/AndroidClient/app/src/main/java/com/yeyupiaoling/androidclient/SSLSocketClient.java b/runtime/android/AndroidClient/app/src/main/java/com/yeyupiaoling/androidclient/SSLSocketClient.java similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/java/com/yeyupiaoling/androidclient/SSLSocketClient.java rename to runtime/android/AndroidClient/app/src/main/java/com/yeyupiaoling/androidclient/SSLSocketClient.java diff --git a/funasr/runtime/android/AndroidClient/app/src/main/res/drawable/edittext_border.xml b/runtime/android/AndroidClient/app/src/main/res/drawable/edittext_border.xml similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/res/drawable/edittext_border.xml rename to runtime/android/AndroidClient/app/src/main/res/drawable/edittext_border.xml diff --git a/funasr/runtime/android/AndroidClient/app/src/main/res/drawable/ic_launcher_background.xml b/runtime/android/AndroidClient/app/src/main/res/drawable/ic_launcher_background.xml similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/res/drawable/ic_launcher_background.xml rename to runtime/android/AndroidClient/app/src/main/res/drawable/ic_launcher_background.xml diff --git a/funasr/runtime/android/AndroidClient/app/src/main/res/drawable/ic_launcher_foreground.xml b/runtime/android/AndroidClient/app/src/main/res/drawable/ic_launcher_foreground.xml similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/res/drawable/ic_launcher_foreground.xml rename to runtime/android/AndroidClient/app/src/main/res/drawable/ic_launcher_foreground.xml diff --git a/funasr/runtime/android/AndroidClient/app/src/main/res/drawable/logo.png b/runtime/android/AndroidClient/app/src/main/res/drawable/logo.png similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/res/drawable/logo.png rename to runtime/android/AndroidClient/app/src/main/res/drawable/logo.png diff --git a/funasr/runtime/android/AndroidClient/app/src/main/res/layout/activity_main.xml b/runtime/android/AndroidClient/app/src/main/res/layout/activity_main.xml similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/res/layout/activity_main.xml rename to runtime/android/AndroidClient/app/src/main/res/layout/activity_main.xml diff --git a/funasr/runtime/android/AndroidClient/app/src/main/res/layout/dialog_input_hotwords.xml b/runtime/android/AndroidClient/app/src/main/res/layout/dialog_input_hotwords.xml similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/res/layout/dialog_input_hotwords.xml rename to runtime/android/AndroidClient/app/src/main/res/layout/dialog_input_hotwords.xml diff --git a/funasr/runtime/android/AndroidClient/app/src/main/res/layout/dialog_input_uri.xml b/runtime/android/AndroidClient/app/src/main/res/layout/dialog_input_uri.xml similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/res/layout/dialog_input_uri.xml rename to runtime/android/AndroidClient/app/src/main/res/layout/dialog_input_uri.xml diff --git a/funasr/runtime/android/AndroidClient/app/src/main/res/menu/menu.xml b/runtime/android/AndroidClient/app/src/main/res/menu/menu.xml similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/res/menu/menu.xml rename to runtime/android/AndroidClient/app/src/main/res/menu/menu.xml diff --git a/funasr/runtime/android/AndroidClient/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml b/runtime/android/AndroidClient/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml rename to runtime/android/AndroidClient/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml diff --git a/funasr/runtime/android/AndroidClient/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml b/runtime/android/AndroidClient/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml rename to runtime/android/AndroidClient/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml diff --git a/funasr/runtime/android/AndroidClient/app/src/main/res/mipmap-hdpi/ic_launcher.webp b/runtime/android/AndroidClient/app/src/main/res/mipmap-hdpi/ic_launcher.webp similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/res/mipmap-hdpi/ic_launcher.webp rename to runtime/android/AndroidClient/app/src/main/res/mipmap-hdpi/ic_launcher.webp diff --git a/funasr/runtime/android/AndroidClient/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp b/runtime/android/AndroidClient/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp rename to runtime/android/AndroidClient/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp diff --git a/funasr/runtime/android/AndroidClient/app/src/main/res/mipmap-mdpi/ic_launcher.webp b/runtime/android/AndroidClient/app/src/main/res/mipmap-mdpi/ic_launcher.webp similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/res/mipmap-mdpi/ic_launcher.webp rename to runtime/android/AndroidClient/app/src/main/res/mipmap-mdpi/ic_launcher.webp diff --git a/funasr/runtime/android/AndroidClient/app/src/main/res/mipmap-mdpi/ic_launcher_round.webp b/runtime/android/AndroidClient/app/src/main/res/mipmap-mdpi/ic_launcher_round.webp similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/res/mipmap-mdpi/ic_launcher_round.webp rename to runtime/android/AndroidClient/app/src/main/res/mipmap-mdpi/ic_launcher_round.webp diff --git a/funasr/runtime/android/AndroidClient/app/src/main/res/mipmap-xhdpi/ic_launcher.webp b/runtime/android/AndroidClient/app/src/main/res/mipmap-xhdpi/ic_launcher.webp similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/res/mipmap-xhdpi/ic_launcher.webp rename to runtime/android/AndroidClient/app/src/main/res/mipmap-xhdpi/ic_launcher.webp diff --git a/funasr/runtime/android/AndroidClient/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp b/runtime/android/AndroidClient/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp rename to runtime/android/AndroidClient/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp diff --git a/funasr/runtime/android/AndroidClient/app/src/main/res/mipmap-xxhdpi/ic_launcher.webp b/runtime/android/AndroidClient/app/src/main/res/mipmap-xxhdpi/ic_launcher.webp similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/res/mipmap-xxhdpi/ic_launcher.webp rename to runtime/android/AndroidClient/app/src/main/res/mipmap-xxhdpi/ic_launcher.webp diff --git a/funasr/runtime/android/AndroidClient/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp b/runtime/android/AndroidClient/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp rename to runtime/android/AndroidClient/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp diff --git a/funasr/runtime/android/AndroidClient/app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp b/runtime/android/AndroidClient/app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp rename to runtime/android/AndroidClient/app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp diff --git a/funasr/runtime/android/AndroidClient/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp b/runtime/android/AndroidClient/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp rename to runtime/android/AndroidClient/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp diff --git a/funasr/runtime/android/AndroidClient/app/src/main/res/values/colors.xml b/runtime/android/AndroidClient/app/src/main/res/values/colors.xml similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/res/values/colors.xml rename to runtime/android/AndroidClient/app/src/main/res/values/colors.xml diff --git a/funasr/runtime/android/AndroidClient/app/src/main/res/values/strings.xml b/runtime/android/AndroidClient/app/src/main/res/values/strings.xml similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/res/values/strings.xml rename to runtime/android/AndroidClient/app/src/main/res/values/strings.xml diff --git a/funasr/runtime/android/AndroidClient/app/src/main/res/values/themes.xml b/runtime/android/AndroidClient/app/src/main/res/values/themes.xml similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/res/values/themes.xml rename to runtime/android/AndroidClient/app/src/main/res/values/themes.xml diff --git a/funasr/runtime/android/AndroidClient/app/src/main/res/xml/backup_rules.xml b/runtime/android/AndroidClient/app/src/main/res/xml/backup_rules.xml similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/res/xml/backup_rules.xml rename to runtime/android/AndroidClient/app/src/main/res/xml/backup_rules.xml diff --git a/funasr/runtime/android/AndroidClient/app/src/main/res/xml/data_extraction_rules.xml b/runtime/android/AndroidClient/app/src/main/res/xml/data_extraction_rules.xml similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/main/res/xml/data_extraction_rules.xml rename to runtime/android/AndroidClient/app/src/main/res/xml/data_extraction_rules.xml diff --git a/funasr/runtime/android/AndroidClient/app/src/test/java/com/yeyupiaoling/androidclient/ExampleUnitTest.kt b/runtime/android/AndroidClient/app/src/test/java/com/yeyupiaoling/androidclient/ExampleUnitTest.kt similarity index 100% rename from funasr/runtime/android/AndroidClient/app/src/test/java/com/yeyupiaoling/androidclient/ExampleUnitTest.kt rename to runtime/android/AndroidClient/app/src/test/java/com/yeyupiaoling/androidclient/ExampleUnitTest.kt diff --git a/funasr/runtime/android/AndroidClient/build.gradle b/runtime/android/AndroidClient/build.gradle similarity index 100% rename from funasr/runtime/android/AndroidClient/build.gradle rename to runtime/android/AndroidClient/build.gradle diff --git a/funasr/runtime/android/AndroidClient/gradle.properties b/runtime/android/AndroidClient/gradle.properties similarity index 100% rename from funasr/runtime/android/AndroidClient/gradle.properties rename to runtime/android/AndroidClient/gradle.properties diff --git a/funasr/runtime/android/AndroidClient/gradle/wrapper/gradle-wrapper.jar b/runtime/android/AndroidClient/gradle/wrapper/gradle-wrapper.jar similarity index 100% rename from funasr/runtime/android/AndroidClient/gradle/wrapper/gradle-wrapper.jar rename to runtime/android/AndroidClient/gradle/wrapper/gradle-wrapper.jar diff --git a/funasr/runtime/android/AndroidClient/gradle/wrapper/gradle-wrapper.properties b/runtime/android/AndroidClient/gradle/wrapper/gradle-wrapper.properties similarity index 100% rename from funasr/runtime/android/AndroidClient/gradle/wrapper/gradle-wrapper.properties rename to runtime/android/AndroidClient/gradle/wrapper/gradle-wrapper.properties diff --git a/funasr/runtime/android/AndroidClient/gradlew b/runtime/android/AndroidClient/gradlew similarity index 100% rename from funasr/runtime/android/AndroidClient/gradlew rename to runtime/android/AndroidClient/gradlew diff --git a/funasr/runtime/android/AndroidClient/gradlew.bat b/runtime/android/AndroidClient/gradlew.bat similarity index 100% rename from funasr/runtime/android/AndroidClient/gradlew.bat rename to runtime/android/AndroidClient/gradlew.bat diff --git a/funasr/runtime/android/AndroidClient/settings.gradle b/runtime/android/AndroidClient/settings.gradle similarity index 100% rename from funasr/runtime/android/AndroidClient/settings.gradle rename to runtime/android/AndroidClient/settings.gradle diff --git a/funasr/runtime/android/images/QRcode.png b/runtime/android/images/QRcode.png similarity index 100% rename from funasr/runtime/android/images/QRcode.png rename to runtime/android/images/QRcode.png diff --git a/funasr/runtime/android/images/image1.png b/runtime/android/images/image1.png similarity index 100% rename from funasr/runtime/android/images/image1.png rename to runtime/android/images/image1.png diff --git a/funasr/runtime/android/images/image2.png b/runtime/android/images/image2.png similarity index 100% rename from funasr/runtime/android/images/image2.png rename to runtime/android/images/image2.png diff --git a/funasr/runtime/android/images/image3.png b/runtime/android/images/image3.png similarity index 100% rename from funasr/runtime/android/images/image3.png rename to runtime/android/images/image3.png diff --git a/funasr/runtime/android/readme.md b/runtime/android/readme.md similarity index 100% rename from funasr/runtime/android/readme.md rename to runtime/android/readme.md diff --git a/funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp.Examples/AliFsmnVadSharp.Examples.csproj b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp.Examples/AliFsmnVadSharp.Examples.csproj similarity index 100% rename from funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp.Examples/AliFsmnVadSharp.Examples.csproj rename to runtime/csharp/AliFsmnVad/AliFsmnVadSharp.Examples/AliFsmnVadSharp.Examples.csproj diff --git a/funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp.Examples/Program.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp.Examples/Program.cs similarity index 100% rename from funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp.Examples/Program.cs rename to runtime/csharp/AliFsmnVad/AliFsmnVadSharp.Examples/Program.cs diff --git a/funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp.sln b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp.sln similarity index 100% rename from funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp.sln rename to runtime/csharp/AliFsmnVad/AliFsmnVadSharp.sln diff --git a/funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVad.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVad.cs similarity index 100% rename from funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVad.cs rename to runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVad.cs diff --git a/funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVadSharp.csproj b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVadSharp.csproj similarity index 100% rename from funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVadSharp.csproj rename to runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVadSharp.csproj diff --git a/funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/DLL/KaldiNativeFbank.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/DLL/KaldiNativeFbank.cs similarity index 100% rename from funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/DLL/KaldiNativeFbank.cs rename to runtime/csharp/AliFsmnVad/AliFsmnVadSharp/DLL/KaldiNativeFbank.cs diff --git a/funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/DLL/KnfOnlineFbank.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/DLL/KnfOnlineFbank.cs similarity index 100% rename from funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/DLL/KnfOnlineFbank.cs rename to runtime/csharp/AliFsmnVad/AliFsmnVadSharp/DLL/KnfOnlineFbank.cs diff --git a/funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/E2EVadModel.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/E2EVadModel.cs similarity index 100% rename from funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/E2EVadModel.cs rename to runtime/csharp/AliFsmnVad/AliFsmnVadSharp/E2EVadModel.cs diff --git a/funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Lib/kaldi-native-fbank-dll.dll b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Lib/kaldi-native-fbank-dll.dll similarity index 100% rename from funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Lib/kaldi-native-fbank-dll.dll rename to runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Lib/kaldi-native-fbank-dll.dll diff --git a/funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/CmvnEntity.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/CmvnEntity.cs similarity index 100% rename from funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/CmvnEntity.cs rename to runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/CmvnEntity.cs diff --git a/funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/E2EVadFrameProbEntity.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/E2EVadFrameProbEntity.cs similarity index 100% rename from funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/E2EVadFrameProbEntity.cs rename to runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/E2EVadFrameProbEntity.cs diff --git a/funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/E2EVadSpeechBufWithDoaEntity.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/E2EVadSpeechBufWithDoaEntity.cs similarity index 100% rename from funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/E2EVadSpeechBufWithDoaEntity.cs rename to runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/E2EVadSpeechBufWithDoaEntity.cs diff --git a/funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/EncoderConfEntity.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/EncoderConfEntity.cs similarity index 100% rename from funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/EncoderConfEntity.cs rename to runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/EncoderConfEntity.cs diff --git a/funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/FrontendConfEntity.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/FrontendConfEntity.cs similarity index 100% rename from funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/FrontendConfEntity.cs rename to runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/FrontendConfEntity.cs diff --git a/funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/SegmentEntity.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/SegmentEntity.cs similarity index 100% rename from funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/SegmentEntity.cs rename to runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/SegmentEntity.cs diff --git a/funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/VadInputEntity.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/VadInputEntity.cs similarity index 100% rename from funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/VadInputEntity.cs rename to runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/VadInputEntity.cs diff --git a/funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/VadOutputEntity.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/VadOutputEntity.cs similarity index 100% rename from funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/VadOutputEntity.cs rename to runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/VadOutputEntity.cs diff --git a/funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/VadPostConfEntity.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/VadPostConfEntity.cs similarity index 100% rename from funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/VadPostConfEntity.cs rename to runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/VadPostConfEntity.cs diff --git a/funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/VadYamlEntity.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/VadYamlEntity.cs similarity index 100% rename from funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/VadYamlEntity.cs rename to runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Model/VadYamlEntity.cs diff --git a/funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Struct/FbankData.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Struct/FbankData.cs similarity index 100% rename from funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Struct/FbankData.cs rename to runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Struct/FbankData.cs diff --git a/funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Utils/YamlHelper.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Utils/YamlHelper.cs similarity index 100% rename from funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Utils/YamlHelper.cs rename to runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Utils/YamlHelper.cs diff --git a/funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/WavFrontend.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/WavFrontend.cs similarity index 100% rename from funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/WavFrontend.cs rename to runtime/csharp/AliFsmnVad/AliFsmnVadSharp/WavFrontend.cs diff --git a/funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/WindowDetector.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/WindowDetector.cs similarity index 100% rename from funasr/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/WindowDetector.cs rename to runtime/csharp/AliFsmnVad/AliFsmnVadSharp/WindowDetector.cs diff --git a/funasr/runtime/csharp/AliFsmnVad/README.md b/runtime/csharp/AliFsmnVad/README.md similarity index 100% rename from funasr/runtime/csharp/AliFsmnVad/README.md rename to runtime/csharp/AliFsmnVad/README.md diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/App.xaml b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/App.xaml similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/App.xaml rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/App.xaml diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/App.xaml.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/App.xaml.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/App.xaml.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/App.xaml.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/AppShell.xaml b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/AppShell.xaml similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/AppShell.xaml rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/AppShell.xaml diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/AppShell.xaml.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/AppShell.xaml.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/AppShell.xaml.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/AppShell.xaml.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MainPage.xaml b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MainPage.xaml similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MainPage.xaml rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MainPage.xaml diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MainPage.xaml.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MainPage.xaml.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MainPage.xaml.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MainPage.xaml.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MauiApp1.csproj b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MauiApp1.csproj similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MauiApp1.csproj rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MauiApp1.csproj diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MauiApp1.csproj.user b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MauiApp1.csproj.user similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MauiApp1.csproj.user rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MauiApp1.csproj.user diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MauiProgram.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MauiProgram.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MauiProgram.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MauiProgram.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MySplashPage.xaml b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MySplashPage.xaml similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MySplashPage.xaml rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MySplashPage.xaml diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MySplashPage.xaml.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MySplashPage.xaml.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MySplashPage.xaml.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/MySplashPage.xaml.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Android/AndroidManifest.xml b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Android/AndroidManifest.xml similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Android/AndroidManifest.xml rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Android/AndroidManifest.xml diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Android/MainActivity.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Android/MainActivity.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Android/MainActivity.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Android/MainActivity.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Android/MainApplication.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Android/MainApplication.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Android/MainApplication.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Android/MainApplication.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Android/Resources/values/colors.xml b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Android/Resources/values/colors.xml similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Android/Resources/values/colors.xml rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Android/Resources/values/colors.xml diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/MacCatalyst/AppDelegate.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/MacCatalyst/AppDelegate.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/MacCatalyst/AppDelegate.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/MacCatalyst/AppDelegate.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/MacCatalyst/Info.plist b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/MacCatalyst/Info.plist similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/MacCatalyst/Info.plist rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/MacCatalyst/Info.plist diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/MacCatalyst/Program.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/MacCatalyst/Program.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/MacCatalyst/Program.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/MacCatalyst/Program.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Tizen/Main.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Tizen/Main.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Tizen/Main.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Tizen/Main.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Tizen/tizen-manifest.xml b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Tizen/tizen-manifest.xml similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Tizen/tizen-manifest.xml rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Tizen/tizen-manifest.xml diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Windows/App.xaml b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Windows/App.xaml similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Windows/App.xaml rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Windows/App.xaml diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Windows/App.xaml.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Windows/App.xaml.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Windows/App.xaml.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Windows/App.xaml.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Windows/Package.appxmanifest b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Windows/Package.appxmanifest similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Windows/Package.appxmanifest rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Windows/Package.appxmanifest diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Windows/app.manifest b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Windows/app.manifest similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Windows/app.manifest rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/Windows/app.manifest diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/iOS/AppDelegate.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/iOS/AppDelegate.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/iOS/AppDelegate.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/iOS/AppDelegate.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/iOS/Info.plist b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/iOS/Info.plist similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/iOS/Info.plist rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/iOS/Info.plist diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/iOS/Program.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/iOS/Program.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/iOS/Program.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Platforms/iOS/Program.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Properties/launchSettings.json b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Properties/launchSettings.json similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Properties/launchSettings.json rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Properties/launchSettings.json diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/RecognitionForFiles.xaml b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/RecognitionForFiles.xaml similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/RecognitionForFiles.xaml rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/RecognitionForFiles.xaml diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/RecognitionForFiles.xaml.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/RecognitionForFiles.xaml.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/RecognitionForFiles.xaml.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/RecognitionForFiles.xaml.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/AppIcon/appicon.svg b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/AppIcon/appicon.svg similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/AppIcon/appicon.svg rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/AppIcon/appicon.svg diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/AppIcon/appiconfg.svg b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/AppIcon/appiconfg.svg similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/AppIcon/appiconfg.svg rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/AppIcon/appiconfg.svg diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Fonts/OpenSans-Regular.ttf b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Fonts/OpenSans-Regular.ttf similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Fonts/OpenSans-Regular.ttf rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Fonts/OpenSans-Regular.ttf diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Fonts/OpenSans-Semibold.ttf b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Fonts/OpenSans-Semibold.ttf similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Fonts/OpenSans-Semibold.ttf rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Fonts/OpenSans-Semibold.ttf diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Images/dotnet_bot.svg b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Images/dotnet_bot.svg similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Images/dotnet_bot.svg rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Images/dotnet_bot.svg diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Raw/AboutAssets.txt b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Raw/AboutAssets.txt similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Raw/AboutAssets.txt rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Raw/AboutAssets.txt diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Splash/splash.svg b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Splash/splash.svg similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Splash/splash.svg rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Splash/splash.svg diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Styles/Colors.xaml b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Styles/Colors.xaml similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Styles/Colors.xaml rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Styles/Colors.xaml diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Styles/Styles.xaml b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Styles/Styles.xaml similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Styles/Styles.xaml rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Resources/Styles/Styles.xaml diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Utils/DownloadHelper.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Utils/DownloadHelper.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Utils/DownloadHelper.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Utils/DownloadHelper.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Utils/SysConf.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Utils/SysConf.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Utils/SysConf.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples.MauiApp/Utils/SysConf.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples/AliParaformerAsr.Examples.csproj b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples/AliParaformerAsr.Examples.csproj similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples/AliParaformerAsr.Examples.csproj rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples/AliParaformerAsr.Examples.csproj diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples/Program.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples/Program.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples/Program.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.Examples/Program.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.sln b/runtime/csharp/AliParaformerAsr/AliParaformerAsr.sln similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr.sln rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr.sln diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/AliParaformerAsr.csproj b/runtime/csharp/AliParaformerAsr/AliParaformerAsr/AliParaformerAsr.csproj similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/AliParaformerAsr.csproj rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr/AliParaformerAsr.csproj diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/CmvnEntity.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/CmvnEntity.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/CmvnEntity.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/CmvnEntity.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/DecoderConfEntity.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/DecoderConfEntity.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/DecoderConfEntity.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/DecoderConfEntity.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/EncoderConfEntity.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/EncoderConfEntity.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/EncoderConfEntity.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/EncoderConfEntity.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/FrontendConfEntity.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/FrontendConfEntity.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/FrontendConfEntity.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/FrontendConfEntity.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/ModelConfEntity.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/ModelConfEntity.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/ModelConfEntity.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/ModelConfEntity.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/OfflineInputEntity.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/OfflineInputEntity.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/OfflineInputEntity.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/OfflineInputEntity.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/OfflineOutputEntity.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/OfflineOutputEntity.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/OfflineOutputEntity.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/OfflineOutputEntity.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/OfflineYamlEntity.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/OfflineYamlEntity.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/OfflineYamlEntity.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/OfflineYamlEntity.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/PostEncoderConfEntity.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/PostEncoderConfEntity.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/PostEncoderConfEntity.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/PostEncoderConfEntity.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/PreEncoderConfEntity.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/PreEncoderConfEntity.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/PreEncoderConfEntity.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/PreEncoderConfEntity.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/PredictorConfEntity.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/PredictorConfEntity.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/PredictorConfEntity.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr/Model/PredictorConfEntity.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/OfflineRecognizer.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr/OfflineRecognizer.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/OfflineRecognizer.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr/OfflineRecognizer.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Utils/YamlHelper.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Utils/YamlHelper.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/Utils/YamlHelper.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr/Utils/YamlHelper.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/WavFrontend.cs b/runtime/csharp/AliParaformerAsr/AliParaformerAsr/WavFrontend.cs similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/AliParaformerAsr/WavFrontend.cs rename to runtime/csharp/AliParaformerAsr/AliParaformerAsr/WavFrontend.cs diff --git a/funasr/runtime/csharp/AliParaformerAsr/README.md b/runtime/csharp/AliParaformerAsr/README.md similarity index 100% rename from funasr/runtime/csharp/AliParaformerAsr/README.md rename to runtime/csharp/AliParaformerAsr/README.md diff --git a/funasr/runtime/csharp/ws-client/FunASRClient_CShape.sln b/runtime/csharp/ws-client/FunASRClient_CShape.sln similarity index 100% rename from funasr/runtime/csharp/ws-client/FunASRClient_CShape.sln rename to runtime/csharp/ws-client/FunASRClient_CShape.sln diff --git a/funasr/runtime/csharp/ws-client/FunASRClient_CShape.suo b/runtime/csharp/ws-client/FunASRClient_CShape.suo similarity index 100% rename from funasr/runtime/csharp/ws-client/FunASRClient_CShape.suo rename to runtime/csharp/ws-client/FunASRClient_CShape.suo diff --git a/funasr/runtime/csharp/ws-client/FunASRWSClient_Offline/FunASRWSClient_Offline.csproj b/runtime/csharp/ws-client/FunASRWSClient_Offline/FunASRWSClient_Offline.csproj similarity index 100% rename from funasr/runtime/csharp/ws-client/FunASRWSClient_Offline/FunASRWSClient_Offline.csproj rename to runtime/csharp/ws-client/FunASRWSClient_Offline/FunASRWSClient_Offline.csproj diff --git a/funasr/runtime/csharp/ws-client/FunASRWSClient_Offline/Program.cs b/runtime/csharp/ws-client/FunASRWSClient_Offline/Program.cs similarity index 100% rename from funasr/runtime/csharp/ws-client/FunASRWSClient_Offline/Program.cs rename to runtime/csharp/ws-client/FunASRWSClient_Offline/Program.cs diff --git a/funasr/runtime/csharp/ws-client/FunASRWSClient_Offline/README.md b/runtime/csharp/ws-client/FunASRWSClient_Offline/README.md similarity index 100% rename from funasr/runtime/csharp/ws-client/FunASRWSClient_Offline/README.md rename to runtime/csharp/ws-client/FunASRWSClient_Offline/README.md diff --git a/funasr/runtime/csharp/ws-client/FunASRWSClient_Offline/WebScoketClient.cs b/runtime/csharp/ws-client/FunASRWSClient_Offline/WebScoketClient.cs similarity index 100% rename from funasr/runtime/csharp/ws-client/FunASRWSClient_Offline/WebScoketClient.cs rename to runtime/csharp/ws-client/FunASRWSClient_Offline/WebScoketClient.cs diff --git a/funasr/runtime/csharp/ws-client/FunASRWSClient_Online/FunASRWSClient_Online.csproj b/runtime/csharp/ws-client/FunASRWSClient_Online/FunASRWSClient_Online.csproj similarity index 100% rename from funasr/runtime/csharp/ws-client/FunASRWSClient_Online/FunASRWSClient_Online.csproj rename to runtime/csharp/ws-client/FunASRWSClient_Online/FunASRWSClient_Online.csproj diff --git a/funasr/runtime/csharp/ws-client/FunASRWSClient_Online/Program.cs b/runtime/csharp/ws-client/FunASRWSClient_Online/Program.cs similarity index 100% rename from funasr/runtime/csharp/ws-client/FunASRWSClient_Online/Program.cs rename to runtime/csharp/ws-client/FunASRWSClient_Online/Program.cs diff --git a/funasr/runtime/csharp/ws-client/FunASRWSClient_Online/README.md b/runtime/csharp/ws-client/FunASRWSClient_Online/README.md similarity index 100% rename from funasr/runtime/csharp/ws-client/FunASRWSClient_Online/README.md rename to runtime/csharp/ws-client/FunASRWSClient_Online/README.md diff --git a/funasr/runtime/csharp/ws-client/FunASRWSClient_Online/WaveCollect.cs b/runtime/csharp/ws-client/FunASRWSClient_Online/WaveCollect.cs similarity index 100% rename from funasr/runtime/csharp/ws-client/FunASRWSClient_Online/WaveCollect.cs rename to runtime/csharp/ws-client/FunASRWSClient_Online/WaveCollect.cs diff --git a/funasr/runtime/csharp/ws-client/FunASRWSClient_Online/WebScoketClient.cs b/runtime/csharp/ws-client/FunASRWSClient_Online/WebScoketClient.cs similarity index 100% rename from funasr/runtime/csharp/ws-client/FunASRWSClient_Online/WebScoketClient.cs rename to runtime/csharp/ws-client/FunASRWSClient_Online/WebScoketClient.cs diff --git a/funasr/runtime/csharp/ws-client/confg/config.ini b/runtime/csharp/ws-client/confg/config.ini similarity index 100% rename from funasr/runtime/csharp/ws-client/confg/config.ini rename to runtime/csharp/ws-client/confg/config.ini diff --git a/funasr/runtime/csharp/ws-client/confg/hotword.txt b/runtime/csharp/ws-client/confg/hotword.txt similarity index 100% rename from funasr/runtime/csharp/ws-client/confg/hotword.txt rename to runtime/csharp/ws-client/confg/hotword.txt diff --git a/funasr/runtime/csharp/ws-client/confg/tmp.wav b/runtime/csharp/ws-client/confg/tmp.wav similarity index 100% rename from funasr/runtime/csharp/ws-client/confg/tmp.wav rename to runtime/csharp/ws-client/confg/tmp.wav diff --git a/funasr/runtime/deploy_tools/funasr-runtime-deploy-offline-cpu-en.sh b/runtime/deploy_tools/funasr-runtime-deploy-offline-cpu-en.sh similarity index 99% rename from funasr/runtime/deploy_tools/funasr-runtime-deploy-offline-cpu-en.sh rename to runtime/deploy_tools/funasr-runtime-deploy-offline-cpu-en.sh index 47f47ebb6..893ab70a5 100644 --- a/funasr/runtime/deploy_tools/funasr-runtime-deploy-offline-cpu-en.sh +++ b/runtime/deploy_tools/funasr-runtime-deploy-offline-cpu-en.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -scriptVersion="0.0.1" -scriptDate="20231010" +scriptVersion="0.0.2" +scriptDate="20231030" # Set color @@ -27,7 +27,7 @@ SUDO_CMD="sudo" DEFAULT_DOCKER_OFFLINE_CPU_EN_LISTS_OSS="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/docker_lists/docker_offline_cpu_en_lists" -DEFAULT_DOCKER_OFFLINE_CPU_EN_LISTS_GIT="https://raw.githubusercontent.com/alibaba-damo-academy/FunASR/main/funasr/runtime/docs/docker_offline_cpu_en_lists" +DEFAULT_DOCKER_OFFLINE_CPU_EN_LISTS_GIT="https://raw.githubusercontent.com/alibaba-damo-academy/FunASR/main/runtime/docs/docker_offline_cpu_en_lists" DEFAULT_DOCKER_IMAGE_LISTS=$DEFAULT_DOCKER_OFFLINE_CPU_EN_LISTS_OSS DEFAULT_FUNASR_DOCKER_URL="registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr" DEFAULT_FUNASR_RUNTIME_RESOURCES="funasr-runtime-resources" @@ -42,7 +42,7 @@ DEFAULT_FUNASR_WORKSPACE_DIR="/workspace/models" DEFAULT_DOCKER_PORT="10095" DEFAULT_PROGRESS_FILENAME="progress.txt" DEFAULT_SERVER_EXEC_NAME="funasr-wss-server" -DEFAULT_DOCKER_EXEC_DIR="/workspace/FunASR/funasr/runtime/websocket/build/bin" +DEFAULT_DOCKER_EXEC_DIR="/workspace/FunASR/runtime/websocket/build/bin" DEFAULT_DOCKER_EXEC_PATH=${DEFAULT_DOCKER_EXEC_DIR}/${DEFAULT_SERVER_EXEC_NAME} DEFAULT_SAMPLES_NAME="funasr_samples" DEFAULT_SAMPLES_DIR="samples" @@ -683,7 +683,7 @@ paramsFromDefault(){ if [ ! -z "$punc_id" ]; then PARAMS_PUNC_ID=$punc_id fi - docker_exec_path=`sed '/^PARAMS_DOCKER_EXEC_PATH=/!d;s/.*=//' ${DEFAULT_FUNASR_CONFIG_FILE}` + # docker_exec_path=`sed '/^PARAMS_DOCKER_EXEC_PATH=/!d;s/.*=//' ${DEFAULT_FUNASR_CONFIG_FILE}` if [ ! -z "$docker_exec_path" ]; then PARAMS_DOCKER_EXEC_PATH=$docker_exec_path fi @@ -939,7 +939,7 @@ dockerRun(){ run_cmd="${SUDO_CMD} docker run" port_map=" -p ${PARAMS_HOST_PORT}:${PARAMS_DOCKER_PORT}" env_params=" --privileged=true" - dir_map_params=" -v ${DEFAULT_FUNASR_CONFIG_DIR}:/workspace/.config -v ${PARAMS_FUNASR_LOCAL_MODELS_DIR}:${PARAMS_DOWNLOAD_MODEL_DIR}" + dir_map_params=" -v ${DEFAULT_FUNASR_CONFIG_DIR}:/workspace/.config -v ${PARAMS_FUNASR_LOCAL_MODELS_DIR}:${PARAMS_DOWNLOAD_MODEL_DIR} -v ${PARAMS_FUNASR_LOCAL_WORKSPACE}:/workspace/resources" serverConfigGeneration env_params=" ${env_params} --env DAEMON_SERVER_CONFIG=${daemon_server_config}" @@ -1060,8 +1060,8 @@ serverConfigGeneration(){ crt_path="\"--certfile\":\"\"" key_path="\"--keyfile\":\"\"" else - crt_path="\"--certfile\":\"/workspace/FunASR/funasr/runtime/ssl_key/server.crt\"" - key_path="\"--keyfile\":\"/workspace/FunASR/funasr/runtime/ssl_key/server.key\"" + crt_path="\"--certfile\":\"/workspace/FunASR/runtime/ssl_key/server.crt\"" + key_path="\"--keyfile\":\"/workspace/FunASR/runtime/ssl_key/server.key\"" fi exec_params="\"exec\":\"${PARAMS_DOCKER_EXEC_PATH}\"" diff --git a/funasr/runtime/deploy_tools/funasr-runtime-deploy-offline-cpu-zh.sh b/runtime/deploy_tools/funasr-runtime-deploy-offline-cpu-zh.sh similarity index 93% rename from funasr/runtime/deploy_tools/funasr-runtime-deploy-offline-cpu-zh.sh rename to runtime/deploy_tools/funasr-runtime-deploy-offline-cpu-zh.sh index 5738d6edd..ecb21af3d 100644 --- a/funasr/runtime/deploy_tools/funasr-runtime-deploy-offline-cpu-zh.sh +++ b/runtime/deploy_tools/funasr-runtime-deploy-offline-cpu-zh.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -scriptVersion="0.0.7" -scriptDate="20230818" +scriptVersion="0.0.8" +scriptDate="20231030" # Set color @@ -27,7 +27,7 @@ SUDO_CMD="sudo" DEFAULT_DOCKER_OFFLINE_CPU_ZH_LISTS_OSS="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/docker_lists/docker_offline_cpu_zh_lists" -DEFAULT_DOCKER_OFFLINE_CPU_ZH_LISTS_GIT="https://raw.githubusercontent.com/alibaba-damo-academy/FunASR/main/funasr/runtime/docs/docker_offline_cpu_zh_lists" +DEFAULT_DOCKER_OFFLINE_CPU_ZH_LISTS_GIT="https://raw.githubusercontent.com/alibaba-damo-academy/FunASR/main/runtime/docs/docker_offline_cpu_zh_lists" DEFAULT_DOCKER_IMAGE_LISTS=$DEFAULT_DOCKER_OFFLINE_CPU_ZH_LISTS_OSS DEFAULT_FUNASR_DOCKER_URL="registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr" DEFAULT_FUNASR_RUNTIME_RESOURCES="funasr-runtime-resources" @@ -42,7 +42,7 @@ DEFAULT_FUNASR_WORKSPACE_DIR="/workspace/models" DEFAULT_DOCKER_PORT="10095" DEFAULT_PROGRESS_FILENAME="progress.txt" DEFAULT_SERVER_EXEC_NAME="funasr-wss-server" -DEFAULT_DOCKER_EXEC_DIR="/workspace/FunASR/funasr/runtime/websocket/build/bin" +DEFAULT_DOCKER_EXEC_DIR="/workspace/FunASR/runtime/websocket/build/bin" DEFAULT_DOCKER_EXEC_PATH=${DEFAULT_DOCKER_EXEC_DIR}/${DEFAULT_SERVER_EXEC_NAME} DEFAULT_SAMPLES_NAME="funasr_samples" DEFAULT_SAMPLES_DIR="samples" @@ -56,6 +56,7 @@ DOCKER_IMAGES=() ASR_MODELS=() VAD_MODELS=() PUNC_MODELS=() +LM_MODELS=() # Handles the download progress bar asr_percent_int=0 @@ -420,6 +421,10 @@ readDockerInfoFromUrl(){ if [ ! -z "$docker_info_cur_val" ]; then PUNC_MODELS[${#PUNC_MODELS[*]}]=$docker_info_cur_val fi + elif [ "$docker_info_cur_key" = "DEFAULT_LM_MODEL:" ]; then + if [ ! -z "$docker_info_cur_val" ]; then + LM_MODELS[${#LM_MODELS[*]}]=$docker_info_cur_val + fi fi done echo -e " $DONE" @@ -497,6 +502,7 @@ selectModels(){ selectAsrModels selectVadModels selectPuncModels + selectLMModels echo } @@ -534,6 +540,17 @@ selectPuncModels(){ echo } +selectLMModels(){ + echo -e " ${YELLOW}Please choose the LM model.${PLAIN}" + menuSelection ${LM_MODELS[*]} + result=$? + index=`expr ${result} - 1` + + PARAMS_LM_ID=${LM_MODELS[${index}]} + echo -e " ${UNDERLINE}You have chosen the LM model:${PLAIN} ${GREEN}${PARAMS_LM_ID}${PLAIN}" + echo +} + # Configure FunASR server host port setting. setupHostPort(){ echo -e "${UNDERLINE}${BOLD}[3/6]${PLAIN}" @@ -600,6 +617,14 @@ complementParameters(){ PARAMS_LOCAL_PUNC_DIR=$(dirname "${PARAMS_LOCAL_PUNC_PATH}") fi + # parameters about LM model + if [ ! -z "$PARAMS_LM_ID" ]; then + PARAMS_DOCKER_LM_PATH=${PARAMS_DOWNLOAD_MODEL_DIR}/${PARAMS_LM_ID} + PARAMS_DOCKER_LM_DIR=$(dirname "${PARAMS_DOCKER_LM_PATH}") + PARAMS_LOCAL_LM_PATH=${PARAMS_FUNASR_LOCAL_MODELS_DIR}/${PARAMS_LM_ID} + PARAMS_LOCAL_LM_DIR=$(dirname "${PARAMS_LOCAL_LM_PATH}") + fi + # parameters about thread_num params_decoder_thread_num=`sed '/^PARAMS_DECODER_THREAD_NUM=/!d;s/.*=//' ${DEFAULT_FUNASR_CONFIG_FILE}` if [ -z "$params_decoder_thread_num" ]; then @@ -683,7 +708,19 @@ paramsFromDefault(){ if [ ! -z "$punc_id" ]; then PARAMS_PUNC_ID=$punc_id fi - docker_exec_path=`sed '/^PARAMS_DOCKER_EXEC_PATH=/!d;s/.*=//' ${DEFAULT_FUNASR_CONFIG_FILE}` + local_lm_path=`sed '/^PARAMS_LOCAL_LM_PATH=/!d;s/.*=//' ${DEFAULT_FUNASR_CONFIG_FILE}` + if [ ! -z "$local_lm_path" ]; then + PARAMS_LOCAL_LM_PATH=$local_lm_path + fi + docker_lm_path=`sed '/^PARAMS_DOCKER_LM_PATH=/!d;s/.*=//' ${DEFAULT_FUNASR_CONFIG_FILE}` + if [ ! -z "$docker_lm_path" ]; then + PARAMS_DOCKER_LM_PATH=$docker_lm_path + fi + lm_id=`sed '/^PARAMS_LM_ID=/!d;s/.*=//' ${DEFAULT_FUNASR_CONFIG_FILE}` + if [ ! -z "$lm_id" ]; then + PARAMS_LM_ID=$lm_id + fi + # docker_exec_path=`sed '/^PARAMS_DOCKER_EXEC_PATH=/!d;s/.*=//' ${DEFAULT_FUNASR_CONFIG_FILE}` if [ ! -z "$docker_exec_path" ]; then PARAMS_DOCKER_EXEC_PATH=$docker_exec_path fi @@ -742,6 +779,12 @@ saveParams(){ echo "PARAMS_DOCKER_PUNC_DIR=${PARAMS_DOCKER_PUNC_DIR}" >> $DEFAULT_FUNASR_CONFIG_FILE echo "PARAMS_PUNC_ID=${PARAMS_PUNC_ID}" >> $DEFAULT_FUNASR_CONFIG_FILE + echo "PARAMS_LOCAL_LM_PATH=${PARAMS_LOCAL_LM_PATH}" >> $DEFAULT_FUNASR_CONFIG_FILE + echo "PARAMS_LOCAL_LM_DIR=${PARAMS_LOCAL_LM_DIR}" >> $DEFAULT_FUNASR_CONFIG_FILE + echo "PARAMS_DOCKER_LM_PATH=${PARAMS_DOCKER_LM_PATH}" >> $DEFAULT_FUNASR_CONFIG_FILE + echo "PARAMS_DOCKER_LM_DIR=${PARAMS_DOCKER_LM_DIR}" >> $DEFAULT_FUNASR_CONFIG_FILE + echo "PARAMS_LM_ID=${PARAMS_LM_ID}" >> $DEFAULT_FUNASR_CONFIG_FILE + echo "PARAMS_LOCAL_VAD_PATH=${PARAMS_LOCAL_VAD_PATH}" >> $DEFAULT_FUNASR_CONFIG_FILE echo "PARAMS_LOCAL_VAD_DIR=${PARAMS_LOCAL_VAD_DIR}" >> $DEFAULT_FUNASR_CONFIG_FILE echo "PARAMS_DOCKER_VAD_PATH=${PARAMS_DOCKER_VAD_PATH}" >> $DEFAULT_FUNASR_CONFIG_FILE @@ -801,6 +844,15 @@ showAllParams(){ echo -e " The path to the local PUNC model directory for the load : ${GREEN}${PARAMS_LOCAL_PUNC_PATH}${PLAIN}" fi echo -e " The PUNC model directory corresponds to the directory in Docker: ${GREEN}${PARAMS_DOCKER_PUNC_PATH}${PLAIN}" + + if [ ! -z "$PARAMS_LM_ID" ]; then + echo -e " The LM model_id used : ${GREEN}${PARAMS_LM_ID}${PLAIN}" + fi + if [ ! -z "$PARAMS_LOCAL_LM_PATH" ]; then + echo -e " The path to the local LM model directory for the load : ${GREEN}${PARAMS_LOCAL_LM_PATH}${PLAIN}" + fi + echo -e " The LM model directory corresponds to the directory in Docker: ${GREEN}${PARAMS_DOCKER_LM_PATH}${PLAIN}" + echo echo -e " The path in the docker of the FunASR service executor : ${GREEN}${PARAMS_DOCKER_EXEC_PATH}${PLAIN}" @@ -939,7 +991,7 @@ dockerRun(){ run_cmd="${SUDO_CMD} docker run" port_map=" -p ${PARAMS_HOST_PORT}:${PARAMS_DOCKER_PORT}" env_params=" --privileged=true" - dir_map_params=" -v ${DEFAULT_FUNASR_CONFIG_DIR}:/workspace/.config -v ${PARAMS_FUNASR_LOCAL_MODELS_DIR}:${PARAMS_DOWNLOAD_MODEL_DIR}" + dir_map_params=" -v ${DEFAULT_FUNASR_CONFIG_DIR}:/workspace/.config -v ${PARAMS_FUNASR_LOCAL_MODELS_DIR}:${PARAMS_DOWNLOAD_MODEL_DIR} -v ${PARAMS_FUNASR_LOCAL_WORKSPACE}:/workspace/resources" serverConfigGeneration env_params=" ${env_params} --env DAEMON_SERVER_CONFIG=${daemon_server_config}" @@ -1046,8 +1098,16 @@ serverConfigGeneration(){ fi punc_params="\"--punc-dir\":\"${PARAMS_DOCKER_PUNC_PATH}\"" fi + if [ ! -z "$PARAMS_LM_ID" ]; then + lm_params="\"--lm-dir\":\"${PARAMS_LM_ID}\"" + else + if [ ! -z "$PARAMS_LOCAL_LM_PATH" ]; then + dir_map_params="${dir_map_params} -v ${PARAMS_LOCAL_VAD_PATH}:${PARAMS_DOCKER_VAD_PATH}" + fi + lm_params="\"--lm-dir\":\"${PARAMS_DOCKER_LM_PATH}\"" + fi download_params="\"--download-model-dir\":\"${PARAMS_DOWNLOAD_MODEL_DIR}\"" - model_params="${asr_params},${vad_params},${punc_params},${download_params}" + model_params="${asr_params},${vad_params},${punc_params},${lm_params},${download_params}" # params about thread_num decoder_params="\"--decoder-thread-num\":\"${PARAMS_DECODER_THREAD_NUM}\"" @@ -1060,8 +1120,8 @@ serverConfigGeneration(){ crt_path="\"--certfile\":\"\"" key_path="\"--keyfile\":\"\"" else - crt_path="\"--certfile\":\"/workspace/FunASR/funasr/runtime/ssl_key/server.crt\"" - key_path="\"--keyfile\":\"/workspace/FunASR/funasr/runtime/ssl_key/server.key\"" + crt_path="\"--certfile\":\"/workspace/FunASR/runtime/ssl_key/server.crt\"" + key_path="\"--keyfile\":\"/workspace/FunASR/runtime/ssl_key/server.key\"" fi exec_params="\"exec\":\"${PARAMS_DOCKER_EXEC_PATH}\"" @@ -1285,6 +1345,21 @@ modelChange(){ PARAMS_DOCKER_PUNC_PATH=${PARAMS_DOCKER_PUNC_DIR}/${model_name} fi fi + result=$(echo ${model_type} | grep "\-\-lm_model") + if [ "$result" != "" ]; then + if [ $local_flag -eq 0 ]; then + PARAMS_LM_ID=$model_id + PARAMS_DOCKER_LM_PATH=${PARAMS_DOWNLOAD_MODEL_DIR}/${PARAMS_LM_ID} + PARAMS_DOCKER_LM_DIR=$(dirname "${PARAMS_DOCKER_LM_PATH}") + PARAMS_LOCAL_LM_PATH=${PARAMS_FUNASR_LOCAL_MODELS_DIR}/${PARAMS_LM_ID} + PARAMS_LOCAL_LM_DIR=$(dirname "${PARAMS_LOCAL_LM_PATH}") + else + model_name=$(basename "${PARAMS_LOCAL_LM_PATH}") + PARAMS_LOCAL_LM_DIR=$(dirname "${PARAMS_LOCAL_LM_PATH}") + PARAMS_DOCKER_LM_DIR=$PARAMS_DOWNLOAD_MODEL_DIR + PARAMS_DOCKER_LM_PATH=${PARAMS_DOCKER_LM_DIR}/${model_name} + fi + fi } threadNumChange() { @@ -1437,7 +1512,7 @@ displayHelp(){ echo -e " ${BOLD}-r, restart, --restart${PLAIN} Restart FunASR docker." echo -e " ${BOLD}-u, update , --update${PLAIN} Update parameters that has already been set." echo -e " update [--workspace] " - echo -e " update [--asr_model | --vad_model | --punc_model] " + echo -e " update [--asr_model | --vad_model | --punc_model | --lm_model] " echo -e " update [--host_port | --docker_port] " echo -e " update [--decode_thread_num | io_thread_num] " echo -e " update [--ssl] <0: close SSL; 1: open SSL, default:1>" @@ -1519,6 +1594,17 @@ PARAMS_DOCKER_PUNC_PATH="" # The punc model ID in ModelScope PARAMS_PUNC_ID="damo/punc_ct-transformer_zh-cn-common-vocab272727-onnx" +# The dir stored lm model in local +PARAMS_LOCAL_LM_DIR="" +# The path of lm model in local +PARAMS_LOCAL_LM_PATH="" +# The dir stored lm model in docker +PARAMS_DOCKER_LM_DIR="" +# The path of lm model in docker +PARAMS_DOCKER_LM_PATH="" +# The lm model ID in ModelScope +PARAMS_LM_ID="damo/speech_ngram_lm_zh-cn-ai-wesp-fst" + # The dir stored vad model in local PARAMS_LOCAL_VAD_DIR="" # The path of vad model in local @@ -1668,7 +1754,7 @@ case "$1" in if [ $# -eq 3 ]; then type=$2 val=$3 - if [ "$type" = "--asr_model" ] || [ "$type" = "--vad_model" ] || [ "$type" = "--punc_model" ]; then + if [ "$type" = "--asr_model" ] || [ "$type" = "--vad_model" ] || [ "$type" = "--punc_model" ] || [ "$type" = "--lm_model" ]; then modelChange $type $val elif [ "$type" = "--decode_thread_num" ] || [ "$type" = "--io_thread_num" ]; then threadNumChange $type $val diff --git a/funasr/runtime/deploy_tools/funasr-runtime-deploy-online-cpu-zh.sh b/runtime/deploy_tools/funasr-runtime-deploy-online-cpu-zh.sh similarity index 99% rename from funasr/runtime/deploy_tools/funasr-runtime-deploy-online-cpu-zh.sh rename to runtime/deploy_tools/funasr-runtime-deploy-online-cpu-zh.sh index 423b2a095..a1e999979 100644 --- a/funasr/runtime/deploy_tools/funasr-runtime-deploy-online-cpu-zh.sh +++ b/runtime/deploy_tools/funasr-runtime-deploy-online-cpu-zh.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -scriptVersion="0.0.2" -scriptDate="20230914" +scriptVersion="0.0.3" +scriptDate="20231030" # Set color @@ -27,7 +27,7 @@ SUDO_CMD="sudo" DEFAULT_DOCKER_ONLINE_CPU_ZH_LISTS_OSS="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/docker_lists/docker_online_cpu_zh_lists" -DEFAULT_DOCKER_ONLINE_CPU_ZH_LISTS_GIT="https://raw.githubusercontent.com/alibaba-damo-academy/FunASR/main/funasr/runtime/docs/docker_online_cpu_zh_lists" +DEFAULT_DOCKER_ONLINE_CPU_ZH_LISTS_GIT="https://raw.githubusercontent.com/alibaba-damo-academy/FunASR/main/runtime/docs/docker_online_cpu_zh_lists" DEFAULT_DOCKER_IMAGE_LISTS=$DEFAULT_DOCKER_ONLINE_CPU_ZH_LISTS_OSS DEFAULT_FUNASR_DOCKER_URL="registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr" DEFAULT_FUNASR_RUNTIME_RESOURCES="funasr-runtime-resources" @@ -42,7 +42,7 @@ DEFAULT_FUNASR_WORKSPACE_DIR="/workspace/models" DEFAULT_DOCKER_PORT="10095" DEFAULT_PROGRESS_FILENAME="progress.txt" DEFAULT_SERVER_EXEC_NAME="funasr-wss-server-2pass" -DEFAULT_DOCKER_EXEC_DIR="/workspace/FunASR/funasr/runtime/websocket/build/bin" +DEFAULT_DOCKER_EXEC_DIR="/workspace/FunASR/runtime/websocket/build/bin" DEFAULT_DOCKER_EXEC_PATH=${DEFAULT_DOCKER_EXEC_DIR}/${DEFAULT_SERVER_EXEC_NAME} DEFAULT_SAMPLES_NAME="funasr_samples" DEFAULT_SAMPLES_DIR="samples" @@ -683,7 +683,7 @@ paramsFromDefault(){ if [ ! -z "$punc_id" ]; then PARAMS_PUNC_ID=$punc_id fi - docker_exec_path=`sed '/^PARAMS_DOCKER_EXEC_PATH=/!d;s/.*=//' ${DEFAULT_FUNASR_CONFIG_FILE}` + # docker_exec_path=`sed '/^PARAMS_DOCKER_EXEC_PATH=/!d;s/.*=//' ${DEFAULT_FUNASR_CONFIG_FILE}` if [ ! -z "$docker_exec_path" ]; then PARAMS_DOCKER_EXEC_PATH=$docker_exec_path fi @@ -939,7 +939,7 @@ dockerRun(){ run_cmd="${SUDO_CMD} docker run" port_map=" -p ${PARAMS_HOST_PORT}:${PARAMS_DOCKER_PORT}" env_params=" --privileged=true" - dir_map_params=" -v ${DEFAULT_FUNASR_CONFIG_DIR}:/workspace/.config -v ${PARAMS_FUNASR_LOCAL_MODELS_DIR}:${PARAMS_DOWNLOAD_MODEL_DIR}" + dir_map_params=" -v ${DEFAULT_FUNASR_CONFIG_DIR}:/workspace/.config -v ${PARAMS_FUNASR_LOCAL_MODELS_DIR}:${PARAMS_DOWNLOAD_MODEL_DIR} -v ${PARAMS_FUNASR_LOCAL_WORKSPACE}:/workspace/resources" serverConfigGeneration env_params=" ${env_params} --env DAEMON_SERVER_CONFIG=${daemon_server_config}" @@ -1060,8 +1060,8 @@ serverConfigGeneration(){ crt_path="\"--certfile\":\"\"" key_path="\"--keyfile\":\"\"" else - crt_path="\"--certfile\":\"/workspace/FunASR/funasr/runtime/ssl_key/server.crt\"" - key_path="\"--keyfile\":\"/workspace/FunASR/funasr/runtime/ssl_key/server.key\"" + crt_path="\"--certfile\":\"/workspace/FunASR/runtime/ssl_key/server.crt\"" + key_path="\"--keyfile\":\"/workspace/FunASR/runtime/ssl_key/server.key\"" fi exec_params="\"exec\":\"${PARAMS_DOCKER_EXEC_PATH}\"" diff --git a/runtime/deploy_tools/install_docker.sh b/runtime/deploy_tools/install_docker.sh new file mode 100644 index 000000000..964765a87 --- /dev/null +++ b/runtime/deploy_tools/install_docker.sh @@ -0,0 +1,82 @@ + +# Set color +RED="\033[31;1m" +GREEN="\033[32;1m" +YELLOW="\033[33;1m" +BLUE="\033[34;1m" +CYAN="\033[36;1m" +PLAIN="\033[0m" + +# Info messages +DONE="${GREEN}[DONE]${PLAIN}" +ERROR="${RED}[ERROR]${PLAIN}" +WARNING="${YELLOW}[WARNING]${PLAIN}" + +# Font Format +BOLD="\033[1m" +UNDERLINE="\033[4m" + +# Current folder +CUR_DIR=`pwd` +SUDO_CMD="sudo" + +# OS +OSID=$(grep ^ID= /etc/os-release | cut -d= -f2) +OSVER=$(lsb_release -cs) +OSNUM=$(grep -oE "[0-9.]+" /etc/issue) +DOCKERINFO=$(${SUDO_CMD} docker info | wc -l) +DOCKERINFOLEN=`expr ${DOCKERINFO} + 0` + + +if [ $DOCKERINFOLEN -gt 30 ]; then + echo -e " ${YELLOW}Docker has installed.${PLAIN}" +else + lowercase_osid=$(echo ${OSID} | tr '[A-Z]' '[a-z]') + echo -e " ${YELLOW}Start install docker for ${lowercase_osid} ${PLAIN}" + DOCKER_INSTALL_CMD="curl -fsSL https://get.docker.com | bash -s docker --mirror Aliyun" + DOCKER_INSTALL_RUN_CMD="" + + case "$lowercase_osid" in + ubuntu) + DOCKER_INSTALL_CMD="curl -fsSL https://test.docker.com -o test-docker.sh" + DOCKER_INSTALL_RUN_CMD="${SUDO_CMD} sh test-docker.sh" + ;; + centos) + DOCKER_INSTALL_CMD="curl -fsSL https://get.docker.com | bash -s docker --mirror Aliyun" + ;; + debian) + DOCKER_INSTALL_CMD="curl -fsSL https://get.docker.com -o get-docker.sh" + DOCKER_INSTALL_RUN_CMD="${SUDO_CMD} sh get-docker.sh" + ;; + \"alios\") + DOCKER_INSTALL_CMD="curl -fsSL https://get.docker.com -o get-docker.sh" + DOCKER_INSTALL_RUN_CMD="${SUDO_CMD} sh get-docker.sh" + ;; + \"alinux\") + DOCKER_INSTALL_CMD="${SUDO_CMD} yum -y install dnf" + DOCKER_INSTALL_RUN_CMD="${SUDO_CMD} dnf -y install docker" + ;; + *) + echo -e " ${RED}$lowercase_osid is not supported.${PLAIN}" + ;; + esac + + echo -e " Get docker installer: ${GREEN}${DOCKER_INSTALL_CMD}${PLAIN}" + echo -e " Get docker run: ${GREEN}${DOCKER_INSTALL_RUN_CMD}${PLAIN}" + + $DOCKER_INSTALL_CMD + if [ ! -z "$DOCKER_INSTALL_RUN_CMD" ]; then + $DOCKER_INSTALL_RUN_CMD + fi + $SUDO_CMD systemctl start docker + + DOCKERINFO=$(${SUDO_CMD} docker info | wc -l) + DOCKERINFOLEN=`expr ${DOCKERINFO} + 0` + if [ $DOCKERINFOLEN -gt 30 ]; then + echo -e " ${GREEN}Docker install success, start docker server.${PLAIN}" + $SUDO_CMD systemctl start docker + else + echo -e " ${RED}Docker install failed!${PLAIN}" + exit 1 + fi +fi \ No newline at end of file diff --git a/funasr/runtime/docs/SDK_advanced_guide_offline.md b/runtime/docs/SDK_advanced_guide_offline.md similarity index 90% rename from funasr/runtime/docs/SDK_advanced_guide_offline.md rename to runtime/docs/SDK_advanced_guide_offline.md index 43a69cd83..091036543 100644 --- a/funasr/runtime/docs/SDK_advanced_guide_offline.md +++ b/runtime/docs/SDK_advanced_guide_offline.md @@ -36,9 +36,9 @@ sudo systemctl start docker Use the following command to pull and launch the Docker image for the FunASR runtime-SDK: ```shell -sudo docker pull registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-cpu-0.2.2 +sudo docker pull registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-cpu-0.3.0 -sudo docker run -p 10095:10095 -it --privileged=true -v /root:/workspace/models registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-cpu-0.2.2 +sudo docker run -p 10095:10095 -it --privileged=true -v /root:/workspace/models registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-cpu-0.3.0 ``` Introduction to command parameters: @@ -58,10 +58,11 @@ nohup bash run_server.sh \ --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx \ --model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-onnx \ --punc-dir damo/punc_ct-transformer_zh-cn-common-vocab272727-onnx \ + --lm-dir damo/speech_ngram_lm_zh-cn-ai-wesp-fst \ --itn-dir thuduj12/fst_itn_zh > log.out 2>&1 & # If you want to close ssl，please add：--certfile 0 -# If you want to deploy the timestamp or hotword model, please set --model-dir to the corresponding model: +# If you want to deploy the timestamp or nn hotword model, please set --model-dir to the corresponding model: # speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-onnx（timestamp） # damo/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404-onnx（hotword） @@ -72,13 +73,14 @@ nohup bash run_server.sh \ The funasr-wss-server supports downloading models from Modelscope. You can set the model download address (--download-model-dir, default is /workspace/models) and the model ID (--model-dir, --vad-dir, --punc-dir). Here is an example: ```shell -cd /workspace/FunASR/funasr/runtime +cd /workspace/FunASR/runtime nohup bash run_server.sh \ --download-model-dir /workspace/models \ --model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-onnx \ --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx \ --punc-dir damo/punc_ct-transformer_zh-cn-common-vocab272727-onnx \ --itn-dir thuduj12/fst_itn_zh \ + --lm-dir damo/speech_ngram_lm_zh-cn-ai-wesp-fst \ --decoder-thread-num 32 \ --io-thread-num 8 \ --port 10095 \ @@ -103,7 +105,7 @@ Introduction to run_server.sh parameters: --io-thread-num: Number of IO threads that the server starts. Default is 1. --certfile : SSL certificate file. Default is ../../../ssl_key/server.crt. If you want to close ssl，set 0 --keyfile : SSL key file. Default is ../../../ssl_key/server.key. ---hotword Hotword file path, one line for each hot word, if the client provides hot words, then combined with the hot words provided by the client. Default is ../../hotwords.txt +--hotword: Hotword file path, one line for each hotword(e.g.:阿里巴巴 20), if the client provides hot words, then combined with the hot words provided by the client. ``` ### Shutting Down the FunASR Service @@ -132,7 +134,8 @@ After executing the above command, the real-time speech transcription service wi [FSMN-VAD](https://www.modelscope.cn/models/damo/speech_fsmn_vad_zh-cn-16k-common-onnx/summary), [Paraformer-lagre](https://www.modelscope.cn/models/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-onnx/summary), [CT-Transformer](https://www.modelscope.cn/models/damo/punc_ct-transformer_zh-cn-common-vocab272727-onnx/summary), -[FST-ITN](https://www.modelscope.cn/models/thuduj12/fst_itn_zh/summary) +[FST-ITN](https://www.modelscope.cn/models/thuduj12/fst_itn_zh/summary), +[Ngram lm](https://www.modelscope.cn/models/damo/speech_ngram_lm_zh-cn-ai-wesp-fst/summary) If you wish to deploy your fine-tuned model (e.g., 10epoch.pb), you need to manually rename the model to model.pb and replace the original model.pb in ModelScope. Then, specify the path as `model_dir`. @@ -156,7 +159,7 @@ Introduction to command parameters: --output_dir: the path to the recognition result output. --ssl: whether to use SSL encryption. The default is to use SSL. --mode: offline mode. ---hotword: If am is hotword model, setting hotword: *.txt(one hotword perline) or hotwords seperate by space (could be: 阿里巴巴 达摩院) +--hotword: Hotword file path, one line for each hotword(e.g.:阿里巴巴 20) --use_itn: whether to use itn, the default value is 1 for enabling and 0 for disabling. ``` @@ -172,7 +175,7 @@ Introduction to command parameters: --port: the port number of the server listener. --wav-path: the audio input. Input can be a path to a wav file or a wav.scp file (a Kaldi-formatted wav list in which each line includes a wav_id followed by a tab and a wav_path). --is-ssl: whether to use SSL encryption. The default is to use SSL. ---hotword: If am is hotword model, setting hotword: *.txt(one hotword perline) or hotwords seperate by space (could be: 阿里巴巴 达摩院) +--hotword: Hotword file path, one line for each hotword(e.g.:阿里巴巴 20) --use-itn: whether to use itn, the default value is 1 for enabling and 0 for disabling. ``` @@ -186,11 +189,11 @@ The code for FunASR-runtime is open source. If the server and client cannot full ### C++ client -https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/runtime/websocket +https://github.com/alibaba-damo-academy/FunASR/tree/main/runtime/websocket ### Python client -https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/runtime/python/websocket +https://github.com/alibaba-damo-academy/FunASR/tree/main/runtime/python/websocket ### C++ server @@ -203,7 +206,7 @@ FUNASR_RESULT result=FsmnVadInfer(vad_hanlde, wav_file.c_str(), NULL, 16000); // Where: vad_hanlde is the return value of FunOfflineInit, wav_file is the path to the audio file, and sampling_rate is the sampling rate (default 16k). ``` -See the usage example for details [docs](https://github.com/alibaba-damo-academy/FunASR/blob/main/funasr/runtime/onnxruntime/bin/funasr-onnx-offline-vad.cpp) +See the usage example for details [docs](https://github.com/alibaba-damo-academy/FunASR/blob/main/runtime/onnxruntime/bin/funasr-onnx-offline-vad.cpp) #### ASR ```text @@ -214,7 +217,7 @@ FUNASR_RESULT result=FunOfflineInfer(asr_hanlde, wav_file.c_str(), RASR_NONE, NU // Where: asr_hanlde is the return value of FunOfflineInit, wav_file is the path to the audio file, and sampling_rate is the sampling rate (default 16k). ``` -See the usage example for details, [docs](https://github.com/alibaba-damo-academy/FunASR/blob/main/funasr/runtime/onnxruntime/bin/funasr-onnx-offline.cpp) +See the usage example for details, [docs](https://github.com/alibaba-damo-academy/FunASR/blob/main/runtime/onnxruntime/bin/funasr-onnx-offline.cpp) #### PUNC ```text @@ -224,4 +227,4 @@ FUNASR_HANDLE punc_hanlde=CTTransformerInit(model_path, thread_num); FUNASR_RESULT result=CTTransformerInfer(punc_hanlde, txt_str.c_str(), RASR_NONE, NULL); // Where: punc_hanlde is the return value of CTTransformerInit, txt_str is the text ``` -See the usage example for details, [docs](https://github.com/alibaba-damo-academy/FunASR/blob/main/funasr/runtime/onnxruntime/bin/funasr-onnx-offline-punc.cpp) +See the usage example for details, [docs](https://github.com/alibaba-damo-academy/FunASR/blob/main/runtime/onnxruntime/bin/funasr-onnx-offline-punc.cpp) diff --git a/funasr/runtime/docs/SDK_advanced_guide_offline_en.md b/runtime/docs/SDK_advanced_guide_offline_en.md similarity index 92% rename from funasr/runtime/docs/SDK_advanced_guide_offline_en.md rename to runtime/docs/SDK_advanced_guide_offline_en.md index b829e67c7..e4650382e 100644 --- a/funasr/runtime/docs/SDK_advanced_guide_offline_en.md +++ b/runtime/docs/SDK_advanced_guide_offline_en.md @@ -36,9 +36,9 @@ sudo systemctl start docker Use the following command to pull and launch the Docker image for the FunASR runtime-SDK: ```shell -sudo docker pull registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-en-cpu-0.1.0 +sudo docker pull registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-en-cpu-0.1.1 -sudo docker run -p 10095:10095 -it --privileged=true -v /root:/workspace/models registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-en-cpu-0.1.0 +sudo docker run -p 10095:10095 -it --privileged=true -v /root:/workspace/models registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-en-cpu-0.1.1 ``` Introduction to command parameters: @@ -69,7 +69,7 @@ nohup bash run_server.sh \ The funasr-wss-server supports downloading models from Modelscope. You can set the model download address (--download-model-dir, default is /workspace/models) and the model ID (--model-dir, --vad-dir, --punc-dir). Here is an example: ```shell -cd /workspace/FunASR/funasr/runtime +cd /workspace/FunASR/runtime nohup bash run_server.sh \ --download-model-dir /workspace/models \ --model-dir damo/speech_paraformer-large_asr_nat-en-16k-common-vocab10020-onnx \ @@ -147,7 +147,7 @@ Introduction to command parameters: --output_dir: the path to the recognition result output. --ssl: whether to use SSL encryption. The default is to use SSL. --mode: offline mode. ---hotword: If am is hotword model, setting hotword: *.txt(one hotword perline) or hotwords seperate by space (could be: 阿里巴巴 达摩院) +--hotword: Hotword file path, one line for each hotword(e.g.:阿里巴巴 20) --use_itn: whether to use itn, the default value is 1 for enabling and 0 for disabling. ``` @@ -163,7 +163,7 @@ Introduction to command parameters: --port: the port number of the server listener. --wav-path: the audio input. Input can be a path to a wav file or a wav.scp file (a Kaldi-formatted wav list in which each line includes a wav_id followed by a tab and a wav_path). --is-ssl: whether to use SSL encryption. The default is to use SSL. ---hotword: If am is hotword model, setting hotword: *.txt(one hotword perline) or hotwords seperate by space (could be: 阿里巴巴 达摩院) +--hotword: Hotword file path, one line for each hotword(e.g.:阿里巴巴 20) --use-itn: whether to use itn, the default value is 1 for enabling and 0 for disabling. ``` @@ -177,11 +177,11 @@ The code for FunASR-runtime is open source. If the server and client cannot full ### C++ client -https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/runtime/websocket +https://github.com/alibaba-damo-academy/FunASR/tree/main/runtime/websocket ### Python client -https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/runtime/python/websocket +https://github.com/alibaba-damo-academy/FunASR/tree/main/runtime/python/websocket ### C++ server @@ -194,7 +194,7 @@ FUNASR_RESULT result=FsmnVadInfer(vad_hanlde, wav_file.c_str(), NULL, 16000); // Where: vad_hanlde is the return value of FunOfflineInit, wav_file is the path to the audio file, and sampling_rate is the sampling rate (default 16k). ``` -See the usage example for details [docs](https://github.com/alibaba-damo-academy/FunASR/blob/main/funasr/runtime/onnxruntime/bin/funasr-onnx-offline-vad.cpp) +See the usage example for details [docs](https://github.com/alibaba-damo-academy/FunASR/blob/main/runtime/onnxruntime/bin/funasr-onnx-offline-vad.cpp) #### ASR ```text @@ -205,7 +205,7 @@ FUNASR_RESULT result=FunOfflineInfer(asr_hanlde, wav_file.c_str(), RASR_NONE, NU // Where: asr_hanlde is the return value of FunOfflineInit, wav_file is the path to the audio file, and sampling_rate is the sampling rate (default 16k). ``` -See the usage example for details, [docs](https://github.com/alibaba-damo-academy/FunASR/blob/main/funasr/runtime/onnxruntime/bin/funasr-onnx-offline.cpp) +See the usage example for details, [docs](https://github.com/alibaba-damo-academy/FunASR/blob/main/runtime/onnxruntime/bin/funasr-onnx-offline.cpp) #### PUNC ```text @@ -215,4 +215,4 @@ FUNASR_HANDLE punc_hanlde=CTTransformerInit(model_path, thread_num); FUNASR_RESULT result=CTTransformerInfer(punc_hanlde, txt_str.c_str(), RASR_NONE, NULL); // Where: punc_hanlde is the return value of CTTransformerInit, txt_str is the text ``` -See the usage example for details, [docs](https://github.com/alibaba-damo-academy/FunASR/blob/main/funasr/runtime/onnxruntime/bin/funasr-onnx-offline-punc.cpp) +See the usage example for details, [docs](https://github.com/alibaba-damo-academy/FunASR/blob/main/runtime/onnxruntime/bin/funasr-onnx-offline-punc.cpp) diff --git a/funasr/runtime/docs/SDK_advanced_guide_offline_en_zh.md b/runtime/docs/SDK_advanced_guide_offline_en_zh.md similarity index 91% rename from funasr/runtime/docs/SDK_advanced_guide_offline_en_zh.md rename to runtime/docs/SDK_advanced_guide_offline_en_zh.md index 48c67c9a5..3bd2b7723 100644 --- a/funasr/runtime/docs/SDK_advanced_guide_offline_en_zh.md +++ b/runtime/docs/SDK_advanced_guide_offline_en_zh.md @@ -13,21 +13,30 @@ FunASR提供可一键本地或者云端服务器部署的英文离线文件转 详细性能测试报告（[点击此处](./benchmark_onnx_cpp.md)） -云服务厂商，针对新用户，有3个月免费试用活动，申请教程（[点击此处](https://github.com/alibaba-damo-academy/FunASR/blob/main/funasr/runtime/docs/aliyun_server_tutorial.md)） +云服务厂商，针对新用户，有3个月免费试用活动，申请教程（[点击此处](https://github.com/alibaba-damo-academy/FunASR/blob/main/runtime/docs/aliyun_server_tutorial.md)） ## 快速上手 + +### docker安装 +如果您已安装docker，忽略本步骤！! +通过下述命令在服务器上安装docker： +```shell +curl -O https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/shell/install_docker.sh； +sudo bash install_docker.sh +``` + ### 镜像启动 通过下述命令拉取并启动FunASR runtime-SDK的docker镜像： ```shell sudo docker pull \ - registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-en-cpu-0.1.0 + registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-en-cpu-0.1.1 mkdir -p ./funasr-runtime-resources/models sudo docker run -p 10095:10095 -it --privileged=true \ -v $PWD/funasr-runtime-resources/models:/workspace/models \ - registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-en-cpu-0.1.0 + registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-en-cpu-0.1.1 ``` 如果您没有安装docker，可参考[Docker安装](#Docker安装) @@ -35,7 +44,7 @@ sudo docker run -p 10095:10095 -it --privileged=true \ docker启动之后，启动 funasr-wss-server服务程序： ```shell -cd FunASR/funasr/runtime +cd FunASR/runtime nohup bash run_server.sh \ --download-model-dir /workspace/models \ --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx \ @@ -112,7 +121,7 @@ python3 funasr_wss_client.py --host "127.0.0.1" --port 10095 --mode offline \ --audio_in 需要进行转写的音频文件，支持文件路径，文件列表wav.scp --thread_num 设置并发发送线程数，默认为1 --ssl 设置是否开启ssl证书校验，默认1开启，设置为0关闭 ---hotword 如果模型为热词模型，可以设置热词: *.txt(每行一个热词) 或者空格分隔的热词字符串(阿里巴巴 达摩院) +--hotword 热词文件，每行一个热词，格式(热词 权重)：阿里巴巴 20 --use_itn 设置是否使用itn，默认1开启，设置为0关闭 ``` @@ -129,7 +138,7 @@ python3 funasr_wss_client.py --host "127.0.0.1" --port 10095 --mode offline \ 需要改为部署机器ip --port 10095 部署端口号 --wav-path 需要进行转写的音频文件，支持文件路径 ---hotword 如果模型为热词模型，可以设置热词: *.txt(每行一个热词) 或者空格分隔的热词字符串 (阿里巴巴 达摩院) +--hotword 热词文件，每行一个热词，格式(热词 权重)：阿里巴巴 20 --use-itn 设置是否使用itn，默认1开启，设置为0关闭 ``` @@ -152,7 +161,7 @@ FunasrWsClient --host localhost --port 10095 --audio_in ./asr_example.wav --mode ### 启动FunASR服务 ```shell -cd /workspace/FunASR/funasr/runtime +cd /workspace/FunASR/runtime nohup bash run_server.sh \ --download-model-dir /workspace/models \ --model-dir damo/speech_paraformer-large_asr_nat-en-16k-common-vocab10020-onnx \ @@ -217,11 +226,11 @@ kill -9 PID FunASR-runtime的代码已开源，如果服务端和客户端不能很好的满足您的需求，您可以根据自己的需求进行进一步的开发： ### c++ 客户端： -https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/runtime/websocket +https://github.com/alibaba-damo-academy/FunASR/tree/main/runtime/websocket ### python 客户端： -https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/runtime/python/websocket +https://github.com/alibaba-damo-academy/FunASR/tree/main/runtime/python/websocket ### 自定义客户端： @@ -241,7 +250,7 @@ FUNASR_RESULT result=FsmnVadInfer(vad_hanlde, wav_file.c_str(), NULL, 16000); // 其中：vad_hanlde为FunOfflineInit返回值，wav_file为音频路径，sampling_rate为采样率(默认16k) ``` -使用示例详见：https://github.com/alibaba-damo-academy/FunASR/blob/main/funasr/runtime/onnxruntime/bin/funasr-onnx-offline-vad.cpp +使用示例详见：https://github.com/alibaba-damo-academy/FunASR/blob/main/runtime/onnxruntime/bin/funasr-onnx-offline-vad.cpp #### ASR ```text @@ -252,7 +261,7 @@ FUNASR_RESULT result=FunOfflineInfer(asr_hanlde, wav_file.c_str(), RASR_NONE, NU // 其中：asr_hanlde为FunOfflineInit返回值，wav_file为音频路径，sampling_rate为采样率(默认16k) ``` -使用示例详见：https://github.com/alibaba-damo-academy/FunASR/blob/main/funasr/runtime/onnxruntime/bin/funasr-onnx-offline.cpp +使用示例详见：https://github.com/alibaba-damo-academy/FunASR/blob/main/runtime/onnxruntime/bin/funasr-onnx-offline.cpp #### PUNC ```text @@ -262,4 +271,4 @@ FUNASR_HANDLE punc_hanlde=CTTransformerInit(model_path, thread_num); FUNASR_RESULT result=CTTransformerInfer(punc_hanlde, txt_str.c_str(), RASR_NONE, NULL); // 其中：punc_hanlde为CTTransformerInit返回值，txt_str为文本 ``` -使用示例详见：https://github.com/alibaba-damo-academy/FunASR/blob/main/funasr/runtime/onnxruntime/bin/funasr-onnx-offline-punc.cpp +使用示例详见：https://github.com/alibaba-damo-academy/FunASR/blob/main/runtime/onnxruntime/bin/funasr-onnx-offline-punc.cpp diff --git a/funasr/runtime/docs/SDK_advanced_guide_offline_zh.md b/runtime/docs/SDK_advanced_guide_offline_zh.md similarity index 87% rename from funasr/runtime/docs/SDK_advanced_guide_offline_zh.md rename to runtime/docs/SDK_advanced_guide_offline_zh.md index ee6550108..8655efc1c 100644 --- a/funasr/runtime/docs/SDK_advanced_guide_offline_zh.md +++ b/runtime/docs/SDK_advanced_guide_offline_zh.md @@ -13,21 +13,30 @@ FunASR提供可一键本地或者云端服务器部署的中文离线文件转 详细性能测试报告（[点击此处](./benchmark_onnx_cpp.md)） -云服务厂商，针对新用户，有3个月免费试用活动，申请教程（[点击此处](https://github.com/alibaba-damo-academy/FunASR/blob/main/funasr/runtime/docs/aliyun_server_tutorial.md)） +云服务厂商，针对新用户，有3个月免费试用活动，申请教程（[点击此处](https://github.com/alibaba-damo-academy/FunASR/blob/main/runtime/docs/aliyun_server_tutorial.md)） ## 快速上手 + +### docker安装 +如果您已安装docker，忽略本步骤！! +通过下述命令在服务器上安装docker： +```shell +curl -O https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/shell/install_docker.sh； +sudo bash install_docker.sh +``` + ### 镜像启动 通过下述命令拉取并启动FunASR runtime-SDK的docker镜像： ```shell sudo docker pull \ - registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-cpu-0.2.2 + registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-cpu-0.3.0 mkdir -p ./funasr-runtime-resources/models sudo docker run -p 10095:10095 -it --privileged=true \ -v $PWD/funasr-runtime-resources/models:/workspace/models \ - registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-cpu-0.2.2 + registry.cn-hangzhou.aliyuncs.com/funasr_repo/funasr:funasr-runtime-sdk-cpu-0.3.0 ``` 如果您没有安装docker，可参考[Docker安装](#Docker安装) @@ -35,16 +44,17 @@ sudo docker run -p 10095:10095 -it --privileged=true \ docker启动之后，启动 funasr-wss-server服务程序： ```shell -cd FunASR/funasr/runtime +cd FunASR/runtime nohup bash run_server.sh \ --download-model-dir /workspace/models \ --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx \ --model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-onnx \ --punc-dir damo/punc_ct-transformer_zh-cn-common-vocab272727-onnx \ + --lm-dir damo/speech_ngram_lm_zh-cn-ai-wesp-fst \ --itn-dir thuduj12/fst_itn_zh > log.out 2>&1 & # 如果您想关闭ssl，增加参数：--certfile 0 -# 如果您想使用时间戳或者热词模型进行部署，请设置--model-dir为对应模型： +# 如果您想使用时间戳或者nn热词模型进行部署，请设置--model-dir为对应模型： # damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-onnx（时间戳） # 或者 damo/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404-onnx（热词） @@ -116,7 +126,7 @@ python3 funasr_wss_client.py --host "127.0.0.1" --port 10095 --mode offline \ --audio_in 需要进行转写的音频文件，支持文件路径，文件列表wav.scp --thread_num 设置并发发送线程数，默认为1 --ssl 设置是否开启ssl证书校验，默认1开启，设置为0关闭 ---hotword 如果模型为热词模型，可以设置热词: *.txt(每行一个热词) 或者空格分隔的热词字符串(阿里巴巴 达摩院) +--hotword 热词文件，每行一个热词，格式(热词 权重)：阿里巴巴 20 --use_itn 设置是否使用itn，默认1开启，设置为0关闭 ``` @@ -133,7 +143,7 @@ python3 funasr_wss_client.py --host "127.0.0.1" --port 10095 --mode offline \ 需要改为部署机器ip --port 10095 部署端口号 --wav-path 需要进行转写的音频文件，支持文件路径 ---hotword 如果模型为热词模型，可以设置热词: *.txt(每行一个热词) 或者空格分隔的热词字符串 (阿里巴巴 达摩院) +--hotword 热词文件，每行一个热词，格式(热词 权重)：阿里巴巴 20 --use-itn 设置是否使用itn，默认1开启，设置为0关闭 ``` @@ -154,7 +164,7 @@ FunasrWsClient --host localhost --port 10095 --audio_in ./asr_example.wav --mode ### 启动FunASR服务 ```shell -cd /workspace/FunASR/funasr/runtime +cd /workspace/FunASR/runtime nohup bash run_server.sh \ --download-model-dir /workspace/models \ --model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-onnx \ @@ -166,7 +176,7 @@ nohup bash run_server.sh \ --port 10095 \ --certfile ../../../ssl_key/server.crt \ --keyfile ../../../ssl_key/server.key \ - --hotword ../../hotwords.txt > log.out 2>&1 & + --hotword ../../hotwords.txt > log.out 2>&1 & ``` **run_server.sh命令参数介绍** ```text @@ -177,13 +187,14 @@ nohup bash run_server.sh \ --vad-quant True为量化VAD模型，False为非量化VAD模型，默认是True --punc-dir modelscope model ID --punc-quant True为量化PUNC模型，False为非量化PUNC模型，默认是True +--lm-dir modelscope model ID --itn-dir modelscope model ID --port 服务端监听的端口号，默认为 10095 --decoder-thread-num 服务端启动的推理线程数，默认为 8 --io-thread-num 服务端启动的IO线程数，默认为 1 --certfile ssl的证书文件，默认为：../../../ssl_key/server.crt，如果需要关闭ssl，参数设置为0 --keyfile ssl的密钥文件，默认为：../../../ssl_key/server.key ---hotword 热词文件路径，每一个热词一行，如果客户端提供热词，则与客户端提供的热词合并一起使用。默认为：../../hotwords.txt +--hotword 热词文件路径，每行一个热词，格式：热词 权重(例如:阿里巴巴 20)，如果客户端提供热词，则与客户端提供的热词合并一起使用。 ``` ### 关闭FunASR服务 @@ -212,7 +223,8 @@ kill -9 PID [FSMN-VAD模型](https://www.modelscope.cn/models/damo/speech_fsmn_vad_zh-cn-16k-common-onnx/summary), [Paraformer-lagre模型](https://www.modelscope.cn/models/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-onnx/summary), [CT-Transformer标点预测模型](https://www.modelscope.cn/models/damo/punc_ct-transformer_zh-cn-common-vocab272727-onnx/summary), -[基于FST的中文ITN](https://www.modelscope.cn/models/thuduj12/fst_itn_zh/summary) +[基于FST的中文ITN](https://www.modelscope.cn/models/thuduj12/fst_itn_zh/summary), +[Ngram中文语言模型](https://www.modelscope.cn/models/damo/speech_ngram_lm_zh-cn-ai-wesp-fst/summary) 如果，您希望部署您finetune后的模型（例如10epoch.pb），需要手动将模型重命名为model.pb，并将原modelscope中模型model.pb替换掉，将路径指定为`model_dir`即可。 @@ -222,11 +234,11 @@ kill -9 PID FunASR-runtime的代码已开源，如果服务端和客户端不能很好的满足您的需求，您可以根据自己的需求进行进一步的开发： ### c++ 客户端： -https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/runtime/websocket +https://github.com/alibaba-damo-academy/FunASR/tree/main/runtime/websocket ### python 客户端： -https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/runtime/python/websocket +https://github.com/alibaba-damo-academy/FunASR/tree/main/runtime/python/websocket ### 自定义客户端： @@ -246,7 +258,7 @@ FUNASR_RESULT result=FsmnVadInfer(vad_hanlde, wav_file.c_str(), NULL, 16000); // 其中：vad_hanlde为FunOfflineInit返回值，wav_file为音频路径，sampling_rate为采样率(默认16k) ``` -使用示例详见：https://github.com/alibaba-damo-academy/FunASR/blob/main/funasr/runtime/onnxruntime/bin/funasr-onnx-offline-vad.cpp +使用示例详见：https://github.com/alibaba-damo-academy/FunASR/blob/main/runtime/onnxruntime/bin/funasr-onnx-offline-vad.cpp #### ASR ```text @@ -257,7 +269,7 @@ FUNASR_RESULT result=FunOfflineInfer(asr_hanlde, wav_file.c_str(), RASR_NONE, NU // 其中：asr_hanlde为FunOfflineInit返回值，wav_file为音频路径，sampling_rate为采样率(默认16k) ``` -使用示例详见：https://github.com/alibaba-damo-academy/FunASR/blob/main/funasr/runtime/onnxruntime/bin/funasr-onnx-offline.cpp +使用示例详见：https://github.com/alibaba-damo-academy/FunASR/blob/main/runtime/onnxruntime/bin/funasr-onnx-offline.cpp #### PUNC ```text @@ -267,4 +279,4 @@ FUNASR_HANDLE punc_hanlde=CTTransformerInit(model_path, thread_num); FUNASR_RESULT result=CTTransformerInfer(punc_hanlde, txt_str.c_str(), RASR_NONE, NULL); // 其中：punc_hanlde为CTTransformerInit返回值，txt_str为文本 ``` -使用示例详见：https://github.com/alibaba-damo-academy/FunASR/blob/main/funasr/runtime/onnxruntime/bin/funasr-onnx-offline-punc.cpp +使用示例详见：https://github.com/alibaba-damo-academy/FunASR/blob/main/runtime/onnxruntime/bin/funasr-onnx-offline-punc.cpp diff --git a/funasr/runtime/docs/SDK_advanced_guide_online.md b/runtime/docs/SDK_advanced_guide_online.md similarity index 95% rename from funasr/runtime/docs/SDK_advanced_guide_online.md rename to runtime/docs/SDK_advanced_guide_online.md index e7910b969..3a26db55a 100644 --- a/funasr/runtime/docs/SDK_advanced_guide_online.md +++ b/runtime/docs/SDK_advanced_guide_online.md @@ -18,7 +18,7 @@ If you do not have Docker installed, please refer to [Docker Installation](https After Docker is launched, start the funasr-wss-server-2pass service program: ```shell -cd FunASR/funasr/runtime +cd FunASR/runtime nohup bash run_server_2pass.sh \ --download-model-dir /workspace/models \ --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx \ @@ -61,7 +61,7 @@ For more detailed usage, please click on the links above. For more client versio Use the flollowing script to start the server ： ```shell -cd /workspace/FunASR/funasr/runtime +cd /workspace/FunASR/runtime nohup bash run_server_2pass.sh \ --model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-onnx \ --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx \ @@ -76,7 +76,7 @@ nohup bash run_server_2pass.sh \ --hotword ../../hotwords.txt > log.out 2>&1 & # If you want to close ssl，please add：--certfile 0 -# If you want to deploy the timestamp or hotword model, please set --model-dir to the corresponding model: +# If you want to deploy the timestamp or nn hotword model, please set --model-dir to the corresponding model: # speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-onnx（timestamp） # damo/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404-onnx（hotword） @@ -98,7 +98,7 @@ nohup bash run_server_2pass.sh \ --io-thread-num: Number of IO threads that the server starts. Default is 1. --certfile : SSL certificate file. Default is ../../../ssl_key/server.crt. If you want to close ssl，set 0 --keyfile : SSL key file. Default is ../../../ssl_key/server.key. ---hotword Hotword file path, one line for each hot word, if the client provides hot words, then combined with the hot words provided by the client. Default is ../../hotwords.txt +--hotword: Hotword file path, one line for each hotword(e.g.:阿里巴巴 20), if the client provides hot words, then combined with the hot words provided by the client. ``` ### Shutting Down the FunASR Service diff --git a/funasr/runtime/docs/SDK_advanced_guide_online_zh.md b/runtime/docs/SDK_advanced_guide_online_zh.md similarity index 93% rename from funasr/runtime/docs/SDK_advanced_guide_online_zh.md rename to runtime/docs/SDK_advanced_guide_online_zh.md index 5fdcdb862..ce9165552 100644 --- a/funasr/runtime/docs/SDK_advanced_guide_online_zh.md +++ b/runtime/docs/SDK_advanced_guide_online_zh.md @@ -6,6 +6,15 @@ FunASR提供可便捷本地或者云端服务器部署的实时语音听写服 本文档为FunASR实时转写服务开发指南。如果您想快速体验实时语音听写服务，可参考[快速上手](#快速上手)。 ## 快速上手 + +### docker安装 +如果您已安装docker，忽略本步骤！! +通过下述命令在服务器上安装docker： +```shell +curl -O https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/shell/install_docker.sh； +sudo bash install_docker.sh +``` + ### 镜像启动 通过下述命令拉取并启动FunASR软件包的docker镜像： @@ -24,7 +33,7 @@ sudo docker run -p 10095:10095 -it --privileged=true \ docker启动之后，启动 funasr-wss-server-2pass服务程序： ```shell -cd FunASR/funasr/runtime +cd FunASR/runtime nohup bash run_server_2pass.sh \ --download-model-dir /workspace/models \ --vad-dir damo/speech_fsmn_vad_zh-cn-16k-common-onnx \ @@ -69,7 +78,7 @@ python3 wss_client_asr.py --host "127.0.0.1" --port 10095 --mode 2pass ### 启动FunASR服务 ```shell -cd /workspace/FunASR/funasr/runtime +cd /workspace/FunASR/runtime nohup bash run_server_2pass.sh \ --model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-onnx \ --online-model-dir damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online-onnx \ @@ -99,7 +108,7 @@ nohup bash run_server_2pass.sh \ --io-thread-num 服务端启动的IO线程数，默认为 1 --certfile ssl的证书文件，默认为：../../../ssl_key/server.crt，如果需要关闭ssl，参数设置为0 --keyfile ssl的密钥文件，默认为：../../../ssl_key/server.key ---hotword 热词文件路径，每一个热词一行，如果客户端提供热词，则与客户端提供的热词合并一起使用。默认为：../../hotwords.txt +--hotword 热词文件路径，每行一个热词，格式：热词 权重(例如:阿里巴巴 20)，如果客户端提供热词，则与客户端提供的热词合并一起使用。 ``` ### 关闭FunASR服务 diff --git a/funasr/runtime/docs/SDK_tutorial.md b/runtime/docs/SDK_tutorial.md similarity index 95% rename from funasr/runtime/docs/SDK_tutorial.md rename to runtime/docs/SDK_tutorial.md index c2579c05f..4421d8214 100644 --- a/funasr/runtime/docs/SDK_tutorial.md +++ b/runtime/docs/SDK_tutorial.md @@ -27,16 +27,16 @@ Cloud service providers offer a 3-month free trial for new users. Application tu Download the deployment tool `funasr-runtime-deploy-offline-cpu-zh.sh` ```shell -curl -O https://raw.githubusercontent.com/alibaba-damo-academy/FunASR/main/funasr/runtime/deploy_tools/funasr-runtime-deploy-offline-cpu-zh.sh; +curl -O https://raw.githubusercontent.com/alibaba-damo-academy/FunASR/main/runtime/deploy_tools/funasr-runtime-deploy-offline-cpu-zh.sh; # If there is a network problem, users in mainland China can use the following command: # curl -O https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/shell/funasr-runtime-deploy-offline-cpu-zh.sh; ``` Execute the deployment tool and press the Enter key at the prompt to complete the installation and deployment of the server. Currently, the convenient deployment tool only supports Linux environments. For other environments, please refer to the development guide ([docs](./SDK_advanced_guide_offline.md)). ```shell -sudo bash funasr-runtime-deploy-offline-cpu-zh.sh install --workspace /root/funasr-runtime-resources +sudo bash funasr-runtime-deploy-offline-cpu-zh.sh install --workspace ./funasr-runtime-resources ``` -Note: If you need to deploy the timestamp model or hotword model, select the corresponding model in step 2 of the installation and deployment process, where 1 is the paraformer-large model, 2 is the paraformer-large timestamp model, and 3 is the paraformer-large hotword model. +Note: If you need to deploy the timestamp model or hotword model, select the corresponding model in step 2 of the installation and deployment process, where 1 is the paraformer-large model, 2 is the paraformer-large timestamp model, and 3 is the paraformer-large hotword model；The server loads the hotword file from: ./funasr-runtime-resources/hotowrds.txt (one hotword per line, formatted as word weight: 阿里巴巴 20). ### Client Testing and Usage @@ -73,7 +73,7 @@ Command parameter instructions: --audio_in is the audio file that needs to be transcribed, supporting file paths and file list wav.scp --thread_num sets the number of concurrent sending threads, default is 1 --ssl sets whether to enable SSL certificate verification, default is 1 to enable, and 0 to disable ---hotword If am is hotword model, setting hotword: *.txt(one hotword perline) or hotwords seperate by space (could be: 阿里巴巴 达摩院) +--hotword: Hotword file path, one line for each hotword(e.g.:阿里巴巴 20) --use_itn: whether to use itn, the default value is 1 for enabling and 0 for disabling. ``` @@ -91,7 +91,7 @@ Command parameter description: --wav-path specifies the audio file to be transcribed, and supports file paths. --thread_num sets the number of concurrent send threads, with a default value of 1. --ssl sets whether to enable SSL certificate verification, with a default value of 1 for enabling and 0 for disabling. ---hotword If am is hotword model, setting hotword: *.txt(one hotword perline) or hotwords seperate by space (could be: 阿里巴巴 达摩院) +--hotword: Hotword file path, one line for each hotword(e.g.:阿里巴巴 20) --use-itn: whether to use itn, the default value is 1 for enabling and 0 for disabling. ``` diff --git a/funasr/runtime/docs/SDK_tutorial_en.md b/runtime/docs/SDK_tutorial_en.md similarity index 96% rename from funasr/runtime/docs/SDK_tutorial_en.md rename to runtime/docs/SDK_tutorial_en.md index be4733282..09dcf4642 100644 --- a/funasr/runtime/docs/SDK_tutorial_en.md +++ b/runtime/docs/SDK_tutorial_en.md @@ -27,7 +27,7 @@ Cloud service providers offer a 3-month free trial for new users. Application tu Download the deployment tool `funasr-runtime-deploy-offline-cpu-en.sh` ```shell -curl -O https://raw.githubusercontent.com/alibaba-damo-academy/FunASR/main/funasr/runtime/deploy_tools/funasr-runtime-deploy-offline-cpu-en.sh; +curl -O https://raw.githubusercontent.com/alibaba-damo-academy/FunASR/main/runtime/deploy_tools/funasr-runtime-deploy-offline-cpu-en.sh; # If there is a network problem, users in mainland China can use the following command: # curl -O https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/shell/funasr-runtime-deploy-offline-cpu-en.sh; ``` @@ -72,7 +72,7 @@ Command parameter instructions: --audio_in is the audio file that needs to be transcribed, supporting file paths and file list wav.scp --thread_num sets the number of concurrent sending threads, default is 1 --ssl sets whether to enable SSL certificate verification, default is 1 to enable, and 0 to disable ---hotword If am is hotword model, setting hotword: *.txt(one hotword perline) or hotwords seperate by space (could be: 阿里巴巴 达摩院) +--hotword: Hotword file path, one line for each hotword(e.g.:阿里巴巴 20) --use_itn: whether to use itn, the default value is 1 for enabling and 0 for disabling. ``` @@ -90,7 +90,7 @@ Command parameter description: --wav-path specifies the audio file to be transcribed, and supports file paths. --thread_num sets the number of concurrent send threads, with a default value of 1. --ssl sets whether to enable SSL certificate verification, with a default value of 1 for enabling and 0 for disabling. ---hotword If am is hotword model, setting hotword: *.txt(one hotword perline) or hotwords seperate by space (could be: 阿里巴巴 达摩院) +--hotword: Hotword file path, one line for each hotword(e.g.:阿里巴巴 20) --use-itn: whether to use itn, the default value is 1 for enabling and 0 for disabling. ``` diff --git a/funasr/runtime/docs/SDK_tutorial_en_zh.md b/runtime/docs/SDK_tutorial_en_zh.md similarity index 94% rename from funasr/runtime/docs/SDK_tutorial_en_zh.md rename to runtime/docs/SDK_tutorial_en_zh.md index 015036a74..9e66dd177 100644 --- a/funasr/runtime/docs/SDK_tutorial_en_zh.md +++ b/runtime/docs/SDK_tutorial_en_zh.md @@ -17,7 +17,7 @@ FunASR提供可便捷本地或者云端服务器部署的离线文件转写服 详细性能测试报告（[点击此处](./benchmark_onnx_cpp.md)） -云服务厂商，针对新用户，有3个月免费试用活动，申请教程（[点击此处](https://github.com/alibaba-damo-academy/FunASR/blob/main/funasr/runtime/docs/aliyun_server_tutorial.md)） +云服务厂商，针对新用户，有3个月免费试用活动，申请教程（[点击此处](https://github.com/alibaba-damo-academy/FunASR/blob/main/runtime/docs/aliyun_server_tutorial.md)） ## 快速上手 @@ -28,7 +28,7 @@ FunASR提供可便捷本地或者云端服务器部署的离线文件转写服 下载部署工具`funasr-runtime-deploy-offline-cpu-en.sh` ```shell -curl -O https://raw.githubusercontent.com/alibaba-damo-academy/FunASR/main/funasr/runtime/deploy_tools/funasr-runtime-deploy-offline-cpu-en.sh; +curl -O https://raw.githubusercontent.com/alibaba-damo-academy/FunASR/main/runtime/deploy_tools/funasr-runtime-deploy-offline-cpu-en.sh; # 如遇到网络问题，中国大陆用户，可以使用下面的命令： # curl -O https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/shell/funasr-runtime-deploy-offline-cpu-en.sh; ``` @@ -75,7 +75,7 @@ python3 funasr_wss_client.py --host "127.0.0.1" --port 10095 --mode offline --au --audio_in 需要进行转写的音频文件，支持文件路径，文件列表wav.scp --thread_num 设置并发发送线程数，默认为1 --ssl 设置是否开启ssl证书校验，默认1开启，设置为0关闭 ---hotword 如果模型为热词模型，可以设置热词: *.txt(每行一个热词) 或者空格分隔的热词字符串 (阿里巴巴 达摩院) +--hotword 热词文件，每行一个热词，格式(热词 权重)：阿里巴巴 20 --use_itn 设置是否使用itn，默认1开启，设置为0关闭 ``` @@ -94,7 +94,7 @@ python3 funasr_wss_client.py --host "127.0.0.1" --port 10095 --mode offline --au --wav-path 需要进行转写的音频文件，支持文件路径 --thread_num 设置并发发送线程数，默认为1 --ssl 设置是否开启ssl证书校验，默认1开启，设置为0关闭 ---hotword 如果模型为热词模型，可以设置热词: *.txt(每行一个热词) 或者空格分隔的热词字符串 (阿里巴巴 达摩院) +--hotword 热词文件，每行一个热词，格式(热词 权重)：阿里巴巴 20 --use-itn 设置是否使用itn，默认1开启，设置为0关闭 ``` diff --git a/funasr/runtime/docs/SDK_tutorial_online.md b/runtime/docs/SDK_tutorial_online.md similarity index 96% rename from funasr/runtime/docs/SDK_tutorial_online.md rename to runtime/docs/SDK_tutorial_online.md index d7a429b12..a38502d3c 100644 --- a/funasr/runtime/docs/SDK_tutorial_online.md +++ b/runtime/docs/SDK_tutorial_online.md @@ -24,7 +24,7 @@ Cloud service providers offer a 3-month free trial for new users. Application tu Download the deployment tool `funasr-runtime-deploy-online-cpu-zh.sh` ```shell -curl -O https://raw.githubusercontent.com/alibaba-damo-academy/FunASR/main/funasr/runtime/deploy_tools/funasr-runtime-deploy-online-cpu-zh.sh; +curl -O https://raw.githubusercontent.com/alibaba-damo-academy/FunASR/main/runtime/deploy_tools/funasr-runtime-deploy-online-cpu-zh.sh; # If there is a network problem, users in mainland China can use the following command: # curl -O https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/shell/funasr-runtime-deploy-online-cpu-zh.sh; ``` @@ -33,7 +33,7 @@ Execute the deployment tool and press the Enter key at the prompt to complete th ```shell sudo bash funasr-runtime-deploy-online-cpu-zh.sh install --workspace ./funasr-runtime-resources ``` -Note: If you need to deploy the timestamp model or hotword model, select the corresponding model in step 2 of the installation and deployment process, where 1 is the paraformer-large model, 2 is the paraformer-large timestamp model, and 3 is the paraformer-large hotword model. +Note: If you need to deploy the timestamp model or hotword model, select the corresponding model in step 2 of the installation and deployment process, where 1 is the paraformer-large model, 2 is the paraformer-large timestamp model, and 3 is the paraformer-large hotword model；The server loads the hotword file from: ./funasr-runtime-resources/hotowrds.txt (one hotword per line, formatted as word weight: 阿里巴巴 20). ### Client Testing and Usage @@ -71,7 +71,7 @@ Command parameter instructions: --audio_in is the audio file that needs to be transcribed, supporting file paths and file list wav.scp --thread_num sets the number of concurrent sending threads, default is 1 --ssl sets whether to enable SSL certificate verification, default is 1 to enable, and 0 to disable ---hotword: If am is hotword model, setting hotword: *.txt(one hotword perline) or hotwords seperate by space (could be: 阿里巴巴 达摩院) +--hotword: Hotword file path, one line for each hotword(e.g.:阿里巴巴 20) --use_itn: whether to use itn, the default value is 1 for enabling and 0 for disabling. ``` @@ -91,7 +91,7 @@ Command parameter description: --wav-path specifies the audio file to be transcribed, and supports file paths. --threa-num sets the number of concurrent send threads, with a default value of 1. --is-ssl sets whether to enable SSL certificate verification, with a default value of 1 for enabling and 0 for disabling. ---hotword: If am is hotword model, setting hotword: *.txt(one hotword perline) or hotwords seperate by space (could be: 阿里巴巴 达摩院) +--hotword: Hotword file path, one line for each hotword(e.g.:阿里巴巴 20) --use-itn: whether to use itn, the default value is 1 for enabling and 0 for disabling. ``` diff --git a/funasr/runtime/docs/SDK_tutorial_online_zh.md b/runtime/docs/SDK_tutorial_online_zh.md similarity index 94% rename from funasr/runtime/docs/SDK_tutorial_online_zh.md rename to runtime/docs/SDK_tutorial_online_zh.md index f0d447a88..21a5aa68f 100644 --- a/funasr/runtime/docs/SDK_tutorial_online_zh.md +++ b/runtime/docs/SDK_tutorial_online_zh.md @@ -14,7 +14,7 @@ FunASR集成了达摩院语音实验室在Modelscope社区开源的语音端点 详细性能测试报告（[点击此处](./benchmark_onnx_cpp.md)） -云服务厂商，针对新用户，有3个月免费试用活动，申请教程（[点击此处](https://github.com/alibaba-damo-academy/FunASR/blob/main/funasr/runtime/docs/aliyun_server_tutorial.md)） +云服务厂商，针对新用户，有3个月免费试用活动，申请教程（[点击此处](https://github.com/alibaba-damo-academy/FunASR/blob/main/runtime/docs/aliyun_server_tutorial.md)） ## 快速上手 @@ -25,7 +25,7 @@ FunASR集成了达摩院语音实验室在Modelscope社区开源的语音端点 下载部署工具`funasr-runtime-deploy-online-cpu-zh.sh` ```shell -curl -O https://raw.githubusercontent.com/alibaba-damo-academy/FunASR/main/funasr/runtime/deploy_tools/funasr-runtime-deploy-online-cpu-zh.sh; +curl -O https://raw.githubusercontent.com/alibaba-damo-academy/FunASR/main/runtime/deploy_tools/funasr-runtime-deploy-online-cpu-zh.sh; # 如遇到网络问题，中国大陆用户，可以使用下面的命令： # curl -O https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/shell/funasr-runtime-deploy-online-cpu-zh.sh; ``` @@ -34,7 +34,7 @@ curl -O https://raw.githubusercontent.com/alibaba-damo-academy/FunASR/main/funas ```shell sudo bash funasr-runtime-deploy-online-cpu-zh.sh install --workspace ./funasr-runtime-resources ``` -**注：如果需要部署时间戳模型或者热词模型，在安装部署步骤2时选择对应模型，其中1为paraformer-large模型，2为paraformer-large 时间戳模型，3为paraformer-large 热词模型** +**注：如果需要部署时间戳模型或者热词模型，在安装部署步骤2时选择对应模型，其中1为paraformer-large模型，2为paraformer-large 时间戳模型，3为paraformer-large nn热词模型；服务端加载热词文件地址为：./funasr-runtime-resources/hotowrds.txt(每行一个热词，格式为 热词 权重：阿里巴巴 20)** ### 客户端测试与使用 @@ -76,7 +76,7 @@ python3 funasr_wss_client.py --host "127.0.0.1" --port 10095 --mode 2pass --audio_in 需要进行转写的音频文件，支持文件路径，文件列表wav.scp --thread_num 设置并发发送线程数，默认为1 --ssl 设置是否开启ssl证书校验，默认1开启，设置为0关闭+ ---hotword 如果模型为热词模型，可以设置热词: *.txt(每行一个热词) 或者空格分隔的热词字符串 (阿里巴巴 达摩院) +--hotword 热词文件，每行一个热词，格式(热词 权重)：阿里巴巴 20 --use_itn 设置是否使用itn，默认1开启，设置为0关闭 ``` @@ -99,7 +99,7 @@ python3 funasr_wss_client.py --host "127.0.0.1" --port 10095 --mode 2pass --wav-path 需要进行转写的音频文件，支持文件路径 --thread-num 设置并发发送线程数，默认为1 --is-ssl 设置是否开启ssl证书校验，默认1开启，设置为0关闭 ---hotword 如果模型为热词模型，可以设置热词: *.txt(每行一个热词) 或者空格分隔的热词字符串 (阿里巴巴 达摩院) +--hotword 热词文件，每行一个热词，格式(热词 权重)：阿里巴巴 20 --use-itn 设置是否使用itn，默认1开启，设置为0关闭 ``` diff --git a/funasr/runtime/docs/SDK_tutorial_zh.md b/runtime/docs/SDK_tutorial_zh.md similarity index 94% rename from funasr/runtime/docs/SDK_tutorial_zh.md rename to runtime/docs/SDK_tutorial_zh.md index 2c0895723..e2572cdf5 100644 --- a/funasr/runtime/docs/SDK_tutorial_zh.md +++ b/runtime/docs/SDK_tutorial_zh.md @@ -17,7 +17,7 @@ FunASR提供可便捷本地或者云端服务器部署的离线文件转写服 详细性能测试报告（[点击此处](./benchmark_onnx_cpp.md)） -云服务厂商，针对新用户，有3个月免费试用活动，申请教程（[点击此处](https://github.com/alibaba-damo-academy/FunASR/blob/main/funasr/runtime/docs/aliyun_server_tutorial.md)） +云服务厂商，针对新用户，有3个月免费试用活动，申请教程（[点击此处](https://github.com/alibaba-damo-academy/FunASR/blob/main/runtime/docs/aliyun_server_tutorial.md)） ## 快速上手 @@ -28,7 +28,7 @@ FunASR提供可便捷本地或者云端服务器部署的离线文件转写服 下载部署工具`funasr-runtime-deploy-offline-cpu-zh.sh` ```shell -curl -O https://raw.githubusercontent.com/alibaba-damo-academy/FunASR/main/funasr/runtime/deploy_tools/funasr-runtime-deploy-offline-cpu-zh.sh; +curl -O https://raw.githubusercontent.com/alibaba-damo-academy/FunASR/main/runtime/deploy_tools/funasr-runtime-deploy-offline-cpu-zh.sh; # 如遇到网络问题，中国大陆用户，可以使用下面的命令： # curl -O https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/shell/funasr-runtime-deploy-offline-cpu-zh.sh; ``` @@ -37,7 +37,8 @@ curl -O https://raw.githubusercontent.com/alibaba-damo-academy/FunASR/main/funas ```shell sudo bash funasr-runtime-deploy-offline-cpu-zh.sh install --workspace ./funasr-runtime-resources ``` -**注：如果需要部署时间戳模型或者热词模型，在安装部署步骤2时选择对应模型，其中1为paraformer-large模型，2为paraformer-large 时间戳模型，3为paraformer-large 热词模型** +**注：如果需要部署时间戳模型或者热词模型，在安装部署步骤2时选择对应模型，其中1为paraformer-large模型，2为paraformer-large 时间戳模型，3为paraformer-large nn热词模型；服务端加载热词文件地址为：./funasr-runtime-resources/hotowrds.txt(每行一个热词，格式为 热词 权重：阿里巴巴 20)** + ### 客户端测试与使用 @@ -76,7 +77,7 @@ python3 funasr_wss_client.py --host "127.0.0.1" --port 10095 --mode offline --au --audio_in 需要进行转写的音频文件，支持文件路径，文件列表wav.scp --thread_num 设置并发发送线程数，默认为1 --ssl 设置是否开启ssl证书校验，默认1开启，设置为0关闭 ---hotword 如果模型为热词模型，可以设置热词: *.txt(每行一个热词) 或者空格分隔的热词字符串 (阿里巴巴 达摩院) +--hotword 热词文件，每行一个热词，格式(热词 权重)：阿里巴巴 20 --use_itn 设置是否使用itn，默认1开启，设置为0关闭 ``` @@ -95,7 +96,7 @@ python3 funasr_wss_client.py --host "127.0.0.1" --port 10095 --mode offline --au --wav-path 需要进行转写的音频文件，支持文件路径 --thread_num 设置并发发送线程数，默认为1 --ssl 设置是否开启ssl证书校验，默认1开启，设置为0关闭 ---hotword 如果模型为热词模型，可以设置热词: *.txt(每行一个热词) 或者空格分隔的热词字符串 (阿里巴巴 达摩院) +--hotword 热词文件，每行一个热词，格式(热词 权重)：阿里巴巴 20 --use-itn 设置是否使用itn，默认1开启，设置为0关闭 ``` diff --git a/funasr/runtime/docs/aliyun_server_tutorial.md b/runtime/docs/aliyun_server_tutorial.md similarity index 100% rename from funasr/runtime/docs/aliyun_server_tutorial.md rename to runtime/docs/aliyun_server_tutorial.md diff --git a/funasr/runtime/docs/benchmark_libtorch.md b/runtime/docs/benchmark_libtorch.md similarity index 100% rename from funasr/runtime/docs/benchmark_libtorch.md rename to runtime/docs/benchmark_libtorch.md diff --git a/funasr/runtime/docs/benchmark_onnx.md b/runtime/docs/benchmark_onnx.md similarity index 100% rename from funasr/runtime/docs/benchmark_onnx.md rename to runtime/docs/benchmark_onnx.md diff --git a/funasr/runtime/docs/benchmark_onnx_cpp.md b/runtime/docs/benchmark_onnx_cpp.md similarity index 62% rename from funasr/runtime/docs/benchmark_onnx_cpp.md rename to runtime/docs/benchmark_onnx_cpp.md index 20811af43..87162ac7f 100644 --- a/funasr/runtime/docs/benchmark_onnx_cpp.md +++ b/runtime/docs/benchmark_onnx_cpp.md @@ -46,7 +46,7 @@ make ## [Paraformer-large](https://www.modelscope.cn/models/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/summary) ```shell ./funasr-onnx-offline-rtf \ - --model-dir ./asrmodel/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch \ + --model-dir ./damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch \ --quantize true \ --wav-path ./aishell1_test.scp \ --thread-num 32 @@ -100,14 +100,14 @@ CER after int8-quant: 1.95% | 96 (onnx fp32) | 115s | 0.003183 | 314 | | 96 (onnx int8) | 80s | 0.002222 | 450 | -## [FSMN-VAD](https://www.modelscope.cn/models/damo/speech_fsmn_vad_zh-cn-16k-common-pytorch/summary) + [Paraformer-large](https://www.modelscope.cn/models/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/summary) + [CT-Transformer](https://www.modelscope.cn/models/damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/summary) +## [FSMN-VAD](https://www.modelscope.cn/models/damo/speech_fsmn_vad_zh-cn-16k-common-onnx/summary) + [Paraformer-large](https://www.modelscope.cn/models/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/summary) + [CT-Transformer](https://www.modelscope.cn/models/damo/punc_ct-transformer_zh-cn-common-vocab272727-onnx/summary) ```shell ./funasr-onnx-offline-rtf \ - --model-dir ./asrmodel/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch \ + --model-dir ./damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch \ --quantize true \ - --vad-dir ./asrmodel/speech_fsmn_vad_zh-cn-16k-common-pytorch \ - --punc-dir ./asrmodel/punc_ct-transformer_zh-cn-common-vocab272727-pytorch \ + --vad-dir ./damo/speech_fsmn_vad_zh-cn-16k-common-onnx \ + --punc-dir ./damo/punc_ct-transformer_zh-cn-common-vocab272727-onnx \ --wav-path ./aishell1_test.scp \ --thread-num 32 @@ -150,14 +150,82 @@ Node: '--quantize false' means fp32, otherwise it will be int8 | 96 (onnx fp32) | 117s | 0.003257 | 307 | | 96 (onnx int8) | 81s | 0.002258 | 442 | -## [FSMN-VAD](https://www.modelscope.cn/models/damo/speech_fsmn_vad_zh-cn-16k-common-pytorch/summary) + [Paraformer-en](https://www.modelscope.cn/models/damo/speech_paraformer-large_asr_nat-en-16k-common-vocab10020-onnx/summary) + [CT-Transformer](https://www.modelscope.cn/models/damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/summary) + + +## [FSMN-VAD](https://www.modelscope.cn/models/damo/speech_fsmn_vad_zh-cn-16k-common-onnx/summary) + [Paraformer-large](https://www.modelscope.cn/models/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/summary) +[Ngram](https://www.modelscope.cn/models/damo/speech_ngram_lm_zh-cn-ai-wesp-fst/summary) + [CT-Transformer](https://www.modelscope.cn/models/damo/punc_ct-transformer_zh-cn-common-vocab272727-onnx/summary) ```shell ./funasr-onnx-offline-rtf \ - --model-dir ./asrmodel/speech_paraformer-large_asr_nat-en-16k-common-vocab10020-onnx \ + --model-dir ./damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch \ --quantize true \ - --vad-dir ./asrmodel/speech_fsmn_vad_zh-cn-16k-common-pytorch \ - --punc-dir ./asrmodel/punc_ct-transformer_zh-cn-common-vocab272727-pytorch \ + --vad-dir ./damo/speech_fsmn_vad_zh-cn-16k-common-onnx \ + --punc-dir ./damo/punc_ct-transformer_zh-cn-common-vocab272727-onnx \ + --lm-dir ./damo/speech_ngram_lm_zh-cn-ai-wesp-fst \ + --wav-path ./aishell1_test.scp \ + --thread-num 32 + +Node: '--quantize false' means fp32, otherwise it will be int8 +``` + + ### Intel(R) Xeon(R) Platinum 8369B CPU @ 2.90GHz 16core-32processor with avx512_vnni + +| concurrent-tasks | processing time(s) | RTF | Speedup Rate | +|------------------|:------------------:|:------:|:------------:| +| 1 (onnx fp32) | 2506s | 0.0694 | 14 | +| 1 (onnx int8) | 1448s | 0.0401 | 25 | +| 8 (onnx fp32) | 326s | 0.0090 | 110 | +| 8 (onnx int8) | 184s | 0.0051 | 195 | +| 16 (onnx fp32) | 178s | 0.0049 | 202 | +| 16 (onnx int8) | 99s | 0.0027 | 361 | +| 32 (onnx fp32) | 152s | 0.0042 | 236 | +| 32 (onnx int8) | 85s | 0.0023 | 422 | +| 64 (onnx fp32) | 157s | 0.0043 | 228 | +| 64 (onnx int8) | 89s | 0.0024 | 403 | +| 96 (onnx fp32) | 158s | 0.0044 | 227 | +| 96 (onnx int8) | 91s | 0.0025 | 396 | + + +```shell +# using hotwords +./funasr-onnx-offline-rtf \ + --model-dir ./damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch \ + --quantize true \ + --vad-dir ./damo/speech_fsmn_vad_zh-cn-16k-common-onnx \ + --punc-dir ./damo/punc_ct-transformer_zh-cn-common-vocab272727-onnx \ + --lm-dir ./damo/speech_ngram_lm_zh-cn-ai-wesp-fst \ + --hotword hotwords_dev_600.txt \ + --wav-path ./aishell1_test.scp \ + --thread-num 32 + +Node: '--quantize false' means fp32, otherwise it will be int8 +``` + + ### Intel(R) Xeon(R) Platinum 8369B CPU @ 2.90GHz 16core-32processor with avx512_vnni + +| concurrent-tasks | processing time(s) | RTF | Speedup Rate | +|------------------|:------------------:|:------:|:------------:| +| 1 (onnx fp32) | 3172s | 0.0878 | 11 | +| 1 (onnx int8) | 2140s | 0.0592 | 16 | +| 8 (onnx fp32) | 412s | 0.0114 | 87 | +| 8 (onnx int8) | 268s | 0.0074 | 134 | +| 16 (onnx fp32) | 218s | 0.0060 | 165 | +| 16 (onnx int8) | 140s | 0.0038 | 257 | +| 32 (onnx fp32) | 183s | 0.0050 | 196 | +| 32 (onnx int8) | 116s | 0.0032 | 310 | +| 64 (onnx fp32) | 188s | 0.0052 | 192 | +| 64 (onnx int8) | 120s | 0.0033 | 299 | +| 96 (onnx fp32) | 191s | 0.0052 | 188 | +| 96 (onnx int8) | 122s | 0.0033 | 294 | + + +## [FSMN-VAD](https://www.modelscope.cn/models/damo/speech_fsmn_vad_zh-cn-16k-common-onnx/summary) + [Paraformer-en](https://www.modelscope.cn/models/damo/speech_paraformer-large_asr_nat-en-16k-common-vocab10020-onnx/summary) + [CT-Transformer](https://www.modelscope.cn/models/damo/punc_ct-transformer_zh-cn-common-vocab272727-onnx/summary) + +```shell +./funasr-onnx-offline-rtf \ + --model-dir ./damo/speech_paraformer-large_asr_nat-en-16k-common-vocab10020-onnx \ + --quantize true \ + --vad-dir ./damo/speech_fsmn_vad_zh-cn-16k-common-onnx \ + --punc-dir ./damo/punc_ct-transformer_zh-cn-common-vocab272727-onnx \ --wav-path ./librispeech_test_clean.scp \ --thread-num 32 diff --git a/funasr/runtime/docs/docker_offline_cpu_en_lists b/runtime/docs/docker_offline_cpu_en_lists similarity index 87% rename from funasr/runtime/docs/docker_offline_cpu_en_lists rename to runtime/docs/docker_offline_cpu_en_lists index e826c74b9..fb3192f2a 100644 --- a/funasr/runtime/docs/docker_offline_cpu_en_lists +++ b/runtime/docs/docker_offline_cpu_en_lists @@ -1,5 +1,5 @@ DOCKER: - funasr-runtime-sdk-en-cpu-0.1.0 + funasr-runtime-sdk-en-cpu-0.1.1 DEFAULT_ASR_MODEL: damo/speech_paraformer-large_asr_nat-en-16k-common-vocab10020-onnx DEFAULT_VAD_MODEL: diff --git a/funasr/runtime/docs/docker_offline_cpu_zh_lists b/runtime/docs/docker_offline_cpu_zh_lists similarity index 82% rename from funasr/runtime/docs/docker_offline_cpu_zh_lists rename to runtime/docs/docker_offline_cpu_zh_lists index df6eddbea..4651784f9 100644 --- a/funasr/runtime/docs/docker_offline_cpu_zh_lists +++ b/runtime/docs/docker_offline_cpu_zh_lists @@ -1,7 +1,7 @@ DOCKER: + funasr-runtime-sdk-cpu-0.3.0 funasr-runtime-sdk-cpu-0.2.2 funasr-runtime-sdk-cpu-0.2.1 - funasr-runtime-sdk-cpu-0.2.0 DEFAULT_ASR_MODEL: damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-onnx damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-onnx @@ -10,3 +10,6 @@ DEFAULT_VAD_MODEL: damo/speech_fsmn_vad_zh-cn-16k-common-onnx DEFAULT_PUNC_MODEL: damo/punc_ct-transformer_zh-cn-common-vocab272727-onnx +DEFAULT_LM_MODEL: + damo/speech_ngram_lm_zh-cn-ai-wesp-fst + NONE diff --git a/funasr/runtime/docs/docker_online_cpu_zh_lists b/runtime/docs/docker_online_cpu_zh_lists similarity index 93% rename from funasr/runtime/docs/docker_online_cpu_zh_lists rename to runtime/docs/docker_online_cpu_zh_lists index 46bd196be..903d2b470 100644 --- a/funasr/runtime/docs/docker_online_cpu_zh_lists +++ b/runtime/docs/docker_online_cpu_zh_lists @@ -1,7 +1,7 @@ DOCKER: + funasr-runtime-sdk-online-cpu-0.1.4 funasr-runtime-sdk-online-cpu-0.1.3 funasr-runtime-sdk-online-cpu-0.1.2 - funasr-runtime-sdk-online-cpu-0.1.1 DEFAULT_ASR_MODEL: damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-onnx damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-onnx diff --git a/funasr/runtime/docs/images/2pass.jpg b/runtime/docs/images/2pass.jpg similarity index 100% rename from funasr/runtime/docs/images/2pass.jpg rename to runtime/docs/images/2pass.jpg diff --git a/funasr/runtime/docs/images/aliyun1.png b/runtime/docs/images/aliyun1.png similarity index 100% rename from funasr/runtime/docs/images/aliyun1.png rename to runtime/docs/images/aliyun1.png diff --git a/funasr/runtime/docs/images/aliyun10.png b/runtime/docs/images/aliyun10.png similarity index 100% rename from funasr/runtime/docs/images/aliyun10.png rename to runtime/docs/images/aliyun10.png diff --git a/funasr/runtime/docs/images/aliyun11.png b/runtime/docs/images/aliyun11.png similarity index 100% rename from funasr/runtime/docs/images/aliyun11.png rename to runtime/docs/images/aliyun11.png diff --git a/funasr/runtime/docs/images/aliyun12.png b/runtime/docs/images/aliyun12.png similarity index 100% rename from funasr/runtime/docs/images/aliyun12.png rename to runtime/docs/images/aliyun12.png diff --git a/funasr/runtime/docs/images/aliyun2.png b/runtime/docs/images/aliyun2.png similarity index 100% rename from funasr/runtime/docs/images/aliyun2.png rename to runtime/docs/images/aliyun2.png diff --git a/funasr/runtime/docs/images/aliyun3.png b/runtime/docs/images/aliyun3.png similarity index 100% rename from funasr/runtime/docs/images/aliyun3.png rename to runtime/docs/images/aliyun3.png diff --git a/funasr/runtime/docs/images/aliyun4.png b/runtime/docs/images/aliyun4.png similarity index 100% rename from funasr/runtime/docs/images/aliyun4.png rename to runtime/docs/images/aliyun4.png diff --git a/funasr/runtime/docs/images/aliyun5.png b/runtime/docs/images/aliyun5.png similarity index 100% rename from funasr/runtime/docs/images/aliyun5.png rename to runtime/docs/images/aliyun5.png diff --git a/funasr/runtime/docs/images/aliyun6.png b/runtime/docs/images/aliyun6.png similarity index 100% rename from funasr/runtime/docs/images/aliyun6.png rename to runtime/docs/images/aliyun6.png diff --git a/funasr/runtime/docs/images/aliyun7.png b/runtime/docs/images/aliyun7.png similarity index 100% rename from funasr/runtime/docs/images/aliyun7.png rename to runtime/docs/images/aliyun7.png diff --git a/funasr/runtime/docs/images/aliyun8.png b/runtime/docs/images/aliyun8.png similarity index 100% rename from funasr/runtime/docs/images/aliyun8.png rename to runtime/docs/images/aliyun8.png diff --git a/funasr/runtime/docs/images/aliyun9.png b/runtime/docs/images/aliyun9.png similarity index 100% rename from funasr/runtime/docs/images/aliyun9.png rename to runtime/docs/images/aliyun9.png diff --git a/funasr/runtime/docs/images/html.png b/runtime/docs/images/html.png similarity index 100% rename from funasr/runtime/docs/images/html.png rename to runtime/docs/images/html.png diff --git a/runtime/docs/lm_train_tutorial.md b/runtime/docs/lm_train_tutorial.md new file mode 100644 index 000000000..e0b427b93 --- /dev/null +++ b/runtime/docs/lm_train_tutorial.md @@ -0,0 +1,40 @@ +# 如何训练LM + +训练脚本详见（[点击此处](../tools/train_compile_ngram.sh)） + +## 数据准备 +```shell +# 下载: 示例训练语料text、lexicon 和 am建模单元units.txt +wget https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/requirements/lm.tar.gz +tar -zxvf lm.tar.gz +``` + +## 训练arpa +```shell +# make sure that srilm is installed +# the format of the text should be: +# BAC009S0002W0122 而 对 楼市 成交 抑制 作用 最 大 的 限 购 +# BAC009S0002W0123 也 成为 地方 政府 的 眼中 钉 + +bash fst/train_lms.sh +``` + +## 生成lexicon +```shell +python3 fst/generate_lexicon.py lm/corpus.dict lm/lexicon.txt lm/lexicon.out +``` + +## 编译TLG.fst +```shell +# Compile the lexicon and token FSTs +fst/compile_dict_token.sh lm lm/tmp lm/lang + +# Compile the language-model FST and the final decoding graph TLG.fst +fst/make_decode_graph.sh lm lm/lang || exit 1; + +# Collect resource files required for decoding +fst/collect_resource_file.sh lm lm/resource + +#编译后的模型资源位于 lm/resource +``` + diff --git a/funasr/runtime/docs/websocket_protocol.md b/runtime/docs/websocket_protocol.md similarity index 87% rename from funasr/runtime/docs/websocket_protocol.md rename to runtime/docs/websocket_protocol.md index cc4052fb3..f03b6881e 100644 --- a/funasr/runtime/docs/websocket_protocol.md +++ b/runtime/docs/websocket_protocol.md @@ -10,7 +10,7 @@ Configuration parameters and meta information are in JSON format, while audio da #### Initial Communication The message (which needs to be serialized in JSON) is: ```text -{"mode": "offline", "wav_name": "wav_name","wav_format":"pcm","is_speaking": True,"wav_format":"pcm","hotwords":"阿里巴巴 达摩院 阿里云","itn":true} +{"mode": "offline", "wav_name": "wav_name", "wav_format":"pcm", "is_speaking": True, "wav_format":"pcm", "hotwords":"{"阿里巴巴":20,"通义实验室":30}", "itn":True} ``` Parameter explanation: ```text @@ -19,7 +19,7 @@ Parameter explanation: `wav_format`: the audio and video file extension, such as pcm, mp3, mp4, etc. `is_speaking`: False indicates the end of a sentence, such as a VAD segmentation point or the end of a WAV file `audio_fs`: when the input audio is in PCM format, the audio sampling rate parameter needs to be added -`hotwords`：If AM is the hotword model, hotword data needs to be sent to the server in string format, with " " used as a separator between hotwords. For example："阿里巴巴 达摩院 阿里云" +`hotwords`：If using the hotword, you need to send the hotword data (string) to the server. For example："{"阿里巴巴":20,"通义实验室":30}" `itn`: whether to use itn, the default value is true for enabling and false for disabling. ``` @@ -59,7 +59,7 @@ Configuration parameters and meta information are in JSON format, while audio da #### Initial Communication The message (which needs to be serialized in JSON) is: ```text -{"mode": "2pass", "wav_name": "wav_name", "is_speaking": True, "wav_format":"pcm", "chunk_size":[5,10,5],"hotwords":"阿里巴巴 达摩院 阿里云","itn":true} +{"mode": "2pass", "wav_name": "wav_name", "is_speaking": True, "wav_format":"pcm", "chunk_size":[5,10,5],"hotwords":"{"阿里巴巴":20,"通义实验室":30}","itn":true} ``` Parameter explanation: ```text @@ -69,7 +69,7 @@ Parameter explanation: `is_speaking`: False indicates the end of a sentence, such as a VAD segmentation point or the end of a WAV file `chunk_size`: indicates the latency configuration of the streaming model, `[5,10,5]` indicates that the current audio is 600ms long, with a 300ms look-ahead and look-back time. `audio_fs`: when the input audio is in PCM format, the audio sampling rate parameter needs to be added -`hotwords`：If AM is the hotword model, hotword data needs to be sent to the server in string format, with " " used as a separator between hotwords. For example："阿里巴巴 达摩院 阿里云" +`hotwords`：If using the hotword, you need to send the hotword data (string) to the server. For example："{"阿里巴巴":20,"通义实验室":30}" `itn`: whether to use itn, the default value is true for enabling and false for disabling. ``` #### Sending Audio Data diff --git a/funasr/runtime/docs/websocket_protocol_zh.md b/runtime/docs/websocket_protocol_zh.md similarity index 85% rename from funasr/runtime/docs/websocket_protocol_zh.md rename to runtime/docs/websocket_protocol_zh.md index 8d5888e80..cfba8f202 100644 --- a/funasr/runtime/docs/websocket_protocol_zh.md +++ b/runtime/docs/websocket_protocol_zh.md @@ -10,7 +10,7 @@ #### 首次通信 message为（需要用json序列化）： ```text -{"mode": "offline", "wav_name": "wav_name","wav_format":"pcm","is_speaking": True,"hotwords":"阿里巴巴 达摩院 阿里云","itn":True} +{"mode": "offline", "wav_name": "wav_name", "wav_format":"pcm", "is_speaking": True, "hotwords":"{"阿里巴巴":20,"通义实验室":30}", "itn":True} ``` 参数介绍： ```text @@ -19,9 +19,10 @@ message为（需要用json序列化）： `wav_format`：表示音视频文件后缀名，可选pcm、mp3、mp4等 `is_speaking`：False 表示断句尾点，例如，vad切割点，或者一条wav结束 `audio_fs`：当输入音频为pcm数据时，需要加上音频采样率参数 -`hotwords`：如果AM为热词模型，需要向服务端发送热词数据，格式为字符串，热词之间用" "分隔，例如 "阿里巴巴 达摩院 阿里云" +`hotwords`：如果使用热词，需要向服务端发送热词数据（字符串），格式为 "{"阿里巴巴":20,"通义实验室":30}" `itn`: 设置是否使用itn，默认True ``` +注：热词权重仅在fst热词服务下生效。 #### 发送音频数据 pcm直接将音频数据，其他格式音频数据，连同头部信息与音视频bytes数据发送，支持多种采样率与音视频格式 @@ -59,7 +60,7 @@ message为（采用json序列化） #### 首次通信 message为（需要用json序列化）： ```text -{"mode": "2pass", "wav_name": "wav_name", "is_speaking": True, "wav_format":"pcm", "chunk_size":[5,10,5], "hotwords":"阿里巴巴 达摩院 阿里云","itn":True} +{"mode": "2pass", "wav_name": "wav_name", "is_speaking": True, "wav_format":"pcm", "chunk_size":[5,10,5], "hotwords":"{"阿里巴巴":20,"通义实验室":30}","itn":True} ``` 参数介绍： ```text @@ -69,9 +70,11 @@ message为（需要用json序列化）： `is_speaking`：表示断句尾点，例如，vad切割点，或者一条wav结束 `chunk_size`：表示流式模型latency配置，`[5,10,5]`，表示当前音频为600ms，并且回看300ms，又看300ms。 `audio_fs`：当输入音频为pcm数据是，需要加上音频采样率参数 -`hotwords`：如果AM为热词模型，需要向服务端发送热词数据，格式为字符串，热词之间用" "分隔，例如 "阿里巴巴 达摩院 阿里云" +`hotwords`：如果使用热词，需要向服务端发送热词数据（字符串），格式为 "{"阿里巴巴":20,"通义实验室":30}" `itn`: 设置是否使用itn，默认True ``` +注：热词权重仅在fst热词服务下生效。 + #### 发送音频数据 直接将音频数据，移除头部信息后的bytes数据发送，支持音频采样率为8000（`message`中需要指定`audio_fs`为8000），16000 #### 发送结束标志 diff --git a/funasr/runtime/grpc/CMakeLists.txt b/runtime/grpc/CMakeLists.txt similarity index 100% rename from funasr/runtime/grpc/CMakeLists.txt rename to runtime/grpc/CMakeLists.txt diff --git a/funasr/runtime/grpc/Readme.md b/runtime/grpc/Readme.md similarity index 100% rename from funasr/runtime/grpc/Readme.md rename to runtime/grpc/Readme.md diff --git a/funasr/runtime/grpc/build.sh b/runtime/grpc/build.sh similarity index 100% rename from funasr/runtime/grpc/build.sh rename to runtime/grpc/build.sh diff --git a/funasr/runtime/grpc/common.cmake b/runtime/grpc/common.cmake similarity index 100% rename from funasr/runtime/grpc/common.cmake rename to runtime/grpc/common.cmake diff --git a/funasr/runtime/grpc/paraformer-server.cc b/runtime/grpc/paraformer-server.cc similarity index 100% rename from funasr/runtime/grpc/paraformer-server.cc rename to runtime/grpc/paraformer-server.cc diff --git a/funasr/runtime/grpc/paraformer-server.h b/runtime/grpc/paraformer-server.h similarity index 100% rename from funasr/runtime/grpc/paraformer-server.h rename to runtime/grpc/paraformer-server.h diff --git a/funasr/runtime/grpc/run_server.sh b/runtime/grpc/run_server.sh similarity index 100% rename from funasr/runtime/grpc/run_server.sh rename to runtime/grpc/run_server.sh diff --git a/funasr/runtime/html5/h5Server.py b/runtime/html5/h5Server.py similarity index 100% rename from funasr/runtime/html5/h5Server.py rename to runtime/html5/h5Server.py diff --git a/funasr/runtime/html5/readme.md b/runtime/html5/readme.md similarity index 100% rename from funasr/runtime/html5/readme.md rename to runtime/html5/readme.md diff --git a/funasr/runtime/html5/readme_zh.md b/runtime/html5/readme_zh.md similarity index 100% rename from funasr/runtime/html5/readme_zh.md rename to runtime/html5/readme_zh.md diff --git a/funasr/runtime/html5/ssl_key b/runtime/html5/ssl_key similarity index 100% rename from funasr/runtime/html5/ssl_key rename to runtime/html5/ssl_key diff --git a/funasr/runtime/html5/static/index.html b/runtime/html5/static/index.html similarity index 88% rename from funasr/runtime/html5/static/index.html rename to runtime/html5/static/index.html index 27be9080a..3aeda038d 100644 --- a/funasr/runtime/html5/static/index.html +++ b/runtime/html5/static/index.html @@ -51,13 +51,15 @@
- - 热词设置(空格隔开,如"阿里巴巴 达摩院 阿里云")： -
- - -
- +
+ 热词设置(一行一个关键字，空格隔开权重,如"阿里巴巴 20")： +
+ + + +
+ +
语音识别结果显示：
diff --git a/funasr/runtime/html5/static/main.js b/runtime/html5/static/main.js similarity index 91% rename from funasr/runtime/html5/static/main.js rename to runtime/html5/static/main.js index 38811cd41..52a8d96d8 100644 --- a/funasr/runtime/html5/static/main.js +++ b/runtime/html5/static/main.js @@ -255,15 +255,35 @@ function on_recoder_mode_change() } } + + function getHotwords(){ - var obj = document.getElementById("varHot"); + + var obj = document.getElementById("varHot"); + + if(typeof(obj) == 'undefined' || obj==null || obj.value.length<=0){ + return ""; + } + let val = obj.value.toString(); - if(typeof(obj) == 'undefined' || obj==null || obj.value.length<=0){ - return ""; - } - let val = obj.value.toString(); - console.log("hotwords="+val); - return val; + console.log("hotwords="+val); + let items = val.split(/[(\r\n)\r\n]+/); //split by \r\n + var jsonresult = {}; + const regexNum = /^[0-9]*$/; // test number + for (item of items) { + + let result = item.split(" "); + if(result.length>=2 && regexNum.test(result[result.length-1])) + { + var wordstr=""; + for(var i=0;i0) { - obj.put("hotwords", FunasrWsClient.hotwords.trim()); + String regex = "\\d+"; + JSONObject jsonitems = new JSONObject(); + String[] items=FunasrWsClient.hotwords.trim().split(" "); + Pattern pattern = Pattern.compile(regex); + String tmpWords=""; + for(int i=0;i #include #include +#include "util.h" #include "funasrruntime.h" #include "tclap/CmdLine.h" #include "com-define.h" @@ -42,16 +43,15 @@ void GetValue(TCLAP::ValueArg& value_arg, string key, std::map chunk_size, vector wav_list, vector wav_ids, - float* total_length, long* total_time, int core_id, ASR_TYPE asr_mode_, string hotwords) { + float* total_length, long* total_time, int core_id, ASR_TYPE asr_mode_, string nn_hotwords_) { struct timeval start, end; long seconds = 0; float n_total_length = 0.0f; long n_total_time = 0; - std::vector> hotwords_embedding = CompileHotwordEmbedding(tpass_handle, hotwords, ASR_TWO_PASS); + std::vector> hotwords_embedding = CompileHotwordEmbedding(tpass_handle, nn_hotwords_, ASR_TWO_PASS); // init online features FUNASR_HANDLE tpass_online_handle=FunTpassOnlineInit(tpass_handle, chunk_size); @@ -220,7 +220,7 @@ int main(int argc, char** argv) TCLAP::ValueArg onnx_thread("", "model-thread-num", "onnxruntime SetIntraOpNumThreads", false, 1, "int32_t"); TCLAP::ValueArg thread_num_("", THREAD_NUM, "multi-thread num for rtf", false, 1, "int32_t"); TCLAP::ValueArg wav_path("", WAV_PATH, "the input could be: wav_path, e.g.: asr_example.wav; pcm_path, e.g.: asr_example.pcm; wav.scp, kaldi style wav list (wav_id \t wav_path)", true, "", "string"); - TCLAP::ValueArg hotword("", HOTWORD, "*.txt(one hotword perline) or hotwords seperate by | (could be: 阿里巴巴 达摩院)", false, "", "string"); + TCLAP::ValueArg hotword("", HOTWORD, "the hotword file, one hotword perline, Format: Hotword Weight (could be: 阿里巴巴 20)", false, "", "string"); cmd.add(offline_model_dir); cmd.add(online_model_dir); @@ -276,25 +276,12 @@ int main(int argc, char** argv) long modle_init_micros = ((seconds * 1000000) + end.tv_usec) - (start.tv_usec); LOG(INFO) << "Model initialization takes " << (double)modle_init_micros / 1000000 << " s"; - // read hotwords - std::string hotword_ = hotword.getValue(); - std::string hotwords_; - - if(is_target_file(hotword_, "txt")){ - ifstream in(hotword_); - if (!in.is_open()) { - LOG(ERROR) << "Failed to open file: " << model_path.at(HOTWORD) ; - return 0; - } - string line; - while(getline(in, line)) - { - hotwords_ +=line+HOTWORD_SEP; - } - in.close(); - }else{ - hotwords_ = hotword_; - } + // hotword file + unordered_map hws_map; + std::string nn_hotwords_ = ""; + std::string hotword_path = model_path.at(HOTWORD); + LOG(INFO) << "hotword path: " << hotword_path; + funasr::ExtractHws(hotword_path, hws_map, nn_hotwords_); // read wav_path vector wav_list; @@ -332,7 +319,7 @@ int main(int argc, char** argv) int rtf_threds = thread_num_.getValue(); for (int i = 0; i < rtf_threds; i++) { - threads.emplace_back(thread(runReg, tpass_hanlde, chunk_size, wav_list, wav_ids, &total_length, &total_time, i, (ASR_TYPE)asr_mode_, hotwords_)); + threads.emplace_back(thread(runReg, tpass_hanlde, chunk_size, wav_list, wav_ids, &total_length, &total_time, i, (ASR_TYPE)asr_mode_, nn_hotwords_)); } for (auto& thread : threads) diff --git a/funasr/runtime/onnxruntime/bin/funasr-onnx-2pass.cpp b/runtime/onnxruntime/bin/funasr-onnx-2pass.cpp similarity index 93% rename from funasr/runtime/onnxruntime/bin/funasr-onnx-2pass.cpp rename to runtime/onnxruntime/bin/funasr-onnx-2pass.cpp index 59fceb786..d0154999d 100644 --- a/funasr/runtime/onnxruntime/bin/funasr-onnx-2pass.cpp +++ b/runtime/onnxruntime/bin/funasr-onnx-2pass.cpp @@ -14,6 +14,7 @@ #include #include #include +#include "util.h" #include "funasrruntime.h" #include "tclap/CmdLine.h" #include "com-define.h" @@ -54,7 +55,7 @@ int main(int argc, char** argv) TCLAP::ValueArg onnx_thread("", "model-thread-num", "onnxruntime SetIntraOpNumThreads", false, 1, "int32_t"); TCLAP::ValueArg wav_path("", WAV_PATH, "the input could be: wav_path, e.g.: asr_example.wav; pcm_path, e.g.: asr_example.pcm; wav.scp, kaldi style wav list (wav_id \t wav_path)", true, "", "string"); - TCLAP::ValueArg hotword("", HOTWORD, "*.txt(one hotword perline) or hotwords seperate by | (could be: 阿里巴巴 达摩院)", false, "", "string"); + TCLAP::ValueArg hotword("", HOTWORD, "the hotword file, one hotword perline, Format: Hotword Weight (could be: 阿里巴巴 20)", false, "", "string"); cmd.add(offline_model_dir); cmd.add(online_model_dir); @@ -109,25 +110,12 @@ int main(int argc, char** argv) long modle_init_micros = ((seconds * 1000000) + end.tv_usec) - (start.tv_usec); LOG(INFO) << "Model initialization takes " << (double)modle_init_micros / 1000000 << " s"; - // read hotwords - std::string hotword_ = hotword.getValue(); - std::string hotwords_=""; - - if(is_target_file(hotword_, "txt")){ - ifstream in(hotword_); - if (!in.is_open()) { - LOG(ERROR) << "Failed to open file: " << model_path.at(HOTWORD) ; - return 0; - } - string line; - while(getline(in, line)) - { - hotwords_ +=line+HOTWORD_SEP; - } - in.close(); - }else{ - hotwords_ = hotword_; - } + // hotword file + unordered_map hws_map; + std::string nn_hotwords_ = ""; + std::string hotword_path = model_path.at(HOTWORD); + LOG(INFO) << "hotword path: " << hotword_path; + funasr::ExtractHws(hotword_path, hws_map, nn_hotwords_); // read wav_path vector wav_list; @@ -156,7 +144,7 @@ int main(int argc, char** argv) wav_ids.emplace_back(default_id); } - std::vector> hotwords_embedding = CompileHotwordEmbedding(tpass_handle, hotwords_, ASR_TWO_PASS); + std::vector> hotwords_embedding = CompileHotwordEmbedding(tpass_handle, nn_hotwords_, ASR_TWO_PASS); // init online features std::vector chunk_size = {5,10,5}; FUNASR_HANDLE tpass_online_handle=FunTpassOnlineInit(tpass_handle, chunk_size); diff --git a/funasr/runtime/onnxruntime/bin/funasr-onnx-offline-punc.cpp b/runtime/onnxruntime/bin/funasr-onnx-offline-punc.cpp similarity index 100% rename from funasr/runtime/onnxruntime/bin/funasr-onnx-offline-punc.cpp rename to runtime/onnxruntime/bin/funasr-onnx-offline-punc.cpp diff --git a/funasr/runtime/onnxruntime/bin/funasr-onnx-offline-rtf.cpp b/runtime/onnxruntime/bin/funasr-onnx-offline-rtf.cpp similarity index 75% rename from funasr/runtime/onnxruntime/bin/funasr-onnx-offline-rtf.cpp rename to runtime/onnxruntime/bin/funasr-onnx-offline-rtf.cpp index c5092774f..41cd03834 100644 --- a/funasr/runtime/onnxruntime/bin/funasr-onnx-offline-rtf.cpp +++ b/runtime/onnxruntime/bin/funasr-onnx-offline-rtf.cpp @@ -22,25 +22,40 @@ #include #include #include - +#include +#include "util.h" using namespace std; std::atomic wav_index(0); std::mutex mtx; void runReg(FUNASR_HANDLE asr_handle, vector wav_list, vector wav_ids, - float* total_length, long* total_time, int core_id, string hotwords) { + float* total_length, long* total_time, int core_id, float glob_beam = 3.0f, float lat_beam = 3.0f, float am_sc = 10.0f, + int fst_inc_wts = 20, string hotword_path = "") { struct timeval start, end; long seconds = 0; float n_total_length = 0.0f; long n_total_time = 0; - std::vector> hotwords_embedding = CompileHotwordEmbedding(asr_handle, hotwords); + + // init wfst decoder + FUNASR_DEC_HANDLE decoder_handle = FunASRWfstDecoderInit(asr_handle, ASR_OFFLINE, glob_beam, lat_beam, am_sc); + + // process fst hotwords list + unordered_map hws_map; + string nn_hotwords_ = ""; + funasr::ExtractHws(hotword_path, hws_map, nn_hotwords_); + + // load hotwords list and build graph + FunWfstDecoderLoadHwsRes(decoder_handle, fst_inc_wts, hws_map); + + std::vector> hotwords_embedding = CompileHotwordEmbedding(asr_handle, nn_hotwords_); // warm up for (size_t i = 0; i < 1; i++) { - FUNASR_RESULT result=FunOfflineInfer(asr_handle, wav_list[0].c_str(), RASR_NONE, NULL, hotwords_embedding, 16000); + FunOfflineReset(asr_handle, decoder_handle); + FUNASR_RESULT result=FunOfflineInfer(asr_handle, wav_list[0].c_str(), RASR_NONE, NULL, hotwords_embedding, 16000, false, decoder_handle); if(result){ FunASRFreeResult(result); } @@ -54,7 +69,7 @@ void runReg(FUNASR_HANDLE asr_handle, vector wav_list, vector wa } gettimeofday(&start, NULL); - FUNASR_RESULT result=FunOfflineInfer(asr_handle, wav_list[i].c_str(), RASR_NONE, NULL, hotwords_embedding, 16000); + FUNASR_RESULT result=FunOfflineInfer(asr_handle, wav_list[i].c_str(), RASR_NONE, NULL, hotwords_embedding, 16000, false, decoder_handle); gettimeofday(&end, NULL); seconds = (end.tv_sec - start.tv_sec); @@ -82,6 +97,8 @@ void runReg(FUNASR_HANDLE asr_handle, vector wav_list, vector wa *total_time = n_total_time; } } + FunWfstDecoderUnloadHwsRes(decoder_handle); + FunASRWfstDecoderUninit(decoder_handle); } bool is_target_file(const std::string& filename, const std::string target) { @@ -113,11 +130,16 @@ int main(int argc, char *argv[]) TCLAP::ValueArg vad_quant("", VAD_QUANT, "true (Default), load the model of model.onnx in vad_dir. If set true, load the model of model_quant.onnx in vad_dir", false, "true", "string"); TCLAP::ValueArg punc_dir("", PUNC_DIR, "the punc model path, which contains model.onnx, punc.yaml", false, "", "string"); TCLAP::ValueArg punc_quant("", PUNC_QUANT, "true (Default), load the model of model.onnx in punc_dir. If set true, load the model of model_quant.onnx in punc_dir", false, "true", "string"); + TCLAP::ValueArg lm_dir("", LM_DIR, "the lm model path, which contains compiled models: TLG.fst, config.yaml ", false, "", "string"); + TCLAP::ValueArg global_beam("", GLOB_BEAM, "the decoding beam for beam searching ", false, 3.0, "float"); + TCLAP::ValueArg lattice_beam("", LAT_BEAM, "the lattice generation beam for beam searching ", false, 3.0, "float"); + TCLAP::ValueArg am_scale("", AM_SCALE, "the acoustic scale for beam searching ", false, 10.0, "float"); + TCLAP::ValueArg fst_inc_wts("", FST_INC_WTS, "the fst hotwords incremental bias", false, 20, "int32_t"); TCLAP::ValueArg itn_dir("", ITN_DIR, "the itn model(fst) path, which contains zh_itn_tagger.fst and zh_itn_verbalizer.fst", false, "", "string"); TCLAP::ValueArg wav_path("", WAV_PATH, "the input could be: wav_path, e.g.: asr_example.wav; pcm_path, e.g.: asr_example.pcm; wav.scp, kaldi style wav list (wav_id \t wav_path)", true, "", "string"); TCLAP::ValueArg thread_num("", THREAD_NUM, "multi-thread num for rtf", true, 0, "int32_t"); - TCLAP::ValueArg hotword("", HOTWORD, "*.txt(one hotword perline) or hotwords seperate by | (could be: 阿里巴巴 达摩院)", false, "", "string"); + TCLAP::ValueArg hotword("", HOTWORD, "the hotword file, one hotword perline, Format: Hotword Weight (could be: 阿里巴巴 20)", false, "", "string"); cmd.add(model_dir); cmd.add(quantize); @@ -126,9 +148,14 @@ int main(int argc, char *argv[]) cmd.add(punc_dir); cmd.add(punc_quant); cmd.add(itn_dir); + cmd.add(lm_dir); + cmd.add(global_beam); + cmd.add(lattice_beam); + cmd.add(am_scale); + cmd.add(hotword); + cmd.add(fst_inc_wts); cmd.add(wav_path); cmd.add(thread_num); - cmd.add(hotword); cmd.parse(argc, argv); std::map model_path; @@ -139,6 +166,8 @@ int main(int argc, char *argv[]) GetValue(punc_dir, PUNC_DIR, model_path); GetValue(punc_quant, PUNC_QUANT, model_path); GetValue(itn_dir, ITN_DIR, model_path); + GetValue(lm_dir, LM_DIR, model_path); + GetValue(hotword, HOTWORD, model_path); GetValue(wav_path, WAV_PATH, model_path); struct timeval start, end; @@ -156,26 +185,6 @@ int main(int argc, char *argv[]) long modle_init_micros = ((seconds * 1000000) + end.tv_usec) - (start.tv_usec); LOG(INFO) << "Model initialization takes " << (double)modle_init_micros / 1000000 << " s"; - // read hotwords - std::string hotword_ = hotword.getValue(); - std::string hotwords_; - - if(is_target_file(hotword_, "txt")){ - ifstream in(hotword_); - if (!in.is_open()) { - LOG(ERROR) << "Failed to open file: " << model_path.at(HOTWORD) ; - return 0; - } - string line; - while(getline(in, line)) - { - hotwords_ +=line+HOTWORD_SEP; - } - in.close(); - }else{ - hotwords_ = hotword_; - } - // read wav_path vector wav_list; vector wav_ids; @@ -212,9 +221,20 @@ int main(int argc, char *argv[]) std::vector threads; int rtf_threds = thread_num.getValue(); + std::string hotword_path = hotword.getValue(); + int value_bias = 20; + value_bias = fst_inc_wts.getValue(); + float glob_beam = 3.0f; + float lat_beam = 3.0f; + float am_sc = 10.0f; + if (lm_dir.isSet()) { + glob_beam = global_beam.getValue(); + lat_beam = lattice_beam.getValue(); + am_sc = am_scale.getValue(); + } for (int i = 0; i < rtf_threds; i++) { - threads.emplace_back(thread(runReg, asr_handle, wav_list, wav_ids, &total_length, &total_time, i, hotwords_)); + threads.emplace_back(thread(runReg, asr_handle, wav_list, wav_ids, &total_length, &total_time, i, glob_beam, lat_beam, am_sc, value_bias, hotword_path)); } for (auto& thread : threads) diff --git a/funasr/runtime/onnxruntime/bin/funasr-onnx-offline-vad.cpp b/runtime/onnxruntime/bin/funasr-onnx-offline-vad.cpp similarity index 100% rename from funasr/runtime/onnxruntime/bin/funasr-onnx-offline-vad.cpp rename to runtime/onnxruntime/bin/funasr-onnx-offline-vad.cpp diff --git a/funasr/runtime/onnxruntime/bin/funasr-onnx-offline.cpp b/runtime/onnxruntime/bin/funasr-onnx-offline.cpp similarity index 71% rename from funasr/runtime/onnxruntime/bin/funasr-onnx-offline.cpp rename to runtime/onnxruntime/bin/funasr-onnx-offline.cpp index 8b0f9b593..67a267d0d 100644 --- a/funasr/runtime/onnxruntime/bin/funasr-onnx-offline.cpp +++ b/runtime/onnxruntime/bin/funasr-onnx-offline.cpp @@ -17,7 +17,8 @@ #include "funasrruntime.h" #include "tclap/CmdLine.h" #include "com-define.h" - +#include +#include "util.h" using namespace std; bool is_target_file(const std::string& filename, const std::string target) { @@ -49,10 +50,14 @@ int main(int argc, char** argv) TCLAP::ValueArg vad_quant("", VAD_QUANT, "true (Default), load the model of model.onnx in vad_dir. If set true, load the model of model_quant.onnx in vad_dir", false, "true", "string"); TCLAP::ValueArg punc_dir("", PUNC_DIR, "the punc model path, which contains model.onnx, punc.yaml", false, "", "string"); TCLAP::ValueArg punc_quant("", PUNC_QUANT, "true (Default), load the model of model.onnx in punc_dir. If set true, load the model of model_quant.onnx in punc_dir", false, "true", "string"); + TCLAP::ValueArg lm_dir("", LM_DIR, "the lm model path, which contains compiled models: TLG.fst, config.yaml ", false, "", "string"); + TCLAP::ValueArg global_beam("", GLOB_BEAM, "the decoding beam for beam searching ", false, 3.0, "float"); + TCLAP::ValueArg lattice_beam("", LAT_BEAM, "the lattice generation beam for beam searching ", false, 3.0, "float"); + TCLAP::ValueArg am_scale("", AM_SCALE, "the acoustic scale for beam searching ", false, 10.0, "float"); + TCLAP::ValueArg fst_inc_wts("", FST_INC_WTS, "the fst hotwords incremental bias", false, 20, "int32_t"); TCLAP::ValueArg itn_dir("", ITN_DIR, "the itn model(fst) path, which contains zh_itn_tagger.fst and zh_itn_verbalizer.fst", false, "", "string"); - - TCLAP::ValueArg wav_path("", WAV_PATH, "the input could be: wav_path, e.g.: asr_example.wav; pcm_path, e.g.: asr_example.pcm; wav.scp, kaldi style wav list (wav_id \t wav_path)", true, "", "string"); - TCLAP::ValueArg hotword("", HOTWORD, "*.txt(one hotword perline) or hotwords seperate by space (could be: 阿里巴巴 达摩院)", false, "", "string"); + TCLAP::ValueArg wav_path("", WAV_PATH, "the input could be: wav_path, e.g.: asr_example.wav; pcm_path, e.g.: asr_example.pcm; wav.scp, kaldi style wav list (wav_id \t wav_path)", true, "", "string"); + TCLAP::ValueArg hotword("", HOTWORD, "the hotword file, one hotword perline, Format: Hotword Weight (could be: 阿里巴巴 20)", false, "", "string"); cmd.add(model_dir); cmd.add(quantize); @@ -61,6 +66,11 @@ int main(int argc, char** argv) cmd.add(punc_dir); cmd.add(punc_quant); cmd.add(itn_dir); + cmd.add(lm_dir); + cmd.add(global_beam); + cmd.add(lattice_beam); + cmd.add(am_scale); + cmd.add(fst_inc_wts); cmd.add(wav_path); cmd.add(hotword); cmd.parse(argc, argv); @@ -73,6 +83,7 @@ int main(int argc, char** argv) GetValue(punc_dir, PUNC_DIR, model_path); GetValue(punc_quant, PUNC_QUANT, model_path); GetValue(itn_dir, ITN_DIR, model_path); + GetValue(lm_dir, LM_DIR, model_path); GetValue(wav_path, WAV_PATH, model_path); struct timeval start, end; @@ -85,32 +96,29 @@ int main(int argc, char** argv) LOG(ERROR) << "FunASR init failed"; exit(-1); } + float glob_beam = 3.0f; + float lat_beam = 3.0f; + float am_sc = 10.0f; + if (lm_dir.isSet()) { + glob_beam = global_beam.getValue(); + lat_beam = lattice_beam.getValue(); + am_sc = am_scale.getValue(); + } + // init wfst decoder + FUNASR_DEC_HANDLE decoder_handle = FunASRWfstDecoderInit(asr_hanlde, ASR_OFFLINE, glob_beam, lat_beam, am_sc); + + // hotword file + unordered_map hws_map; + std::string nn_hotwords_ = ""; + std::string hotword_path = hotword.getValue(); + LOG(INFO) << "hotword path: " << hotword_path; + funasr::ExtractHws(hotword_path, hws_map, nn_hotwords_); gettimeofday(&end, NULL); long seconds = (end.tv_sec - start.tv_sec); long modle_init_micros = ((seconds * 1000000) + end.tv_usec) - (start.tv_usec); LOG(INFO) << "Model initialization takes " << (double)modle_init_micros / 1000000 << " s"; - // read hotwords - std::string hotword_ = hotword.getValue(); - std::string hotwords_; - - if(is_target_file(hotword_, "txt")){ - ifstream in(hotword_); - if (!in.is_open()) { - LOG(ERROR) << "Failed to open file: " << model_path.at(HOTWORD) ; - return 0; - } - string line; - while(getline(in, line)) - { - hotwords_ +=line+HOTWORD_SEP; - } - in.close(); - }else{ - hotwords_ = hotword_; - } - // read wav_path vector wav_list; vector wav_ids; @@ -140,12 +148,16 @@ int main(int argc, char** argv) float snippet_time = 0.0f; long taking_micros = 0; - std::vector> hotwords_embedding = CompileHotwordEmbedding(asr_hanlde, hotwords_); + + // load hotwords list and build graph + FunWfstDecoderLoadHwsRes(decoder_handle, fst_inc_wts.getValue(), hws_map); + + std::vector> hotwords_embedding = CompileHotwordEmbedding(asr_hanlde, nn_hotwords_); for (int i = 0; i < wav_list.size(); i++) { auto& wav_file = wav_list[i]; auto& wav_id = wav_ids[i]; gettimeofday(&start, NULL); - FUNASR_RESULT result=FunOfflineInfer(asr_hanlde, wav_file.c_str(), RASR_NONE, NULL, hotwords_embedding, 16000); + FUNASR_RESULT result=FunOfflineInfer(asr_hanlde, wav_file.c_str(), RASR_NONE, NULL, hotwords_embedding, 16000, false, decoder_handle); gettimeofday(&end, NULL); seconds = (end.tv_sec - start.tv_sec); taking_micros += ((seconds * 1000000) + end.tv_usec) - (start.tv_usec); @@ -166,10 +178,12 @@ int main(int argc, char** argv) LOG(ERROR) << ("No return data!\n"); } } - + FunWfstDecoderUnloadHwsRes(decoder_handle); LOG(INFO) << "Audio length: " << (double)snippet_time << " s"; LOG(INFO) << "Model inference takes: " << (double)taking_micros / 1000000 <<" s"; LOG(INFO) << "Model inference RTF: " << (double)taking_micros/ (snippet_time*1000000); + + FunASRWfstDecoderUninit(decoder_handle); FunOfflineUninit(asr_hanlde); return 0; } diff --git a/funasr/runtime/onnxruntime/bin/funasr-onnx-online-asr.cpp b/runtime/onnxruntime/bin/funasr-onnx-online-asr.cpp similarity index 100% rename from funasr/runtime/onnxruntime/bin/funasr-onnx-online-asr.cpp rename to runtime/onnxruntime/bin/funasr-onnx-online-asr.cpp diff --git a/funasr/runtime/onnxruntime/bin/funasr-onnx-online-punc.cpp b/runtime/onnxruntime/bin/funasr-onnx-online-punc.cpp similarity index 100% rename from funasr/runtime/onnxruntime/bin/funasr-onnx-online-punc.cpp rename to runtime/onnxruntime/bin/funasr-onnx-online-punc.cpp diff --git a/funasr/runtime/onnxruntime/bin/funasr-onnx-online-rtf.cpp b/runtime/onnxruntime/bin/funasr-onnx-online-rtf.cpp similarity index 100% rename from funasr/runtime/onnxruntime/bin/funasr-onnx-online-rtf.cpp rename to runtime/onnxruntime/bin/funasr-onnx-online-rtf.cpp diff --git a/funasr/runtime/onnxruntime/bin/funasr-onnx-online-vad.cpp b/runtime/onnxruntime/bin/funasr-onnx-online-vad.cpp similarity index 100% rename from funasr/runtime/onnxruntime/bin/funasr-onnx-online-vad.cpp rename to runtime/onnxruntime/bin/funasr-onnx-online-vad.cpp diff --git a/funasr/runtime/onnxruntime/include/audio.h b/runtime/onnxruntime/include/audio.h similarity index 100% rename from funasr/runtime/onnxruntime/include/audio.h rename to runtime/onnxruntime/include/audio.h diff --git a/funasr/runtime/onnxruntime/include/com-define.h b/runtime/onnxruntime/include/com-define.h similarity index 89% rename from funasr/runtime/onnxruntime/include/com-define.h rename to runtime/onnxruntime/include/com-define.h index 0ecb9c2b6..57908e6f2 100644 --- a/funasr/runtime/onnxruntime/include/com-define.h +++ b/runtime/onnxruntime/include/com-define.h @@ -15,6 +15,12 @@ namespace funasr { #define MODEL_DIR "model-dir" #define OFFLINE_MODEL_DIR "model-dir" #define ONLINE_MODEL_DIR "online-model-dir" +#define LM_DIR "lm-dir" +#define GLOB_BEAM "global-beam" +#define LAT_BEAM "lattice-beam" +#define AM_SCALE "am-scale" +// #define FST_HOTWORD "fst-hotword" +#define FST_INC_WTS "fst-inc-wts" #define VAD_DIR "vad-dir" #define PUNC_DIR "punc-dir" #define QUANTIZE "quantize" @@ -46,9 +52,11 @@ namespace funasr { #define VAD_CONFIG_NAME "vad.yaml" #define AM_CMVN_NAME "am.mvn" #define AM_CONFIG_NAME "config.yaml" +#define LM_CONFIG_NAME "config.yaml" #define PUNC_CONFIG_NAME "punc.yaml" #define MODEL_SEG_DICT "seg_dict" #define HOTWORD "hotword" +// #define NN_HOTWORD "nn-hotword" #define ITN_DIR "itn-dir" #define ITN_TAGGER_NAME "zh_itn_tagger.fst" @@ -59,6 +67,8 @@ namespace funasr { #define DECODER_NAME "decoder.onnx" #define QUANT_DECODER_NAME "decoder_quant.onnx" +#define LM_FST_RES "TLG.fst" + // vad #ifndef VAD_SILENCE_DURATION #define VAD_SILENCE_DURATION 800 diff --git a/funasr/runtime/onnxruntime/include/funasrruntime.h b/runtime/onnxruntime/include/funasrruntime.h similarity index 86% rename from funasr/runtime/onnxruntime/include/funasrruntime.h rename to runtime/onnxruntime/include/funasrruntime.h index 3b5ae8f0d..3b52f38d0 100644 --- a/funasr/runtime/onnxruntime/include/funasrruntime.h +++ b/runtime/onnxruntime/include/funasrruntime.h @@ -1,7 +1,7 @@ #pragma once #include #include - +#include #ifdef WIN32 #ifdef _FUNASR_API_EXPORT #define _FUNASRAPI __declspec(dllexport) @@ -21,6 +21,7 @@ typedef void* FUNASR_HANDLE; typedef void* FUNASR_RESULT; +typedef void* FUNASR_DEC_HANDLE; typedef unsigned char FUNASR_BOOL; #define FUNASR_TRUE 1 @@ -58,6 +59,8 @@ typedef void (* QM_CALLBACK)(int cur_step, int n_total); // n_total: total steps // ASR _FUNASRAPI FUNASR_HANDLE FunASRInit(std::map& model_path, int thread_num, ASR_TYPE type=ASR_OFFLINE); _FUNASRAPI FUNASR_HANDLE FunASROnlineInit(FUNASR_HANDLE asr_handle, std::vector chunk_size={5,10,5}); +_FUNASRAPI void FunASRReset(FUNASR_HANDLE handle, FUNASR_DEC_HANDLE dec_handle=nullptr); + // buffer _FUNASRAPI FUNASR_RESULT FunASRInferBuffer(FUNASR_HANDLE handle, const char* sz_buf, int n_len, FUNASR_MODE mode, QM_CALLBACK fn_callback, bool input_finished=true, int sampling_rate=16000, std::string wav_format="pcm"); // file, support wav & pcm @@ -93,14 +96,15 @@ _FUNASRAPI void CTTransformerUninit(FUNASR_HANDLE handle); //OfflineStream _FUNASRAPI FUNASR_HANDLE FunOfflineInit(std::map& model_path, int thread_num); +_FUNASRAPI void FunOfflineReset(FUNASR_HANDLE handle, FUNASR_DEC_HANDLE dec_handle=nullptr); // buffer _FUNASRAPI FUNASR_RESULT FunOfflineInferBuffer(FUNASR_HANDLE handle, const char* sz_buf, int n_len, FUNASR_MODE mode, QM_CALLBACK fn_callback, const std::vector> &hw_emb, - int sampling_rate=16000, std::string wav_format="pcm", bool itn=true); + int sampling_rate=16000, std::string wav_format="pcm", bool itn=true, FUNASR_DEC_HANDLE dec_handle=nullptr); // file, support wav & pcm _FUNASRAPI FUNASR_RESULT FunOfflineInfer(FUNASR_HANDLE handle, const char* sz_filename, FUNASR_MODE mode, QM_CALLBACK fn_callback, const std::vector> &hw_emb, - int sampling_rate=16000, bool itn=true); + int sampling_rate=16000, bool itn=true, FUNASR_DEC_HANDLE dec_handle=nullptr); #if !defined(__APPLE__) _FUNASRAPI const std::vector> CompileHotwordEmbedding(FUNASR_HANDLE handle, std::string &hotwords, ASR_TYPE mode=ASR_OFFLINE); #endif @@ -118,4 +122,9 @@ _FUNASRAPI FUNASR_RESULT FunTpassInferBuffer(FUNASR_HANDLE handle, FUNASR_HANDLE _FUNASRAPI void FunTpassUninit(FUNASR_HANDLE handle); _FUNASRAPI void FunTpassOnlineUninit(FUNASR_HANDLE handle); +// wfst decoder +_FUNASRAPI FUNASR_DEC_HANDLE FunASRWfstDecoderInit(FUNASR_HANDLE handle, int asr_type, float glob_beam, float lat_beam, float am_scale); +_FUNASRAPI void FunASRWfstDecoderUninit(FUNASR_DEC_HANDLE handle); +_FUNASRAPI void FunWfstDecoderLoadHwsRes(FUNASR_DEC_HANDLE handle, int inc_bias, std::unordered_map &hws_map); +_FUNASRAPI void FunWfstDecoderUnloadHwsRes(FUNASR_DEC_HANDLE handle); diff --git a/funasr/runtime/onnxruntime/include/itn-model.h b/runtime/onnxruntime/include/itn-model.h similarity index 100% rename from funasr/runtime/onnxruntime/include/itn-model.h rename to runtime/onnxruntime/include/itn-model.h diff --git a/funasr/runtime/onnxruntime/include/model.h b/runtime/onnxruntime/include/model.h similarity index 81% rename from funasr/runtime/onnxruntime/include/model.h rename to runtime/onnxruntime/include/model.h index 7f1e0acb3..356fca330 100644 --- a/funasr/runtime/onnxruntime/include/model.h +++ b/runtime/onnxruntime/include/model.h @@ -9,11 +9,15 @@ namespace funasr { class Model { public: virtual ~Model(){}; + virtual void StartUtterance() = 0; + virtual void EndUtterance() = 0; virtual void Reset() = 0; virtual void InitAsr(const std::string &am_model, const std::string &am_cmvn, const std::string &am_config, int thread_num){}; virtual void InitAsr(const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config, int thread_num){}; virtual void InitAsr(const std::string &am_model, const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config, int thread_num){}; - virtual std::string Forward(float *din, int len, bool input_finished, const std::vector> &hw_emb={{0.0}}){return "";}; + virtual void InitLm(const std::string &lm_file, const std::string &lm_config){}; + virtual void InitFstDecoder(){}; + virtual std::string Forward(float *din, int len, bool input_finished, const std::vector> &hw_emb={{0.0}}, void* wfst_decoder=nullptr){return "";}; virtual std::string Rescoring() = 0; virtual void InitHwCompiler(const std::string &hw_model, int thread_num){}; virtual void InitSegDict(const std::string &seg_dict_model){}; diff --git a/funasr/runtime/onnxruntime/include/offline-stream.h b/runtime/onnxruntime/include/offline-stream.h similarity index 100% rename from funasr/runtime/onnxruntime/include/offline-stream.h rename to runtime/onnxruntime/include/offline-stream.h diff --git a/funasr/runtime/onnxruntime/include/punc-model.h b/runtime/onnxruntime/include/punc-model.h similarity index 100% rename from funasr/runtime/onnxruntime/include/punc-model.h rename to runtime/onnxruntime/include/punc-model.h diff --git a/funasr/runtime/onnxruntime/include/tclap/Arg.h b/runtime/onnxruntime/include/tclap/Arg.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tclap/Arg.h rename to runtime/onnxruntime/include/tclap/Arg.h diff --git a/funasr/runtime/onnxruntime/include/tclap/ArgException.h b/runtime/onnxruntime/include/tclap/ArgException.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tclap/ArgException.h rename to runtime/onnxruntime/include/tclap/ArgException.h diff --git a/funasr/runtime/onnxruntime/include/tclap/ArgTraits.h b/runtime/onnxruntime/include/tclap/ArgTraits.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tclap/ArgTraits.h rename to runtime/onnxruntime/include/tclap/ArgTraits.h diff --git a/funasr/runtime/onnxruntime/include/tclap/CmdLine.h b/runtime/onnxruntime/include/tclap/CmdLine.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tclap/CmdLine.h rename to runtime/onnxruntime/include/tclap/CmdLine.h diff --git a/funasr/runtime/onnxruntime/include/tclap/CmdLineInterface.h b/runtime/onnxruntime/include/tclap/CmdLineInterface.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tclap/CmdLineInterface.h rename to runtime/onnxruntime/include/tclap/CmdLineInterface.h diff --git a/funasr/runtime/onnxruntime/include/tclap/CmdLineOutput.h b/runtime/onnxruntime/include/tclap/CmdLineOutput.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tclap/CmdLineOutput.h rename to runtime/onnxruntime/include/tclap/CmdLineOutput.h diff --git a/funasr/runtime/onnxruntime/include/tclap/Constraint.h b/runtime/onnxruntime/include/tclap/Constraint.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tclap/Constraint.h rename to runtime/onnxruntime/include/tclap/Constraint.h diff --git a/funasr/runtime/onnxruntime/include/tclap/DocBookOutput.h b/runtime/onnxruntime/include/tclap/DocBookOutput.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tclap/DocBookOutput.h rename to runtime/onnxruntime/include/tclap/DocBookOutput.h diff --git a/funasr/runtime/onnxruntime/include/tclap/HelpVisitor.h b/runtime/onnxruntime/include/tclap/HelpVisitor.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tclap/HelpVisitor.h rename to runtime/onnxruntime/include/tclap/HelpVisitor.h diff --git a/funasr/runtime/onnxruntime/include/tclap/IgnoreRestVisitor.h b/runtime/onnxruntime/include/tclap/IgnoreRestVisitor.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tclap/IgnoreRestVisitor.h rename to runtime/onnxruntime/include/tclap/IgnoreRestVisitor.h diff --git a/funasr/runtime/onnxruntime/include/tclap/MultiArg.h b/runtime/onnxruntime/include/tclap/MultiArg.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tclap/MultiArg.h rename to runtime/onnxruntime/include/tclap/MultiArg.h diff --git a/funasr/runtime/onnxruntime/include/tclap/MultiSwitchArg.h b/runtime/onnxruntime/include/tclap/MultiSwitchArg.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tclap/MultiSwitchArg.h rename to runtime/onnxruntime/include/tclap/MultiSwitchArg.h diff --git a/funasr/runtime/onnxruntime/include/tclap/OptionalUnlabeledTracker.h b/runtime/onnxruntime/include/tclap/OptionalUnlabeledTracker.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tclap/OptionalUnlabeledTracker.h rename to runtime/onnxruntime/include/tclap/OptionalUnlabeledTracker.h diff --git a/funasr/runtime/onnxruntime/include/tclap/StandardTraits.h b/runtime/onnxruntime/include/tclap/StandardTraits.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tclap/StandardTraits.h rename to runtime/onnxruntime/include/tclap/StandardTraits.h diff --git a/funasr/runtime/onnxruntime/include/tclap/StdOutput.h b/runtime/onnxruntime/include/tclap/StdOutput.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tclap/StdOutput.h rename to runtime/onnxruntime/include/tclap/StdOutput.h diff --git a/funasr/runtime/onnxruntime/include/tclap/SwitchArg.h b/runtime/onnxruntime/include/tclap/SwitchArg.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tclap/SwitchArg.h rename to runtime/onnxruntime/include/tclap/SwitchArg.h diff --git a/funasr/runtime/onnxruntime/include/tclap/UnlabeledMultiArg.h b/runtime/onnxruntime/include/tclap/UnlabeledMultiArg.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tclap/UnlabeledMultiArg.h rename to runtime/onnxruntime/include/tclap/UnlabeledMultiArg.h diff --git a/funasr/runtime/onnxruntime/include/tclap/UnlabeledValueArg.h b/runtime/onnxruntime/include/tclap/UnlabeledValueArg.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tclap/UnlabeledValueArg.h rename to runtime/onnxruntime/include/tclap/UnlabeledValueArg.h diff --git a/funasr/runtime/onnxruntime/include/tclap/ValueArg.h b/runtime/onnxruntime/include/tclap/ValueArg.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tclap/ValueArg.h rename to runtime/onnxruntime/include/tclap/ValueArg.h diff --git a/funasr/runtime/onnxruntime/include/tclap/ValuesConstraint.h b/runtime/onnxruntime/include/tclap/ValuesConstraint.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tclap/ValuesConstraint.h rename to runtime/onnxruntime/include/tclap/ValuesConstraint.h diff --git a/funasr/runtime/onnxruntime/include/tclap/VersionVisitor.h b/runtime/onnxruntime/include/tclap/VersionVisitor.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tclap/VersionVisitor.h rename to runtime/onnxruntime/include/tclap/VersionVisitor.h diff --git a/funasr/runtime/onnxruntime/include/tclap/Visitor.h b/runtime/onnxruntime/include/tclap/Visitor.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tclap/Visitor.h rename to runtime/onnxruntime/include/tclap/Visitor.h diff --git a/funasr/runtime/onnxruntime/include/tclap/XorHandler.h b/runtime/onnxruntime/include/tclap/XorHandler.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tclap/XorHandler.h rename to runtime/onnxruntime/include/tclap/XorHandler.h diff --git a/funasr/runtime/onnxruntime/include/tclap/ZshCompletionOutput.h b/runtime/onnxruntime/include/tclap/ZshCompletionOutput.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tclap/ZshCompletionOutput.h rename to runtime/onnxruntime/include/tclap/ZshCompletionOutput.h diff --git a/funasr/runtime/onnxruntime/include/tclap/sstream.h b/runtime/onnxruntime/include/tclap/sstream.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tclap/sstream.h rename to runtime/onnxruntime/include/tclap/sstream.h diff --git a/funasr/runtime/onnxruntime/include/tpass-online-stream.h b/runtime/onnxruntime/include/tpass-online-stream.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tpass-online-stream.h rename to runtime/onnxruntime/include/tpass-online-stream.h diff --git a/funasr/runtime/onnxruntime/include/tpass-stream.h b/runtime/onnxruntime/include/tpass-stream.h similarity index 100% rename from funasr/runtime/onnxruntime/include/tpass-stream.h rename to runtime/onnxruntime/include/tpass-stream.h diff --git a/funasr/runtime/onnxruntime/include/vad-model.h b/runtime/onnxruntime/include/vad-model.h similarity index 100% rename from funasr/runtime/onnxruntime/include/vad-model.h rename to runtime/onnxruntime/include/vad-model.h diff --git a/funasr/runtime/onnxruntime/readme.md b/runtime/onnxruntime/readme.md similarity index 92% rename from funasr/runtime/onnxruntime/readme.md rename to runtime/onnxruntime/readme.md index 6735a7c45..d63010fb6 100644 --- a/funasr/runtime/onnxruntime/readme.md +++ b/runtime/onnxruntime/readme.md @@ -1,4 +1,4 @@ -# Please ref to [websocket service](https://github.com/alibaba-damo-academy/FunASR/tree/main/funasr/runtime/websocket) +# Please ref to [websocket service](https://github.com/alibaba-damo-academy/FunASR/tree/main/runtime/websocket) # If you want to compile the file yourself, you can follow the steps below. ## Building for Linux/Unix @@ -27,7 +27,7 @@ apt-get install libssl-dev #ubuntu ### Build runtime ```shell -git clone https://github.com/alibaba-damo-academy/FunASR.git && cd FunASR/funasr/runtime/onnxruntime +git clone https://github.com/alibaba-damo-academy/FunASR.git && cd FunASR/runtime/onnxruntime mkdir build && cd build cmake -DCMAKE_BUILD_TYPE=release .. -DONNXRUNTIME_DIR=/path/to/onnxruntime-linux-x64-1.14.0 -DFFMPEG_DIR=/path/to/ffmpeg-N-111383-g20b8688092-linux64-gpl-shared make -j 4 diff --git a/funasr/runtime/onnxruntime/src/CMakeLists.txt b/runtime/onnxruntime/src/CMakeLists.txt similarity index 76% rename from funasr/runtime/onnxruntime/src/CMakeLists.txt rename to runtime/onnxruntime/src/CMakeLists.txt index 4e8ac8ad2..042be49d5 100644 --- a/funasr/runtime/onnxruntime/src/CMakeLists.txt +++ b/runtime/onnxruntime/src/CMakeLists.txt @@ -7,7 +7,7 @@ message("files: "${files}) add_library(funasr SHARED ${files}) if(WIN32) - set(EXTRA_LIBS pthread yaml-cpp csrc fst glog gflags) + set(EXTRA_LIBS pthread yaml-cpp csrc kaldi-decoder fst glog gflags) if(CMAKE_CL_64) target_link_directories(funasr PUBLIC ${CMAKE_SOURCE_DIR}/win/lib/x64) else() @@ -17,11 +17,12 @@ if(WIN32) target_compile_definitions(funasr PUBLIC -D_FUNASR_API_EXPORT) else() - set(EXTRA_LIBS pthread yaml-cpp csrc fst glog gflags avutil avcodec avformat swresample) + set(EXTRA_LIBS pthread yaml-cpp csrc kaldi-decoder fst glog gflags avutil avcodec avformat swresample) include_directories(${ONNXRUNTIME_DIR}/include) include_directories(${FFMPEG_DIR}/include) endif() #message("CXX_FLAGS "${CMAKE_CXX_FLAGS}) include_directories(${CMAKE_SOURCE_DIR}/include) +include_directories(${CMAKE_SOURCE_DIR}/third_party) target_link_libraries(funasr PUBLIC onnxruntime ${EXTRA_LIBS}) diff --git a/funasr/runtime/onnxruntime/src/alignedmem.cpp b/runtime/onnxruntime/src/alignedmem.cpp similarity index 100% rename from funasr/runtime/onnxruntime/src/alignedmem.cpp rename to runtime/onnxruntime/src/alignedmem.cpp diff --git a/funasr/runtime/onnxruntime/src/alignedmem.h b/runtime/onnxruntime/src/alignedmem.h similarity index 100% rename from funasr/runtime/onnxruntime/src/alignedmem.h rename to runtime/onnxruntime/src/alignedmem.h diff --git a/funasr/runtime/onnxruntime/src/audio.cpp b/runtime/onnxruntime/src/audio.cpp similarity index 100% rename from funasr/runtime/onnxruntime/src/audio.cpp rename to runtime/onnxruntime/src/audio.cpp diff --git a/runtime/onnxruntime/src/bias-lm.cpp b/runtime/onnxruntime/src/bias-lm.cpp new file mode 100644 index 000000000..5be87f82b --- /dev/null +++ b/runtime/onnxruntime/src/bias-lm.cpp @@ -0,0 +1,178 @@ +#include "bias-lm.h" + +namespace funasr { +void print(std::queue &q) { + std::queue data = q; + while (!data.empty()) + { + cout << data.front() << " "; + data.pop(); + } + cout << endl; +} + +void BiasLm::LoadCfgFromYaml(const char* filename, BiasLmOption &opt) { + YAML::Node config; + try { + config = YAML::LoadFile(filename); + } catch(exception const &e) { + LOG(INFO) << "Error loading file, yaml file error or not exist."; + exit(-1); + } + try { + YAML::Node bias_lm_conf = config["bias_lm_conf"]; + opt_.incre_bias_ = bias_lm_conf["increment_weight"].as(); + } catch(exception const &e) { + } +} + +void BiasLm::BuildGraph(std::vector> &split_id_vec, + std::vector &custom_weight) { + if (split_id_vec.empty()) { + LOG(INFO) << "Skip building biaslm graph, hotword not exits."; + return ; + } + assert(split_id_vec.size() == custom_weight.size()); + // Build prefix tree + std::unique_ptr prefix_tree(new fst::StdVectorFst()); + StateId start_state = prefix_tree->AddState(); + prefix_tree->SetStart(start_state); + int id = 0; + for (auto& x : split_id_vec) { + StateId state = start_state; + StateId next_state = state; + float w = custom_weight[id++]; + std::vector split_id = x; + for (int j = 0; j < split_id.size(); j++) { + next_state = prefix_tree->AddState(); + if (j == split_id.size() - 1) { + prefix_tree->SetFinal(next_state, w); + } + prefix_tree->AddArc(state, Arc(split_id[j], split_id[j], opt_.incre_bias_, next_state)); + state = next_state; + } + } + graph_ = std::unique_ptr(new fst::StdVectorFst()); + fst::Determinize(*prefix_tree, graph_.get()); + + int num_node = graph_->NumStates(); + node_list_.resize(num_node); + for (auto& x : split_id_vec) { + StateId cur_state = 0; + StateId next_state = 0; + std::vector split_id = x; + for (int j = 0; j < split_id.size(); j++) { + Matcher matcher(*graph_, fst::MATCH_INPUT); + matcher.SetState(cur_state); + if (matcher.Find(split_id[j])) { + next_state = matcher.Value().nextstate; + if (graph_->Final(next_state) != Weight::Zero()) { + node_list_[next_state].is_final_ = true; + } + node_list_[next_state].score_ = opt_.incre_bias_ * (j + 1); + cur_state = next_state; + } + } + } + + // Build Aho-Corasick Automata + std::queue q; + Matcher matcher(*graph_, fst::MATCH_INPUT); + // Back off state of all child nodes of the root node points to the root node + for (ArcIterator aiter(*graph_, start_state); !aiter.Done(); aiter.Next()) { + const Arc& arc = aiter.Value(); + node_list_[arc.nextstate].back_off_ = start_state; + float back_off_score = (node_list_[arc.nextstate].is_final_ ? 0 : + node_list_[start_state].score_ - node_list_[arc.nextstate].score_); + graph_->AddArc(arc.nextstate, Arc(0, 0, back_off_score, start_state)); + q.push(arc.nextstate); + } + while (!q.empty()) { + StateId state_id = q.front(); + q.pop(); + for (ArcIterator aiter(*graph_, state_id); !aiter.Done(); aiter.Next()) { + const Arc& arc = aiter.Value(); + StateId next_state = arc.nextstate; + StateId temp_state = node_list_[state_id].back_off_; + if (next_state == start_state || next_state == temp_state) { + continue; + } + while (true) { + matcher.SetState(temp_state); + if (matcher.Find(arc.ilabel)) { + node_list_[next_state].back_off_ = matcher.Value().nextstate; + break; + } else if (temp_state == start_state) { + node_list_[next_state].back_off_ = start_state; + break; + } + temp_state = node_list_[temp_state].back_off_; + } + float back_off_score = (node_list_[next_state].is_final_ ? 0 : + node_list_[node_list_[next_state].back_off_].score_ - + node_list_[next_state].score_); + graph_->AddArc(next_state, Arc(0, 0, back_off_score, + node_list_[next_state].back_off_)); + q.push(next_state); + } + } + fst::ArcSort(graph_.get(), fst::StdILabelCompare()); + //graph_->Write("graph.final.fst"); +} + +float BiasLm::BiasLmScore(const StateId &his_state, const Label &lab, Label &new_state) { + if (lab < 1 || lab > phn_set_.Size() || !graph_) { return VALUE_ZERO; } + StateId cur_state = his_state; + StateId next_state; + float score = VALUE_ZERO; + Matcher matcher(*graph_, fst::MATCH_INPUT); + while (true) { + StateId prev_state = cur_state; + matcher.SetState(cur_state); + if (matcher.Find(lab)) { + next_state = matcher.Value().nextstate; + score += matcher.Value().weight.Value(); + if (node_list_[next_state].is_final_) { + score = score + graph_->Final(next_state).Value(); + } + cur_state = next_state; + break; + } else { + ArcIterator aiter(*graph_, cur_state); + const Arc& arc = aiter.Value(); + if (arc.ilabel == 0) { + score += arc.weight.Value(); + next_state = arc.nextstate; + cur_state = next_state; + } + if (prev_state == ROOT_NODE && cur_state == ROOT_NODE) { + break; + } + } + } + new_state = cur_state; + return score; +} + +void BiasLm::VocabIdToPhnIdVector(int vocab_id, std::vector &phn_ids) { + bool is_oov = false; + phn_ids.clear(); + std::string word = vocab_.Id2String(vocab_id); + std::vector phn_vec; + Utf8ToCharset(word, phn_vec); + for (auto& phn : phn_vec) { + if (!phn_set_.Find(phn)) { + is_oov = true; + break; + } else { + phn_ids.push_back(phn_set_.String2Id(phn)); + } + } + if (is_oov) { phn_ids.clear(); } +} + +std::string BiasLm::GetPhoneLabel(int phone_id) { + if (phone_id < 0 || phone_id >= phn_set_.Size()) { return ""; } + return phn_set_.Id2String(phone_id); +} +} diff --git a/runtime/onnxruntime/src/bias-lm.h b/runtime/onnxruntime/src/bias-lm.h new file mode 100644 index 000000000..7254d493b --- /dev/null +++ b/runtime/onnxruntime/src/bias-lm.h @@ -0,0 +1,137 @@ +#ifndef BIAS_LM_ +#define BIAS_LM_ +#include +#include "util.h" +#include "fst/fstlib.h" +#include "phone-set.h" +#include "vocab.h" +#include "util/text-utils.h" +#include +// node type +#define ROOT_NODE 0 +#define VALUE_ZERO 0.0f + +namespace funasr { +typedef fst::StdArc Arc; +typedef typename Arc::StateId StateId; +typedef typename Arc::Weight Weight; +typedef typename Arc::Label Label; +typedef typename fst::SortedMatcher Matcher; +typedef typename fst::ArcIterator ArcIterator; + +class Node { + public: + Node() : score_(0.0f), is_final_(false), back_off_(-1) {} + float score_; + bool is_final_; + StateId back_off_; +}; + +class BiasLmOption { + public: + BiasLmOption() : incre_bias_(20.0f), scale_(1.0f) {} + float incre_bias_; + float scale_; +}; + +class BiasLm { + public: + BiasLm(const string &hws_file, const string &cfg_file, + const PhoneSet& phn_set, const Vocab& vocab) : + phn_set_(phn_set), vocab_(vocab) { + std::string line; + std::ifstream ifs_hws(hws_file.c_str()); + std::vector custom_weight; + std::vector> split_id_vec; + + struct timeval start, end; + gettimeofday(&start, NULL); + + LoadCfgFromYaml(cfg_file.c_str(), opt_); + while (getline(ifs_hws, line)) { + Trim(&line); + if (line.empty()) { + continue; + } + float score = 1.0f; + bool is_oov = false; + std::vector text; + std::vector split_str; + std::vector split_id; + SplitStringToVector(line, "\t", true, &text); + if (text.size() > 1) { + score = std::stof(text[1]); + } + Utf8ToCharset(text[0], split_str); + for (auto &str : split_str) { + split_id.push_back(phn_set_.String2Id(str)); + if (!phn_set_.Find(str)) { + is_oov = true; + break; + } + } + if (!is_oov) { + split_id_vec.push_back(split_id); + custom_weight.push_back(score); + } + } + BuildGraph(split_id_vec, custom_weight); + ifs_hws.close(); + + gettimeofday(&end, NULL); + long seconds = (end.tv_sec - start.tv_sec); + long modle_init_micros = ((seconds * 1000000) + end.tv_usec) - (start.tv_usec); + LOG(INFO) << "Build bias lm takes " << (double)modle_init_micros / 1000000 << " s"; + } + + BiasLm(unordered_map &hws_map, int inc_bias, + const PhoneSet& phn_set, const Vocab& vocab) : + phn_set_(phn_set), vocab_(vocab) { + std::vector custom_weight; + std::vector> split_id_vec; + + struct timeval start, end; + gettimeofday(&start, NULL); + opt_.incre_bias_ = inc_bias; + for (const pair& kv : hws_map) { + float score = 1.0f; + bool is_oov = false; + std::vector text; + std::vector split_str; + std::vector split_id; + score = kv.second; + Utf8ToCharset(kv.first, split_str); + for (auto &str : split_str) { + split_id.push_back(phn_set_.String2Id(str)); + if (!phn_set_.Find(str)) { + is_oov = true; + break; + } + } + if (!is_oov) { + split_id_vec.push_back(split_id); + custom_weight.push_back(score); + } + } + BuildGraph(split_id_vec, custom_weight); + + gettimeofday(&end, NULL); + long seconds = (end.tv_sec - start.tv_sec); + long modle_init_micros = ((seconds * 1000000) + end.tv_usec) - (start.tv_usec); + LOG(INFO) << "Build bias lm takes " << (double)modle_init_micros / 1000000 << " s"; + } + + void BuildGraph(std::vector> &vec, std::vector &wts); + float BiasLmScore(const StateId &cur_state, const Label &lab, Label &new_state); + void VocabIdToPhnIdVector(int vocab_id, std::vector &phn_ids); + void LoadCfgFromYaml(const char* filename, BiasLmOption &opt); + std::string GetPhoneLabel(int phone_id); + private: + const PhoneSet& phn_set_; + const Vocab& vocab_; + std::unique_ptr graph_ = nullptr; + std::vector node_list_; + BiasLmOption opt_; +}; +} // namespace funasr +#endif // BIAS_LM_ diff --git a/funasr/runtime/onnxruntime/src/common-struct.h b/runtime/onnxruntime/src/common-struct.h similarity index 100% rename from funasr/runtime/onnxruntime/src/common-struct.h rename to runtime/onnxruntime/src/common-struct.h diff --git a/funasr/runtime/onnxruntime/src/commonfunc.h b/runtime/onnxruntime/src/commonfunc.h similarity index 100% rename from funasr/runtime/onnxruntime/src/commonfunc.h rename to runtime/onnxruntime/src/commonfunc.h diff --git a/funasr/runtime/onnxruntime/src/ct-transformer-online.cpp b/runtime/onnxruntime/src/ct-transformer-online.cpp similarity index 100% rename from funasr/runtime/onnxruntime/src/ct-transformer-online.cpp rename to runtime/onnxruntime/src/ct-transformer-online.cpp diff --git a/funasr/runtime/onnxruntime/src/ct-transformer-online.h b/runtime/onnxruntime/src/ct-transformer-online.h similarity index 100% rename from funasr/runtime/onnxruntime/src/ct-transformer-online.h rename to runtime/onnxruntime/src/ct-transformer-online.h diff --git a/funasr/runtime/onnxruntime/src/ct-transformer.cpp b/runtime/onnxruntime/src/ct-transformer.cpp similarity index 100% rename from funasr/runtime/onnxruntime/src/ct-transformer.cpp rename to runtime/onnxruntime/src/ct-transformer.cpp diff --git a/funasr/runtime/onnxruntime/src/ct-transformer.h b/runtime/onnxruntime/src/ct-transformer.h similarity index 100% rename from funasr/runtime/onnxruntime/src/ct-transformer.h rename to runtime/onnxruntime/src/ct-transformer.h diff --git a/funasr/runtime/onnxruntime/src/e2e-vad.h b/runtime/onnxruntime/src/e2e-vad.h similarity index 100% rename from funasr/runtime/onnxruntime/src/e2e-vad.h rename to runtime/onnxruntime/src/e2e-vad.h diff --git a/funasr/runtime/onnxruntime/src/encode_converter.cpp b/runtime/onnxruntime/src/encode_converter.cpp similarity index 100% rename from funasr/runtime/onnxruntime/src/encode_converter.cpp rename to runtime/onnxruntime/src/encode_converter.cpp diff --git a/funasr/runtime/onnxruntime/src/encode_converter.h b/runtime/onnxruntime/src/encode_converter.h similarity index 100% rename from funasr/runtime/onnxruntime/src/encode_converter.h rename to runtime/onnxruntime/src/encode_converter.h diff --git a/funasr/runtime/onnxruntime/src/fsmn-vad-online.cpp b/runtime/onnxruntime/src/fsmn-vad-online.cpp similarity index 100% rename from funasr/runtime/onnxruntime/src/fsmn-vad-online.cpp rename to runtime/onnxruntime/src/fsmn-vad-online.cpp diff --git a/funasr/runtime/onnxruntime/src/fsmn-vad-online.h b/runtime/onnxruntime/src/fsmn-vad-online.h similarity index 100% rename from funasr/runtime/onnxruntime/src/fsmn-vad-online.h rename to runtime/onnxruntime/src/fsmn-vad-online.h diff --git a/funasr/runtime/onnxruntime/src/fsmn-vad.cpp b/runtime/onnxruntime/src/fsmn-vad.cpp similarity index 100% rename from funasr/runtime/onnxruntime/src/fsmn-vad.cpp rename to runtime/onnxruntime/src/fsmn-vad.cpp diff --git a/funasr/runtime/onnxruntime/src/fsmn-vad.h b/runtime/onnxruntime/src/fsmn-vad.h similarity index 100% rename from funasr/runtime/onnxruntime/src/fsmn-vad.h rename to runtime/onnxruntime/src/fsmn-vad.h diff --git a/funasr/runtime/onnxruntime/src/funasrruntime.cpp b/runtime/onnxruntime/src/funasrruntime.cpp similarity index 89% rename from funasr/runtime/onnxruntime/src/funasrruntime.cpp rename to runtime/onnxruntime/src/funasrruntime.cpp index 91ef18893..f16851f1a 100644 --- a/funasr/runtime/onnxruntime/src/funasrruntime.cpp +++ b/runtime/onnxruntime/src/funasrruntime.cpp @@ -216,7 +216,7 @@ // APIs for Offline-stream Infer _FUNASRAPI FUNASR_RESULT FunOfflineInferBuffer(FUNASR_HANDLE handle, const char* sz_buf, int n_len, FUNASR_MODE mode, QM_CALLBACK fn_callback, const std::vector> &hw_emb, - int sampling_rate, std::string wav_format, bool itn) + int sampling_rate, std::string wav_format, bool itn, FUNASR_DEC_HANDLE dec_handle) { funasr::OfflineStream* offline_stream = (funasr::OfflineStream*)handle; if (!offline_stream) @@ -256,7 +256,12 @@ std::string cur_stamp = "["; std::string lang = (offline_stream->asr_handle)->GetLang(); while (audio.Fetch(buff, len, flag, start_time) > 0) { - string msg = (offline_stream->asr_handle)->Forward(buff, len, true, hw_emb); + // dec reset + funasr::WfstDecoder* wfst_decoder = (funasr::WfstDecoder*)dec_handle; + if (wfst_decoder){ + wfst_decoder->StartUtterance(); + } + string msg = (offline_stream->asr_handle)->Forward(buff, len, true, hw_emb, dec_handle); std::vector msg_vec = funasr::split(msg, '|'); if(msg_vec.size()==0){ continue; @@ -297,7 +302,7 @@ } _FUNASRAPI FUNASR_RESULT FunOfflineInfer(FUNASR_HANDLE handle, const char* sz_filename, FUNASR_MODE mode, QM_CALLBACK fn_callback, - const std::vector> &hw_emb, int sampling_rate, bool itn) + const std::vector> &hw_emb, int sampling_rate, bool itn, FUNASR_DEC_HANDLE dec_handle) { funasr::OfflineStream* offline_stream = (funasr::OfflineStream*)handle; if (!offline_stream) @@ -340,7 +345,12 @@ std::string cur_stamp = "["; std::string lang = (offline_stream->asr_handle)->GetLang(); while (audio.Fetch(buff, len, flag, start_time) > 0) { - string msg = (offline_stream->asr_handle)->Forward(buff, len, true, hw_emb); + // dec reset + funasr::WfstDecoder* wfst_decoder = (funasr::WfstDecoder*)dec_handle; + if (wfst_decoder){ + wfst_decoder->StartUtterance(); + } + string msg = (offline_stream->asr_handle)->Forward(buff, len, true, hw_emb, dec_handle); std::vector msg_vec = funasr::split(msg, '|'); if(msg_vec.size()==0){ continue; @@ -630,6 +640,25 @@ } } + // APIs for decoder status reset + _FUNASRAPI void FunASRReset(FUNASR_HANDLE handle, FUNASR_DEC_HANDLE dec_handle) + { + funasr::Model* recog_obj = (funasr::Model*)handle; + recog_obj->StartUtterance(); + funasr::WfstDecoder* wfst_decoder = (funasr::WfstDecoder*)dec_handle; + if (wfst_decoder) + wfst_decoder->StartUtterance(); + } + + _FUNASRAPI void FunOfflineReset(FUNASR_HANDLE handle, FUNASR_DEC_HANDLE dec_handle) + { + funasr::OfflineStream* recog_obj = (funasr::OfflineStream*)handle; + recog_obj->asr_handle->StartUtterance(); + funasr::WfstDecoder* wfst_decoder = (funasr::WfstDecoder*)dec_handle; + if (wfst_decoder) + wfst_decoder->StartUtterance(); + } + // APIs for Uninit _FUNASRAPI void FunASRUninit(FUNASR_HANDLE handle) { @@ -691,4 +720,38 @@ delete tpass_online_stream; } + _FUNASRAPI FUNASR_DEC_HANDLE FunASRWfstDecoderInit(FUNASR_HANDLE handle, int asr_type, float glob_beam, float lat_beam, float am_scale) + { + funasr::WfstDecoder* mm = nullptr; + if (asr_type == ASR_OFFLINE) { + funasr::OfflineStream* offline_stream = (funasr::OfflineStream*)handle; + funasr::Paraformer* paraformer = (funasr::Paraformer*)offline_stream->asr_handle.get(); + if (paraformer->lm_) + mm = new funasr::WfstDecoder(paraformer->lm_.get(), + paraformer->GetPhoneSet(), paraformer->GetVocab(), glob_beam, lat_beam, am_scale); + } + return mm; + } + _FUNASRAPI void FunASRWfstDecoderUninit(FUNASR_DEC_HANDLE handle) + { + funasr::WfstDecoder* wfst_decoder = (funasr::WfstDecoder*)handle; + if (!wfst_decoder) + return; + delete wfst_decoder; + } + + _FUNASRAPI void FunWfstDecoderLoadHwsRes(FUNASR_DEC_HANDLE handle, int inc_bias, unordered_map &hws_map) + { + funasr::WfstDecoder* wfst_decoder = (funasr::WfstDecoder*)handle; + if (!wfst_decoder) + return; + wfst_decoder->LoadHwsRes(inc_bias, hws_map); + } + _FUNASRAPI void FunWfstDecoderUnloadHwsRes(FUNASR_DEC_HANDLE handle) + { + funasr::WfstDecoder* wfst_decoder = (funasr::WfstDecoder*)handle; + if (!wfst_decoder) + return; + wfst_decoder->UnloadHwsRes(); + } diff --git a/funasr/runtime/onnxruntime/src/itn-processor.cpp b/runtime/onnxruntime/src/itn-processor.cpp similarity index 100% rename from funasr/runtime/onnxruntime/src/itn-processor.cpp rename to runtime/onnxruntime/src/itn-processor.cpp diff --git a/funasr/runtime/onnxruntime/src/itn-processor.h b/runtime/onnxruntime/src/itn-processor.h similarity index 100% rename from funasr/runtime/onnxruntime/src/itn-processor.h rename to runtime/onnxruntime/src/itn-processor.h diff --git a/funasr/runtime/onnxruntime/src/itn-token-parser.cpp b/runtime/onnxruntime/src/itn-token-parser.cpp similarity index 100% rename from funasr/runtime/onnxruntime/src/itn-token-parser.cpp rename to runtime/onnxruntime/src/itn-token-parser.cpp diff --git a/funasr/runtime/onnxruntime/src/itn-token-parser.h b/runtime/onnxruntime/src/itn-token-parser.h similarity index 100% rename from funasr/runtime/onnxruntime/src/itn-token-parser.h rename to runtime/onnxruntime/src/itn-token-parser.h diff --git a/funasr/runtime/onnxruntime/src/model.cpp b/runtime/onnxruntime/src/model.cpp similarity index 100% rename from funasr/runtime/onnxruntime/src/model.cpp rename to runtime/onnxruntime/src/model.cpp diff --git a/funasr/runtime/onnxruntime/src/offline-stream.cpp b/runtime/onnxruntime/src/offline-stream.cpp similarity index 92% rename from funasr/runtime/onnxruntime/src/offline-stream.cpp rename to runtime/onnxruntime/src/offline-stream.cpp index caea72e7f..2709ca6b2 100644 --- a/funasr/runtime/onnxruntime/src/offline-stream.cpp +++ b/runtime/onnxruntime/src/offline-stream.cpp @@ -61,6 +61,13 @@ OfflineStream::OfflineStream(std::map& model_path, int asr_handle->InitAsr(am_model_path, am_cmvn_path, am_config_path, thread_num); } + // Lm resource + if (model_path.find(LM_DIR) != model_path.end() && model_path.at(LM_DIR) != "") { + string fst_path, lm_config_path, hws_path; + fst_path = PathAppend(model_path.at(LM_DIR), LM_FST_RES); + lm_config_path = PathAppend(model_path.at(LM_DIR), LM_CONFIG_NAME); + asr_handle->InitLm(fst_path, lm_config_path); + } // PUNC model if(model_path.find(PUNC_DIR) != model_path.end()){ diff --git a/funasr/runtime/onnxruntime/src/paraformer-online.cpp b/runtime/onnxruntime/src/paraformer-online.cpp similarity index 100% rename from funasr/runtime/onnxruntime/src/paraformer-online.cpp rename to runtime/onnxruntime/src/paraformer-online.cpp diff --git a/funasr/runtime/onnxruntime/src/paraformer-online.h b/runtime/onnxruntime/src/paraformer-online.h similarity index 97% rename from funasr/runtime/onnxruntime/src/paraformer-online.h rename to runtime/onnxruntime/src/paraformer-online.h index 86ee36382..932785c56 100644 --- a/funasr/runtime/onnxruntime/src/paraformer-online.h +++ b/runtime/onnxruntime/src/paraformer-online.h @@ -39,6 +39,14 @@ namespace funasr { vector &de_szOutputNames, vector &means_list, vector &vars_list); + + void StartUtterance() + { + } + + void EndUtterance() + { + } Paraformer* para_handle_ = nullptr; // from para_handle_ diff --git a/funasr/runtime/onnxruntime/src/paraformer.cpp b/runtime/onnxruntime/src/paraformer.cpp similarity index 72% rename from funasr/runtime/onnxruntime/src/paraformer.cpp rename to runtime/onnxruntime/src/paraformer.cpp index 5bbaeef4c..b6babfefe 100644 --- a/funasr/runtime/onnxruntime/src/paraformer.cpp +++ b/runtime/onnxruntime/src/paraformer.cpp @@ -66,6 +66,7 @@ void Paraformer::InitAsr(const std::string &am_model, const std::string &am_cmvn m_szOutputNames.push_back(item.c_str()); vocab = new Vocab(am_config.c_str()); LoadConfigFromYaml(am_config.c_str()); + phone_set_ = new PhoneSet(am_config.c_str()); LoadCmvn(am_cmvn.c_str()); } @@ -143,6 +144,7 @@ void Paraformer::InitAsr(const std::string &en_model, const std::string &de_mode de_szOutputNames_.push_back(item.c_str()); vocab = new Vocab(am_config.c_str()); + phone_set_ = new PhoneSet(am_config.c_str()); LoadCmvn(am_cmvn.c_str()); } @@ -184,6 +186,20 @@ void Paraformer::InitAsr(const std::string &am_model, const std::string &en_mode m_szOutputNames.push_back(item.c_str()); } +void Paraformer::InitLm(const std::string &lm_file, + const std::string &lm_cfg_file) { + try { + lm_ = std::shared_ptr>( + fst::Fst::Read(lm_file)); + if (vocab) { delete vocab; } + vocab = new Vocab(lm_cfg_file.c_str()); + LOG(INFO) << "Successfully load lm file " << lm_file; + } catch (std::exception const &e) { + LOG(ERROR) << "Error when load lm file: " << e.what(); + exit(0); + } +} + void Paraformer::LoadConfigFromYaml(const char* filename){ YAML::Node config; @@ -286,6 +302,14 @@ Paraformer::~Paraformer() delete seg_dict; } +void Paraformer::StartUtterance() +{ +} + +void Paraformer::EndUtterance() +{ +} + void Paraformer::Reset() { } @@ -377,229 +401,15 @@ string Paraformer::GreedySearch(float * in, int n_len, int64_t token_nums, bool } } -string Paraformer::PostProcess(std::vector &raw_char, std::vector> ×tamp_list){ - std::vector> timestamp_merge; - int i; - list words; - int is_pre_english = false; - int pre_english_len = 0; - int is_combining = false; - string combine = ""; - - float begin=-1; - for (i=0; i" || word == "" || word == "") - continue; - // step2 combie phoneme to full word - { - int sub_word = !(word.find("@@") == string::npos); - // process word start and middle part - if (sub_word) { - combine += word.erase(word.length() - 2); - if(!is_combining){ - begin = timestamp_list[i][0]; - } - is_combining = true; - continue; - } - // process word end part - else if (is_combining) { - combine += word; - is_combining = false; - word = combine; - combine = ""; - } - } - - // step3 process english word deal with space , turn abbreviation to upper case - { - // input word is chinese, not need process - if (vocab->IsChinese(word)) { - words.push_back(word); - timestamp_merge.emplace_back(timestamp_list[i]); - is_pre_english = false; - } - // input word is english word - else { - // pre word is chinese - if (!is_pre_english) { - // word[0] = word[0] - 32; - words.push_back(word); - begin = (begin==-1)?timestamp_list[i][0]:begin; - std::vector vec = {begin, timestamp_list[i][1]}; - timestamp_merge.emplace_back(vec); - begin = -1; - pre_english_len = word.size(); - } - // pre word is english word - else { - // single letter turn to upper case - // if (word.size() == 1) { - // word[0] = word[0] - 32; - // } - - if (pre_english_len > 1) { - words.push_back(" "); - words.push_back(word); - begin = (begin==-1)?timestamp_list[i][0]:begin; - std::vector vec = {begin, timestamp_list[i][1]}; - timestamp_merge.emplace_back(vec); - begin = -1; - pre_english_len = word.size(); - } - else { - // if (word.size() > 1) { - // words.push_back(" "); - // } - words.push_back(" "); - words.push_back(word); - begin = (begin==-1)?timestamp_list[i][0]:begin; - std::vector vec = {begin, timestamp_list[i][1]}; - timestamp_merge.emplace_back(vec); - begin = -1; - pre_english_len = word.size(); - } - } - is_pre_english = true; - } - } - } - string stamp_str=""; - for (i=0; iSearch(in, len, token_nums); } -void Paraformer::TimestampOnnx(std::vector& us_alphas, - std::vector us_cif_peak, - std::vector& char_list, - std::string &res_str, - std::vector> ×tamp_vec, - float begin_time, - float total_offset){ - if (char_list.empty()) { - return ; - } - - const float START_END_THRESHOLD = 5.0; - const float MAX_TOKEN_DURATION = 30.0; - const float TIME_RATE = 10.0 * 6 / 1000 / 3; - // 3 times upsampled, cif_peak is flattened into a 1D array - std::vector cif_peak = us_cif_peak; - int num_frames = cif_peak.size(); - if (char_list.back() == "") { - char_list.pop_back(); - } - if (char_list.empty()) { - return ; - } - vector> timestamp_list; - vector new_char_list; - vector fire_place; - // for bicif model trained with large data, cif2 actually fires when a character starts - // so treat the frames between two peaks as the duration of the former token - for (int i = 0; i < num_frames; i++) { - if (cif_peak[i] > 1.0 - 1e-4) { - fire_place.push_back(i + total_offset); - } - } - int num_peak = fire_place.size(); - if(num_peak != (int)char_list.size() + 1){ - float sum = std::accumulate(us_alphas.begin(), us_alphas.end(), 0.0f); - float scale = sum/((int)char_list.size() + 1); - if(scale == 0){ - return; - } - cif_peak.clear(); - sum = 0.0; - for(auto &alpha:us_alphas){ - alpha = alpha/scale; - sum += alpha; - cif_peak.emplace_back(sum); - if(sum>=1.0 - 1e-4){ - sum -=(1.0 - 1e-4); - } - } - - fire_place.clear(); - for (int i = 0; i < num_frames; i++) { - if (cif_peak[i] > 1.0 - 1e-4) { - fire_place.push_back(i + total_offset); - } - } - } - - num_peak = fire_place.size(); - if(fire_place.size() == 0){ - return; - } - - // begin silence - if (fire_place[0] > START_END_THRESHOLD) { - new_char_list.push_back(""); - timestamp_list.push_back({0.0, fire_place[0] * TIME_RATE}); - } - - // tokens timestamp - for (int i = 0; i < num_peak - 1; i++) { - new_char_list.push_back(char_list[i]); - if (i == num_peak - 2 || MAX_TOKEN_DURATION < 0 || fire_place[i + 1] - fire_place[i] < MAX_TOKEN_DURATION) { - timestamp_list.push_back({fire_place[i] * TIME_RATE, fire_place[i + 1] * TIME_RATE}); - } else { - // cut the duration to token and sil of the 0-weight frames last long - float _split = fire_place[i] + MAX_TOKEN_DURATION; - timestamp_list.push_back({fire_place[i] * TIME_RATE, _split * TIME_RATE}); - timestamp_list.push_back({_split * TIME_RATE, fire_place[i + 1] * TIME_RATE}); - new_char_list.push_back(""); - } - } - - // tail token and end silence - if(timestamp_list.size()==0){ - LOG(ERROR)<<"timestamp_list's size is 0!"; - return; - } - if (num_frames - fire_place.back() > START_END_THRESHOLD) { - float _end = (num_frames + fire_place.back()) / 2.0; - timestamp_list.back()[1] = _end * TIME_RATE; - timestamp_list.push_back({_end * TIME_RATE, num_frames * TIME_RATE}); - new_char_list.push_back(""); - } else { - timestamp_list.back()[1] = num_frames * TIME_RATE; - } - - if (begin_time) { // add offset time in model with vad - for (auto& timestamp : timestamp_list) { - timestamp[0] += begin_time / 1000.0; - timestamp[1] += begin_time / 1000.0; - } - } - - assert(new_char_list.size() == timestamp_list.size()); - - for (int i = 0; i < (int)new_char_list.size(); i++) { - res_str += new_char_list[i] + " " + to_string(timestamp_list[i][0]) + " " + to_string(timestamp_list[i][1]) + ";"; - } - - for (int i = 0; i < (int)new_char_list.size(); i++) { - if(new_char_list[i] != ""){ - timestamp_vec.push_back(timestamp_list[i]); - } - } +string Paraformer::FinalizeDecode(WfstDecoder* &wfst_decoder, + bool is_stamp, std::vector us_alphas, std::vector us_cif_peak) +{ + return wfst_decoder->FinalizeDecode(is_stamp, us_alphas, us_cif_peak); } vector Paraformer::ApplyLfr(const std::vector &in) @@ -641,9 +451,9 @@ vector Paraformer::ApplyLfr(const std::vector &in) } } -string Paraformer::Forward(float* din, int len, bool input_finished, const std::vector> &hw_emb) +string Paraformer::Forward(float* din, int len, bool input_finished, const std::vector> &hw_emb, void* decoder_handle) { - + WfstDecoder* wfst_decoder = (WfstDecoder*)decoder_handle; int32_t in_feat_dim = fbank_opts_.mel_opts.num_bins; std::vector wav_feats = FbankKaldi(MODEL_SAMPLE_RATE, din, len); wav_feats = ApplyLfr(wav_feats); @@ -725,9 +535,23 @@ string Paraformer::Forward(float* din, int len, bool input_finished, const std:: for (int i = 0; i < us_peaks_shape[1]; i++) { us_peaks[i] = us_peaks_data[i]; } - result = GreedySearch(floatData, *encoder_out_lens, outputShape[2], true, us_alphas, us_peaks); + if (lm_ == nullptr) { + result = GreedySearch(floatData, *encoder_out_lens, outputShape[2], true, us_alphas, us_peaks); + } else { + result = BeamSearch(wfst_decoder, floatData, *encoder_out_lens, outputShape[2]); + if (input_finished) { + result = FinalizeDecode(wfst_decoder, true, us_alphas, us_peaks); + } + } }else{ - result = GreedySearch(floatData, *encoder_out_lens, outputShape[2]); + if (lm_ == nullptr) { + result = GreedySearch(floatData, *encoder_out_lens, outputShape[2]); + } else { + result = BeamSearch(wfst_decoder, floatData, *encoder_out_lens, outputShape[2]); + if (input_finished) { + result = FinalizeDecode(wfst_decoder); + } + } } } catch (std::exception const &e) @@ -773,11 +597,19 @@ std::vector> Paraformer::CompileHotwordEmbedding(std::string } std::vector hw_vector(max_hotword_len, 0); int vector_len = std::min(max_hotword_len, (int)chars.size()); - for (int i=0; iGetIdByToken(chars[i]); + int chs_oov = false; + for (int i=0; iString2Id(chars[i]); + if(hw_vector[i] == -1){ + chs_oov = true; + break; + } } - std::cout << std::endl; + if(chs_oov){ + LOG(INFO) << "OOV: " << hotword; + continue; + } + LOG(INFO) << hotword; lengths.push_back(vector_len); real_hw_size += 1; hotword_matrix.insert(hotword_matrix.end(), hw_vector.begin(), hw_vector.end()); @@ -834,6 +666,16 @@ std::vector> Paraformer::CompileHotwordEmbedding(std::string return result; } +Vocab* Paraformer::GetVocab() +{ + return vocab; +} + +PhoneSet* Paraformer::GetPhoneSet() +{ + return phone_set_; +} + string Paraformer::Rescoring() { LOG(ERROR)<<"Not Imp!!!!!!"; diff --git a/funasr/runtime/onnxruntime/src/paraformer.h b/runtime/onnxruntime/src/paraformer.h similarity index 79% rename from funasr/runtime/onnxruntime/src/paraformer.h rename to runtime/onnxruntime/src/paraformer.h index 455078e1b..58de34532 100644 --- a/funasr/runtime/onnxruntime/src/paraformer.h +++ b/runtime/onnxruntime/src/paraformer.h @@ -5,6 +5,10 @@ #pragma once #include "precomp.h" +#include "fst/fstlib.h" +#include "fst/symbol-table.h" +#include "bias-lm.h" +#include "phone-set.h" namespace funasr { @@ -17,6 +21,7 @@ namespace funasr { private: Vocab* vocab = nullptr; SegDict* seg_dict = nullptr; + PhoneSet* phone_set_ = nullptr; //const float scale = 22.6274169979695; const float scale = 1.0; @@ -47,15 +52,22 @@ namespace funasr { std::vector> CompileHotwordEmbedding(std::string &hotwords); void Reset(); vector FbankKaldi(float sample_rate, const float* waves, int len); - string Forward(float* din, int len, bool input_finished=true, const std::vector> &hw_emb={{0.0}}); - string GreedySearch( float* in, int n_len, int64_t token_nums, bool is_stamp=false, std::vector us_alphas={0}, std::vector us_cif_peak={0}); - void TimestampOnnx(std::vector &us_alphas, vector us_cif_peak, vector& char_list, std::string &res_str, - vector> ×tamp_list, float begin_time = 0.0, float total_offset = -1.5); - string PostProcess(std::vector &raw_char, std::vector> ×tamp_list); + string Forward(float* din, int len, bool input_finished=true, const std::vector> &hw_emb={{0.0}}, void* wfst_decoder=nullptr); + string GreedySearch( float* in, int n_len, int64_t token_nums, + bool is_stamp=false, std::vector us_alphas={0}, std::vector us_cif_peak={0}); string Rescoring(); string GetLang(){return language;}; - + + void StartUtterance(); + void EndUtterance(); + void InitLm(const std::string &lm_file, const std::string &lm_cfg_file); + string BeamSearch(WfstDecoder* &wfst_decoder, float* in, int n_len, int64_t token_nums); + string FinalizeDecode(WfstDecoder* &wfst_decoder, + bool is_stamp=false, std::vector us_alphas={0}, std::vector us_cif_peak={0}); + Vocab* GetVocab(); + PhoneSet* GetPhoneSet(); + knf::FbankOptions fbank_opts_; vector means_list_; vector vars_list_; @@ -82,7 +94,10 @@ namespace funasr { vector de_strInputNames, de_strOutputNames; vector de_szInputNames_; vector de_szOutputNames_; - + + // lm + std::shared_ptr> lm_ = nullptr; + string window_type = "hamming"; int frame_length = 25; int frame_shift = 10; diff --git a/runtime/onnxruntime/src/phone-set.cpp b/runtime/onnxruntime/src/phone-set.cpp new file mode 100644 index 000000000..167fa010a --- /dev/null +++ b/runtime/onnxruntime/src/phone-set.cpp @@ -0,0 +1,76 @@ +#include "phone-set.h" +#include +#include + +#include +#include +#include +#include +#include + +using namespace std; + +namespace funasr { +PhoneSet::PhoneSet(const char *filename) { + ifstream in(filename); + LoadPhoneSetFromYaml(filename); +} +PhoneSet::~PhoneSet() +{ +} + +void PhoneSet::LoadPhoneSetFromYaml(const char* filename) { + YAML::Node config; + try{ + config = YAML::LoadFile(filename); + }catch(exception const &e){ + LOG(INFO) << "Error loading file, yaml file error or not exist."; + exit(-1); + } + YAML::Node myList = config["token_list"]; + int id = 0; + for (YAML::const_iterator it = myList.begin(); it != myList.end(); ++it, id++) { + phone_.push_back(it->as()); + phn2Id_.emplace(it->as(), id); + } +} + +int PhoneSet::Size() const { + return phone_.size(); +} + +int PhoneSet::String2Id(string phn_str) const { + if (phn2Id_.count(phn_str)) { + return phn2Id_.at(phn_str); + } else { + //LOG(INFO) << "Phone unit not exist."; + return -1; + } +} + +string PhoneSet::Id2String(int id) const { + if (id < 0 || id > Size()) { + //LOG(INFO) << "Phone id not exist."; + return ""; + } else { + return phone_[id]; + } +} + +bool PhoneSet::Find(string phn_str) const { + return phn2Id_.count(phn_str) > 0; +} + +int PhoneSet::GetBegSilPhnId() const { + return String2Id(UNIT_BEG_SIL_SYMBOL); +} + +int PhoneSet::GetEndSilPhnId() const { + return String2Id(UNIT_END_SIL_SYMBOL); +} + +int PhoneSet::GetBlkPhnId() const { + return String2Id(UNIT_BLK_SYMBOL); +} + +} diff --git a/runtime/onnxruntime/src/phone-set.h b/runtime/onnxruntime/src/phone-set.h new file mode 100644 index 000000000..972910408 --- /dev/null +++ b/runtime/onnxruntime/src/phone-set.h @@ -0,0 +1,34 @@ +#ifndef PHONESET_H +#define PHONESET_H + +#include +#include +#include +#include +#define UNIT_BEG_SIL_SYMBOL "" +#define UNIT_END_SIL_SYMBOL "" +#define UNIT_BLK_SYMBOL "" + +using namespace std; + +namespace funasr { +class PhoneSet { + public: + PhoneSet(const char *filename); + ~PhoneSet(); + int Size() const; + int String2Id(string str) const; + string Id2String(int id) const; + bool Find(string str) const; + int GetBegSilPhnId() const; + int GetEndSilPhnId() const; + int GetBlkPhnId() const; + + private: + vector phone_; + unordered_map phn2Id_; + void LoadPhoneSetFromYaml(const char* filename); +}; + +} // namespace funasr +#endif diff --git a/funasr/runtime/onnxruntime/src/precomp.h b/runtime/onnxruntime/src/precomp.h similarity index 93% rename from funasr/runtime/onnxruntime/src/precomp.h rename to runtime/onnxruntime/src/precomp.h index 5c49fd6a5..d74608cdc 100644 --- a/funasr/runtime/onnxruntime/src/precomp.h +++ b/runtime/onnxruntime/src/precomp.h @@ -49,7 +49,7 @@ using namespace std; #include "kaldi-native-fbank/csrc/feature-fbank.h" #include "kaldi-native-fbank/csrc/online-feature.h" - +#include "kaldi/decoder/lattice-faster-online-decoder.h" // mine #include @@ -68,6 +68,8 @@ using namespace std; #include "fsmn-vad.h" #include "encode_converter.h" #include "vocab.h" +#include "phone-set.h" +#include "wfst-decoder.h" #include "audio.h" #include "fsmn-vad-online.h" #include "tensor.h" diff --git a/funasr/runtime/onnxruntime/src/predefine-coe.h b/runtime/onnxruntime/src/predefine-coe.h similarity index 100% rename from funasr/runtime/onnxruntime/src/predefine-coe.h rename to runtime/onnxruntime/src/predefine-coe.h diff --git a/funasr/runtime/onnxruntime/src/punc-model.cpp b/runtime/onnxruntime/src/punc-model.cpp similarity index 100% rename from funasr/runtime/onnxruntime/src/punc-model.cpp rename to runtime/onnxruntime/src/punc-model.cpp diff --git a/funasr/runtime/onnxruntime/src/resample.cpp b/runtime/onnxruntime/src/resample.cpp similarity index 100% rename from funasr/runtime/onnxruntime/src/resample.cpp rename to runtime/onnxruntime/src/resample.cpp diff --git a/funasr/runtime/onnxruntime/src/resample.h b/runtime/onnxruntime/src/resample.h similarity index 100% rename from funasr/runtime/onnxruntime/src/resample.h rename to runtime/onnxruntime/src/resample.h diff --git a/funasr/runtime/onnxruntime/src/seg_dict.cpp b/runtime/onnxruntime/src/seg_dict.cpp similarity index 100% rename from funasr/runtime/onnxruntime/src/seg_dict.cpp rename to runtime/onnxruntime/src/seg_dict.cpp diff --git a/funasr/runtime/onnxruntime/src/seg_dict.h b/runtime/onnxruntime/src/seg_dict.h similarity index 100% rename from funasr/runtime/onnxruntime/src/seg_dict.h rename to runtime/onnxruntime/src/seg_dict.h diff --git a/funasr/runtime/onnxruntime/src/tensor.h b/runtime/onnxruntime/src/tensor.h similarity index 98% rename from funasr/runtime/onnxruntime/src/tensor.h rename to runtime/onnxruntime/src/tensor.h index a2a7bc366..4f05f6d61 100644 --- a/funasr/runtime/onnxruntime/src/tensor.h +++ b/runtime/onnxruntime/src/tensor.h @@ -2,7 +2,9 @@ #define TENSOR_H #include "alignedmem.h" - +#include "stdio.h" +#include +#include using namespace std; namespace funasr { diff --git a/funasr/runtime/onnxruntime/src/tokenizer.cpp b/runtime/onnxruntime/src/tokenizer.cpp similarity index 100% rename from funasr/runtime/onnxruntime/src/tokenizer.cpp rename to runtime/onnxruntime/src/tokenizer.cpp diff --git a/funasr/runtime/onnxruntime/src/tokenizer.h b/runtime/onnxruntime/src/tokenizer.h similarity index 100% rename from funasr/runtime/onnxruntime/src/tokenizer.h rename to runtime/onnxruntime/src/tokenizer.h diff --git a/funasr/runtime/onnxruntime/src/tpass-online-stream.cpp b/runtime/onnxruntime/src/tpass-online-stream.cpp similarity index 100% rename from funasr/runtime/onnxruntime/src/tpass-online-stream.cpp rename to runtime/onnxruntime/src/tpass-online-stream.cpp diff --git a/funasr/runtime/onnxruntime/src/tpass-stream.cpp b/runtime/onnxruntime/src/tpass-stream.cpp similarity index 100% rename from funasr/runtime/onnxruntime/src/tpass-stream.cpp rename to runtime/onnxruntime/src/tpass-stream.cpp diff --git a/funasr/runtime/onnxruntime/src/utf8-string.cpp b/runtime/onnxruntime/src/utf8-string.cpp similarity index 100% rename from funasr/runtime/onnxruntime/src/utf8-string.cpp rename to runtime/onnxruntime/src/utf8-string.cpp diff --git a/funasr/runtime/onnxruntime/src/utf8-string.h b/runtime/onnxruntime/src/utf8-string.h similarity index 100% rename from funasr/runtime/onnxruntime/src/utf8-string.h rename to runtime/onnxruntime/src/utf8-string.h diff --git a/runtime/onnxruntime/src/util.cpp b/runtime/onnxruntime/src/util.cpp new file mode 100644 index 000000000..005de5797 --- /dev/null +++ b/runtime/onnxruntime/src/util.cpp @@ -0,0 +1,672 @@ + +#include "precomp.h" + +namespace funasr { +float *LoadParams(const char *filename) +{ + + FILE *fp; + fp = fopen(filename, "rb"); + fseek(fp, 0, SEEK_END); + uint32_t nFileLen = ftell(fp); + fseek(fp, 0, SEEK_SET); + + float *params_addr = (float *)AlignedMalloc(32, nFileLen); + int n = fread(params_addr, 1, nFileLen, fp); + fclose(fp); + + return params_addr; +} + +int ValAlign(int val, int align) +{ + float tmp = ceil((float)val / (float)align) * (float)align; + return (int)tmp; +} + +void DispParams(float *din, int size) +{ + int i; + for (i = 0; i < size; i++) { + printf("%f ", din[i]); + } + printf("\n"); +} +void SaveDataFile(const char *filename, void *data, uint32_t len) +{ + FILE *fp; + fp = fopen(filename, "wb+"); + fwrite(data, 1, len, fp); + fclose(fp); +} + +void BasicNorm(Tensor *&din, float norm) +{ + + int Tmax = din->size[2]; + + int i, j; + for (i = 0; i < Tmax; i++) { + float sum = 0; + for (j = 0; j < 512; j++) { + int ii = i * 512 + j; + sum += din->buff[ii] * din->buff[ii]; + } + float mean = sqrt(sum / 512 + norm); + for (j = 0; j < 512; j++) { + int ii = i * 512 + j; + din->buff[ii] = din->buff[ii] / mean; + } + } +} + +void FindMax(float *din, int len, float &max_val, int &max_idx) +{ + int i; + max_val = -INFINITY; + max_idx = -1; + for (i = 0; i < len; i++) { + if (din[i] > max_val) { + max_val = din[i]; + max_idx = i; + } + } +} + +string PathAppend(const string &p1, const string &p2) +{ + + char sep = '/'; + string tmp = p1; + +#ifdef _WIN32 + sep = '\\'; +#endif + + if (p1[p1.length()-1] != sep) { // Need to add a + tmp += sep; // path separator + return (tmp + p2); + } else + return (p1 + p2); +} + +void Relu(Tensor *din) +{ + int i; + for (i = 0; i < din->buff_size; i++) { + float val = din->buff[i]; + din->buff[i] = val < 0 ? 0 : val; + } +} + +void Swish(Tensor *din) +{ + int i; + for (i = 0; i < din->buff_size; i++) { + float val = din->buff[i]; + din->buff[i] = val / (1 + exp(-val)); + } +} + +void Sigmoid(Tensor *din) +{ + int i; + for (i = 0; i < din->buff_size; i++) { + float val = din->buff[i]; + din->buff[i] = 1 / (1 + exp(-val)); + } +} + +void DoubleSwish(Tensor *din) +{ + int i; + for (i = 0; i < din->buff_size; i++) { + float val = din->buff[i]; + din->buff[i] = val / (1 + exp(-val + 1)); + } +} + +void Softmax(float *din, int mask, int len) +{ + float *tmp = (float *)malloc(mask * sizeof(float)); + int i; + float sum = 0; + float max = -INFINITY; + + for (i = 0; i < mask; i++) { + max = max < din[i] ? din[i] : max; + } + + for (i = 0; i < mask; i++) { + tmp[i] = exp(din[i] - max); + sum += tmp[i]; + } + for (i = 0; i < mask; i++) { + din[i] = tmp[i] / sum; + } + free(tmp); + for (i = mask; i < len; i++) { + din[i] = 0; + } +} + +void LogSoftmax(float *din, int len) +{ + float *tmp = (float *)malloc(len * sizeof(float)); + int i; + float sum = 0; + for (i = 0; i < len; i++) { + tmp[i] = exp(din[i]); + sum += tmp[i]; + } + for (i = 0; i < len; i++) { + din[i] = log(tmp[i] / sum); + } + free(tmp); +} + +void Glu(Tensor *din, Tensor *dout) +{ + int mm = din->buff_size / 1024; + int i, j; + for (i = 0; i < mm; i++) { + for (j = 0; j < 512; j++) { + int in_off = i * 1024 + j; + int out_off = i * 512 + j; + float a = din->buff[in_off]; + float b = din->buff[in_off + 512]; + dout->buff[out_off] = a / (1 + exp(-b)); + } + } +} + +bool is_target_file(const std::string& filename, const std::string target) { + std::size_t pos = filename.find_last_of("."); + if (pos == std::string::npos) { + return false; + } + std::string extension = filename.substr(pos + 1); + return (extension == target); +} + +void KeepChineseCharacterAndSplit(const std::string &input_str, + std::vector &chinese_characters) { + chinese_characters.resize(0); + std::vector u16_buf; + u16_buf.resize(std::max(u16_buf.size(), input_str.size() + 1)); + U16CHAR_T* pu16 = u16_buf.data(); + U8CHAR_T * pu8 = (U8CHAR_T*)input_str.data(); + size_t ilen = input_str.size(); + size_t len = EncodeConverter::Utf8ToUtf16(pu8, ilen, pu16, ilen + 1); + for (size_t i = 0; i < len; i++) { + if (EncodeConverter::IsChineseCharacter(pu16[i])) { + U8CHAR_T u8buf[4]; + size_t n = EncodeConverter::Utf16ToUtf8(pu16 + i, u8buf); + u8buf[n] = '\0'; + chinese_characters.push_back((const char*)u8buf); + } + } +} + +void SplitChiEngCharacters(const std::string &input_str, + std::vector &characters) { + characters.resize(0); + std::string eng_word = ""; + U16CHAR_T space = 0x0020; + std::vector u16_buf; + u16_buf.resize(std::max(u16_buf.size(), input_str.size() + 1)); + U16CHAR_T* pu16 = u16_buf.data(); + U8CHAR_T * pu8 = (U8CHAR_T*)input_str.data(); + size_t ilen = input_str.size(); + size_t len = EncodeConverter::Utf8ToUtf16(pu8, ilen, pu16, ilen + 1); + for (size_t i = 0; i < len; i++) { + if (EncodeConverter::IsChineseCharacter(pu16[i])) { + if(!eng_word.empty()){ + characters.push_back(eng_word); + eng_word = ""; + } + U8CHAR_T u8buf[4]; + size_t n = EncodeConverter::Utf16ToUtf8(pu16 + i, u8buf); + u8buf[n] = '\0'; + characters.push_back((const char*)u8buf); + } else if (pu16[i] == space){ + if(!eng_word.empty()){ + characters.push_back(eng_word); + eng_word = ""; + } + }else{ + U8CHAR_T u8buf[4]; + size_t n = EncodeConverter::Utf16ToUtf8(pu16 + i, u8buf); + u8buf[n] = '\0'; + eng_word += (const char*)u8buf; + } + } + if(!eng_word.empty()){ + characters.push_back(eng_word); + eng_word = ""; + } +} + +std::vector split(const std::string &s, char delim) { + std::vector elems; + std::stringstream ss(s); + std::string item; + while(std::getline(ss, item, delim)) { + elems.push_back(item); + } + return elems; +} + +template +void PrintMat(const std::vector> &mat, const std::string &name) { + std::cout << name << ":" << std::endl; + for (auto item : mat) { + for (auto item_ : item) { + std::cout << item_ << " "; + } + std::cout << std::endl; + } +} + +size_t Utf8ToCharset(const std::string &input, std::vector &output) { + std::string ch; + for (size_t i = 0, len = 0; i != input.length(); i += len) { + unsigned char byte = (unsigned)input[i]; + if (byte >= 0xFC) // lenght 6 + len = 6; + else if (byte >= 0xF8) + len = 5; + else if (byte >= 0xF0) + len = 4; + else if (byte >= 0xE0) + len = 3; + else if (byte >= 0xC0) + len = 2; + else + len = 1; + ch = input.substr(i, len); + output.push_back(ch); + } + return output.size(); +} + +int Str2IntFunc(string str) +{ + const char *ch_array = str.c_str(); + if (((ch_array[0] & 0xf0) != 0xe0) || ((ch_array[1] & 0xc0) != 0x80) || + ((ch_array[2] & 0xc0) != 0x80)) + return 0; + int val = ((ch_array[0] & 0x0f) << 12) | ((ch_array[1] & 0x3f) << 6) | + (ch_array[2] & 0x3f); + return val; +} + +bool IsChinese(string ch) +{ + if (ch.size() != 3) { + return false; + } + int unicode = Str2IntFunc(ch); + if (unicode >= 19968 && unicode <= 40959) { + return true; + } + return false; +} + +string PostProcess(std::vector &raw_char, std::vector> ×tamp_list){ + std::vector> timestamp_merge; + int i; + list words; + int is_pre_english = false; + int pre_english_len = 0; + int is_combining = false; + string combine = ""; + + float begin=-1; + for (i=0; i" || word == "" || word == "") + continue; + // step2 combie phoneme to full word + { + int sub_word = !(word.find("@@") == string::npos); + // process word start and middle part + if (sub_word) { + combine += word.erase(word.length() - 2); + if(!is_combining){ + begin = timestamp_list[i][0]; + } + is_combining = true; + continue; + } + // process word end part + else if (is_combining) { + combine += word; + is_combining = false; + word = combine; + combine = ""; + } + } + + // step3 process english word deal with space , turn abbreviation to upper case + { + // input word is chinese, not need process + if (IsChinese(word)) { + words.push_back(word); + timestamp_merge.emplace_back(timestamp_list[i]); + is_pre_english = false; + } + // input word is english word + else { + // pre word is chinese + if (!is_pre_english) { + // word[0] = word[0] - 32; + words.push_back(word); + begin = (begin==-1)?timestamp_list[i][0]:begin; + std::vector vec = {begin, timestamp_list[i][1]}; + timestamp_merge.emplace_back(vec); + begin = -1; + pre_english_len = word.size(); + } + // pre word is english word + else { + // single letter turn to upper case + // if (word.size() == 1) { + // word[0] = word[0] - 32; + // } + + if (pre_english_len > 1) { + words.push_back(" "); + words.push_back(word); + begin = (begin==-1)?timestamp_list[i][0]:begin; + std::vector vec = {begin, timestamp_list[i][1]}; + timestamp_merge.emplace_back(vec); + begin = -1; + pre_english_len = word.size(); + } + else { + // if (word.size() > 1) { + // words.push_back(" "); + // } + words.push_back(" "); + words.push_back(word); + begin = (begin==-1)?timestamp_list[i][0]:begin; + std::vector vec = {begin, timestamp_list[i][1]}; + timestamp_merge.emplace_back(vec); + begin = -1; + pre_english_len = word.size(); + } + } + is_pre_english = true; + } + } + } + string stamp_str=""; + for (i=0; i& us_alphas, + std::vector us_cif_peak, + std::vector& char_list, + std::string &res_str, + std::vector> ×tamp_vec, + float begin_time, + float total_offset){ + if (char_list.empty()) { + return ; + } + + const float START_END_THRESHOLD = 5.0; + const float MAX_TOKEN_DURATION = 30.0; + const float TIME_RATE = 10.0 * 6 / 1000 / 3; + // 3 times upsampled, cif_peak is flattened into a 1D array + std::vector cif_peak = us_cif_peak; + int num_frames = cif_peak.size(); + if (char_list.back() == "") { + char_list.pop_back(); + } + if (char_list.empty()) { + return ; + } + vector> timestamp_list; + vector new_char_list; + vector fire_place; + // for bicif model trained with large data, cif2 actually fires when a character starts + // so treat the frames between two peaks as the duration of the former token + for (int i = 0; i < num_frames; i++) { + if (cif_peak[i] > 1.0 - 1e-4) { + fire_place.push_back(i + total_offset); + } + } + int num_peak = fire_place.size(); + if(num_peak != (int)char_list.size() + 1){ + float sum = std::accumulate(us_alphas.begin(), us_alphas.end(), 0.0f); + float scale = sum/((int)char_list.size() + 1); + if(scale == 0){ + return; + } + cif_peak.clear(); + sum = 0.0; + for(auto &alpha:us_alphas){ + alpha = alpha/scale; + sum += alpha; + cif_peak.emplace_back(sum); + if(sum>=1.0 - 1e-4){ + sum -=(1.0 - 1e-4); + } + } + + fire_place.clear(); + for (int i = 0; i < num_frames; i++) { + if (cif_peak[i] > 1.0 - 1e-4) { + fire_place.push_back(i + total_offset); + } + } + } + + num_peak = fire_place.size(); + if(fire_place.size() == 0){ + return; + } + + // begin silence + if (fire_place[0] > START_END_THRESHOLD) { + new_char_list.push_back(""); + timestamp_list.push_back({0.0, fire_place[0] * TIME_RATE}); + } + + // tokens timestamp + for (int i = 0; i < num_peak - 1; i++) { + new_char_list.push_back(char_list[i]); + if (i == num_peak - 2 || MAX_TOKEN_DURATION < 0 || fire_place[i + 1] - fire_place[i] < MAX_TOKEN_DURATION) { + timestamp_list.push_back({fire_place[i] * TIME_RATE, fire_place[i + 1] * TIME_RATE}); + } else { + // cut the duration to token and sil of the 0-weight frames last long + float _split = fire_place[i] + MAX_TOKEN_DURATION; + timestamp_list.push_back({fire_place[i] * TIME_RATE, _split * TIME_RATE}); + timestamp_list.push_back({_split * TIME_RATE, fire_place[i + 1] * TIME_RATE}); + new_char_list.push_back(""); + } + } + + // tail token and end silence + if(timestamp_list.size()==0){ + LOG(ERROR)<<"timestamp_list's size is 0!"; + return; + } + if (num_frames - fire_place.back() > START_END_THRESHOLD) { + float _end = (num_frames + fire_place.back()) / 2.0; + timestamp_list.back()[1] = _end * TIME_RATE; + timestamp_list.push_back({_end * TIME_RATE, num_frames * TIME_RATE}); + new_char_list.push_back(""); + } else { + timestamp_list.back()[1] = num_frames * TIME_RATE; + } + + if (begin_time) { // add offset time in model with vad + for (auto& timestamp : timestamp_list) { + timestamp[0] += begin_time / 1000.0; + timestamp[1] += begin_time / 1000.0; + } + } + + assert(new_char_list.size() == timestamp_list.size()); + + for (int i = 0; i < (int)new_char_list.size(); i++) { + res_str += new_char_list[i] + " " + to_string(timestamp_list[i][0]) + " " + to_string(timestamp_list[i][1]) + ";"; + } + + for (int i = 0; i < (int)new_char_list.size(); i++) { + if(new_char_list[i] != ""){ + timestamp_vec.push_back(timestamp_list[i]); + } + } +} + +bool IsTargetFile(const std::string& filename, const std::string target) { + std::size_t pos = filename.find_last_of("."); + if (pos == std::string::npos) { + return false; + } + std::string extension = filename.substr(pos + 1); + return (extension == target); +} + +void Trim(std::string *str) { + const char *white_chars = " \t\n\r\f\v"; + + std::string::size_type pos = str->find_last_not_of(white_chars); + if (pos != std::string::npos) { + str->erase(pos + 1); + pos = str->find_first_not_of(white_chars); + if (pos != std::string::npos) str->erase(0, pos); + } else { + str->erase(str->begin(), str->end()); + } +} + +void SplitStringToVector(const std::string &full, const char *delim, + bool omit_empty_strings, + std::vector *out) { + size_t start = 0, found = 0, end = full.size(); + out->clear(); + while (found != std::string::npos) { + found = full.find_first_of(delim, start); + // start != end condition is for when the delimiter is at the end + if (!omit_empty_strings || (found != start && start != end)) + out->push_back(full.substr(start, found - start)); + start = found + 1; + } +} + +void ExtractHws(string hws_file, unordered_map &hws_map) +{ + if(hws_file.empty()){ + return; + } + std::string line; + std::ifstream ifs_hws(hws_file.c_str()); + if(!ifs_hws.is_open()){ + LOG(ERROR) << "Unable to open hotwords file: " << hws_file + << ". If you have not set hotwords, please ignore this message."; + return; + } + LOG(INFO) << "hotwords: "; + while (getline(ifs_hws, line)) { + Trim(&line); + if (line.empty()) { + continue; + } + float score = 1.0f; + std::vector text; + SplitStringToVector(line, " ", true, &text); + + if (text.size() > 1) { + try{ + score = std::stof(text[text.size() - 1]); + }catch (std::exception const &e) + { + LOG(ERROR)< &hws_map, string& nn_hotwords_) +{ + if(hws_file.empty()){ + return; + } + std::string line; + std::ifstream ifs_hws(hws_file.c_str()); + if(!ifs_hws.is_open()){ + LOG(ERROR) << "Unable to open hotwords file: " << hws_file + << ". If you have not set hotwords, please ignore this message."; + return; + } + LOG(INFO) << "hotwords: "; + while (getline(ifs_hws, line)) { + Trim(&line); + if (line.empty()) { + continue; + } + float score = 1.0f; + std::vector text; + SplitStringToVector(line, " ", true, &text); + + if (text.size() > 1) { + try{ + score = std::stof(text[text.size() - 1]); + }catch (std::exception const &e) + { + LOG(ERROR)< +#include +#include +#include "tensor.h" + +using namespace std; + +namespace funasr { +extern float *LoadParams(const char *filename); + +extern void SaveDataFile(const char *filename, void *data, uint32_t len); +extern void Relu(Tensor *din); +extern void Swish(Tensor *din); +extern void Sigmoid(Tensor *din); +extern void DoubleSwish(Tensor *din); + +extern void Softmax(float *din, int mask, int len); + +extern void LogSoftmax(float *din, int len); +extern int ValAlign(int val, int align); +extern void DispParams(float *din, int size); + +extern void BasicNorm(Tensor *&din, float norm); + +extern void FindMax(float *din, int len, float &max_val, int &max_idx); + +extern void Glu(Tensor *din, Tensor *dout); + +string PathAppend(const string &p1, const string &p2); +bool is_target_file(const std::string& filename, const std::string target); + +void KeepChineseCharacterAndSplit(const std::string &input_str, + std::vector &chinese_characters); +void SplitChiEngCharacters(const std::string &input_str, + std::vector &characters); + +std::vector split(const std::string &s, char delim); + +template +void PrintMat(const std::vector> &mat, const std::string &name); +void Trim(std::string *str); +size_t Utf8ToCharset(const std::string &input, std::vector &output); +void SplitStringToVector(const std::string &full, const char *delim, + bool omit_empty_strings, + std::vector *out); +string PostProcess(std::vector &raw_char, + std::vector> ×tamp_list); +void TimestampOnnx( std::vector& us_alphas, + std::vector us_cif_peak, + std::vector& char_list, + std::string &res_str, + std::vector> ×tamp_vec, + float begin_time = 0.0, + float total_offset = -1.5); +bool IsTargetFile(const std::string& filename, const std::string target); +void ExtractHws(string hws_file, unordered_map &hws_map); +void ExtractHws(string hws_file, unordered_map &hws_map, string& nn_hotwords_); +} // namespace funasr +#endif diff --git a/funasr/runtime/onnxruntime/src/vad-model.cpp b/runtime/onnxruntime/src/vad-model.cpp similarity index 100% rename from funasr/runtime/onnxruntime/src/vad-model.cpp rename to runtime/onnxruntime/src/vad-model.cpp diff --git a/funasr/runtime/onnxruntime/src/vocab.cpp b/runtime/onnxruntime/src/vocab.cpp similarity index 92% rename from funasr/runtime/onnxruntime/src/vocab.cpp rename to runtime/onnxruntime/src/vocab.cpp index 2babc4082..bdedd84f3 100644 --- a/funasr/runtime/onnxruntime/src/vocab.cpp +++ b/runtime/onnxruntime/src/vocab.cpp @@ -52,6 +52,16 @@ void Vocab::Vector2String(vector in, std::vector &preds) } } +string Vocab::Vector2String(vector in) +{ + int i; + stringstream ss; + for (auto it = in.begin(); it != in.end(); it++) { + ss << vocab[*it]; + } + return ss.str(); +} + int Str2Int(string str) { const char *ch_array = str.c_str(); @@ -63,6 +73,16 @@ int Str2Int(string str) return val; } +string Vocab::Id2String(int id) const +{ + if (id < 0 || id >= vocab.size()) { + LOG(INFO) << "Error vocabulary id, this id do not exit."; + return ""; + } else { + return vocab[id]; + } +} + bool Vocab::IsChinese(string ch) { if (ch.size() != 3) { @@ -195,7 +215,7 @@ string Vocab::Vector2StringV2(vector in, std::string language) return ss.str(); } -int Vocab::Size() +int Vocab::Size() const { return vocab.size(); } diff --git a/funasr/runtime/onnxruntime/src/vocab.h b/runtime/onnxruntime/src/vocab.h similarity index 86% rename from funasr/runtime/onnxruntime/src/vocab.h rename to runtime/onnxruntime/src/vocab.h index 23b4bd6e9..8834b972d 100644 --- a/funasr/runtime/onnxruntime/src/vocab.h +++ b/runtime/onnxruntime/src/vocab.h @@ -19,10 +19,12 @@ class Vocab { public: Vocab(const char *filename); ~Vocab(); - int Size(); + int Size() const; bool IsChinese(string ch); void Vector2String(vector in, std::vector &preds); + string Vector2String(vector in); string Vector2StringV2(vector in, std::string language=""); + string Id2String(int id) const; string WordFormat(std::string word); int GetIdByToken(const std::string &token); }; diff --git a/runtime/onnxruntime/src/wfst-decoder.cpp b/runtime/onnxruntime/src/wfst-decoder.cpp new file mode 100644 index 000000000..0bd2bff80 --- /dev/null +++ b/runtime/onnxruntime/src/wfst-decoder.cpp @@ -0,0 +1,114 @@ +#include +namespace funasr { +WfstDecoder::WfstDecoder(fst::Fst* lm, + PhoneSet* phone_set, Vocab* vocab, + float glob_beam, float lat_beam, float am_scale) +:dec_opts_(glob_beam, lat_beam, am_scale), decodable_(dec_opts_.acoustic_scale), + lm_(lm), phone_set_(phone_set), vocab_(vocab) { + decoder_ = std::shared_ptr( + new kaldi::LatticeFasterOnlineDecoder(*lm_, dec_opts_)); +} + +WfstDecoder::~WfstDecoder() { +} + +void WfstDecoder::StartUtterance() { + if (decoder_) { + cur_frame_ = 0; + cur_token_ = 0; + decodable_.Reset(); + decoder_->InitDecoding(); + } +} + +void WfstDecoder::EndUtterance() { +} + +string WfstDecoder::Search(float *in, int len, int64_t token_num) { + string result; + if (len == 0) { + return ""; + } + std::vector> logp_vec; + int blk_phn_id = phone_set_->GetBlkPhnId(); + for (int i = 0; i < len - 1; i++) { + std::vector tmp_logp; + for (int j = 0; j < token_num; j++) { + tmp_logp.push_back((in + i * token_num)[j]); + } + logp_vec.push_back(tmp_logp); + } + for (int i = 0; i < logp_vec.size(); i++) { + cur_frame_++; + decodable_.AcceptLoglikes(logp_vec[i]); + decoder_->AdvanceDecoding(&decodable_, 1); + cur_token_++; + } + if (cur_token_ > 0) { + std::vector words; + kaldi::Lattice lattice; + decoder_->GetBestPath(&lattice, false); + std::vector alignment; + kaldi::LatticeWeight weight; + fst::GetLinearSymbolSequence(lattice, &alignment, &words, &weight); + result = vocab_->Vector2StringV2(words); + } + return result; +} + +string WfstDecoder::FinalizeDecode(bool is_stamp, std::vector us_alphas, std::vector us_cif_peak) { + string result; + if (cur_token_ > 0) { + std::vector words; + kaldi::Lattice lattice; + decodable_.SetFinished(); + decoder_->FinalizeDecoding(); + decoder_->GetBestPath(&lattice, true); + std::vector alignment; + kaldi::LatticeWeight weight; + fst::GetLinearSymbolSequence(lattice, &alignment, &words, &weight); + + if(!is_stamp){ + return vocab_->Vector2StringV2(words); + }else{ + std::vector char_list; + std::vector> timestamp_list; + std::string res_str; + vocab_->Vector2String(words, char_list); + // split chinese word to char + std::vector split_chars; + for(auto& word:char_list){ + std::vector word2char; + SplitChiEngCharacters(word, word2char); + split_chars.insert(split_chars.end(), word2char.begin(), word2char.end()); + } + // std::vector raw_char(char_list); + TimestampOnnx(us_alphas, us_cif_peak, split_chars, res_str, timestamp_list); + + return PostProcess(split_chars, timestamp_list); + } + } + return result; +} + +void WfstDecoder::LoadHwsRes(int inc_bias, unordered_map &hws_map) { + try { + if (!hws_map.empty()) { + bias_lm_ = std::make_shared(hws_map, inc_bias, + *phone_set_, *vocab_); + decoder_->SetBiasLm(bias_lm_); + } + } catch (std::exception const &e) { + LOG(ERROR) << "Error when load wfst hotwords resource: " << e.what(); + exit(0); + } +} + +void WfstDecoder::UnloadHwsRes() { + if (bias_lm_) { + decoder_->ClearBiasLm(); + bias_lm_.reset(); + } +} + +} // namespace funasr diff --git a/runtime/onnxruntime/src/wfst-decoder.h b/runtime/onnxruntime/src/wfst-decoder.h new file mode 100644 index 000000000..1acf6d89c --- /dev/null +++ b/runtime/onnxruntime/src/wfst-decoder.h @@ -0,0 +1,86 @@ +#ifndef WFST_DECODER_ +#define WFST_DECODER_ +#include "kaldi/decoder/lattice-faster-online-decoder.h" +#include "model.h" +#include "fst/fstlib.h" +#include "fst/symbol-table.h" +#include "bias-lm.h" +#include "phone-set.h" +#include "util.h" + +#define MAX_SCORE 10.0f +namespace funasr { +class Decodable : public kaldi::DecodableInterface { + public: + Decodable(float scale = 1.0f) : scale_(scale) { + Reset(); + } + void Reset() { + num_frames_ = 0; + finished_ = false; + logp_.clear(); + } + + int NumFramesReady() const { return num_frames_; } + + bool IsLastFrame(int frame) const { + return finished_ && (frame == num_frames_ - 1); + } + + float LogLikelihood(int frm, int id) { + CHECK_GT(id, 0); + CHECK_LT(frm, num_frames_); + return scale_ * logp_[id - 1]; + } + + void AcceptLoglikes(const std::vector& logp) { + num_frames_++; + logp_ = logp; + } + + int NumIndices() const { return 0; } + void SetFinished() { finished_ = true; } + + private: + int num_frames_ = 0; + float scale_ = 1.0f; + bool finished_ = false; + std::vector logp_; +}; + +struct DecodeOptions : public kaldi::LatticeFasterDecoderConfig { + DecodeOptions(float glob_beam = 3.0f, float lat_beam = 3.0f, float ac_sc = 10.0f) : + kaldi::LatticeFasterDecoderConfig(glob_beam, lat_beam), acoustic_scale(ac_sc) { + } + float acoustic_scale; +}; + +class WfstDecoder { + public: + WfstDecoder(fst::Fst* lm, + PhoneSet* phone_set, + Vocab* vocab, + float glob_beam, + float lat_beam, + float am_scale); + ~WfstDecoder(); + void StartUtterance(); + void EndUtterance(); + string Search(float *in, int len, int64_t token_nums); + string FinalizeDecode(bool is_stamp=false, std::vector us_alphas={0}, std::vector us_cif_peak={0}); + void LoadHwsRes(int inc_bias, unordered_map &hws_map); + void UnloadHwsRes(); + + private: + Vocab* vocab_ = nullptr; + PhoneSet* phone_set_ = nullptr; + int cur_frame_ = 0; + int cur_token_ = 0; + DecodeOptions dec_opts_; + Decodable decodable_; + fst::Fst* lm_ = nullptr; + std::shared_ptr decoder_ = nullptr; + std::shared_ptr bias_lm_ = nullptr; +}; +} // namespace funasr +#endif // WFST_DECODER_ diff --git a/funasr/runtime/onnxruntime/third_party/download_ffmpeg.sh b/runtime/onnxruntime/third_party/download_ffmpeg.sh similarity index 100% rename from funasr/runtime/onnxruntime/third_party/download_ffmpeg.sh rename to runtime/onnxruntime/third_party/download_ffmpeg.sh diff --git a/funasr/runtime/onnxruntime/third_party/download_onnxruntime.sh b/runtime/onnxruntime/third_party/download_onnxruntime.sh similarity index 100% rename from funasr/runtime/onnxruntime/third_party/download_onnxruntime.sh rename to runtime/onnxruntime/third_party/download_onnxruntime.sh diff --git a/funasr/runtime/onnxruntime/third_party/gflags/.gitattributes b/runtime/onnxruntime/third_party/gflags/.gitattributes similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/.gitattributes rename to runtime/onnxruntime/third_party/gflags/.gitattributes diff --git a/funasr/runtime/onnxruntime/third_party/gflags/.gitmodules b/runtime/onnxruntime/third_party/gflags/.gitmodules similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/.gitmodules rename to runtime/onnxruntime/third_party/gflags/.gitmodules diff --git a/funasr/runtime/onnxruntime/third_party/gflags/.travis.yml b/runtime/onnxruntime/third_party/gflags/.travis.yml similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/.travis.yml rename to runtime/onnxruntime/third_party/gflags/.travis.yml diff --git a/funasr/runtime/onnxruntime/third_party/gflags/AUTHORS.txt b/runtime/onnxruntime/third_party/gflags/AUTHORS.txt similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/AUTHORS.txt rename to runtime/onnxruntime/third_party/gflags/AUTHORS.txt diff --git a/funasr/runtime/onnxruntime/third_party/gflags/CMakeLists.txt b/runtime/onnxruntime/third_party/gflags/CMakeLists.txt similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/CMakeLists.txt rename to runtime/onnxruntime/third_party/gflags/CMakeLists.txt diff --git a/funasr/runtime/onnxruntime/third_party/gflags/COPYING.txt b/runtime/onnxruntime/third_party/gflags/COPYING.txt similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/COPYING.txt rename to runtime/onnxruntime/third_party/gflags/COPYING.txt diff --git a/funasr/runtime/onnxruntime/third_party/gflags/ChangeLog.txt b/runtime/onnxruntime/third_party/gflags/ChangeLog.txt similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/ChangeLog.txt rename to runtime/onnxruntime/third_party/gflags/ChangeLog.txt diff --git a/funasr/runtime/onnxruntime/third_party/gflags/INSTALL.md b/runtime/onnxruntime/third_party/gflags/INSTALL.md similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/INSTALL.md rename to runtime/onnxruntime/third_party/gflags/INSTALL.md diff --git a/funasr/runtime/onnxruntime/third_party/gflags/README.md b/runtime/onnxruntime/third_party/gflags/README.md similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/README.md rename to runtime/onnxruntime/third_party/gflags/README.md diff --git a/funasr/runtime/onnxruntime/third_party/gflags/WORKSPACE b/runtime/onnxruntime/third_party/gflags/WORKSPACE similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/WORKSPACE rename to runtime/onnxruntime/third_party/gflags/WORKSPACE diff --git a/funasr/runtime/onnxruntime/third_party/gflags/appveyor.yml b/runtime/onnxruntime/third_party/gflags/appveyor.yml similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/appveyor.yml rename to runtime/onnxruntime/third_party/gflags/appveyor.yml diff --git a/funasr/runtime/onnxruntime/third_party/gflags/bazel/gflags.bzl b/runtime/onnxruntime/third_party/gflags/bazel/gflags.bzl similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/bazel/gflags.bzl rename to runtime/onnxruntime/third_party/gflags/bazel/gflags.bzl diff --git a/funasr/runtime/onnxruntime/third_party/gflags/cmake/README_runtime.txt b/runtime/onnxruntime/third_party/gflags/cmake/README_runtime.txt similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/cmake/README_runtime.txt rename to runtime/onnxruntime/third_party/gflags/cmake/README_runtime.txt diff --git a/funasr/runtime/onnxruntime/third_party/gflags/cmake/cmake_uninstall.cmake.in b/runtime/onnxruntime/third_party/gflags/cmake/cmake_uninstall.cmake.in similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/cmake/cmake_uninstall.cmake.in rename to runtime/onnxruntime/third_party/gflags/cmake/cmake_uninstall.cmake.in diff --git a/funasr/runtime/onnxruntime/third_party/gflags/cmake/config.cmake.in b/runtime/onnxruntime/third_party/gflags/cmake/config.cmake.in similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/cmake/config.cmake.in rename to runtime/onnxruntime/third_party/gflags/cmake/config.cmake.in diff --git a/funasr/runtime/onnxruntime/third_party/gflags/cmake/execute_test.cmake b/runtime/onnxruntime/third_party/gflags/cmake/execute_test.cmake similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/cmake/execute_test.cmake rename to runtime/onnxruntime/third_party/gflags/cmake/execute_test.cmake diff --git a/funasr/runtime/onnxruntime/third_party/gflags/cmake/package.cmake.in b/runtime/onnxruntime/third_party/gflags/cmake/package.cmake.in similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/cmake/package.cmake.in rename to runtime/onnxruntime/third_party/gflags/cmake/package.cmake.in diff --git a/funasr/runtime/onnxruntime/third_party/gflags/cmake/package.pc.in b/runtime/onnxruntime/third_party/gflags/cmake/package.pc.in similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/cmake/package.pc.in rename to runtime/onnxruntime/third_party/gflags/cmake/package.pc.in diff --git a/funasr/runtime/onnxruntime/third_party/gflags/cmake/utils.cmake b/runtime/onnxruntime/third_party/gflags/cmake/utils.cmake similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/cmake/utils.cmake rename to runtime/onnxruntime/third_party/gflags/cmake/utils.cmake diff --git a/funasr/runtime/onnxruntime/third_party/gflags/cmake/version.cmake.in b/runtime/onnxruntime/third_party/gflags/cmake/version.cmake.in similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/cmake/version.cmake.in rename to runtime/onnxruntime/third_party/gflags/cmake/version.cmake.in diff --git a/funasr/runtime/onnxruntime/third_party/gflags/src/config.h b/runtime/onnxruntime/third_party/gflags/src/config.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/src/config.h rename to runtime/onnxruntime/third_party/gflags/src/config.h diff --git a/funasr/runtime/onnxruntime/third_party/gflags/src/defines.h.in b/runtime/onnxruntime/third_party/gflags/src/defines.h.in similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/src/defines.h.in rename to runtime/onnxruntime/third_party/gflags/src/defines.h.in diff --git a/funasr/runtime/onnxruntime/third_party/gflags/src/gflags.cc b/runtime/onnxruntime/third_party/gflags/src/gflags.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/src/gflags.cc rename to runtime/onnxruntime/third_party/gflags/src/gflags.cc diff --git a/funasr/runtime/onnxruntime/third_party/gflags/src/gflags.h.in b/runtime/onnxruntime/third_party/gflags/src/gflags.h.in similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/src/gflags.h.in rename to runtime/onnxruntime/third_party/gflags/src/gflags.h.in diff --git a/funasr/runtime/onnxruntime/third_party/gflags/src/gflags_completions.cc b/runtime/onnxruntime/third_party/gflags/src/gflags_completions.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/src/gflags_completions.cc rename to runtime/onnxruntime/third_party/gflags/src/gflags_completions.cc diff --git a/funasr/runtime/onnxruntime/third_party/gflags/src/gflags_completions.h.in b/runtime/onnxruntime/third_party/gflags/src/gflags_completions.h.in similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/src/gflags_completions.h.in rename to runtime/onnxruntime/third_party/gflags/src/gflags_completions.h.in diff --git a/funasr/runtime/onnxruntime/third_party/gflags/src/gflags_completions.sh b/runtime/onnxruntime/third_party/gflags/src/gflags_completions.sh similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/src/gflags_completions.sh rename to runtime/onnxruntime/third_party/gflags/src/gflags_completions.sh diff --git a/funasr/runtime/onnxruntime/third_party/gflags/src/gflags_declare.h.in b/runtime/onnxruntime/third_party/gflags/src/gflags_declare.h.in similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/src/gflags_declare.h.in rename to runtime/onnxruntime/third_party/gflags/src/gflags_declare.h.in diff --git a/funasr/runtime/onnxruntime/third_party/gflags/src/gflags_ns.h.in b/runtime/onnxruntime/third_party/gflags/src/gflags_ns.h.in similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/src/gflags_ns.h.in rename to runtime/onnxruntime/third_party/gflags/src/gflags_ns.h.in diff --git a/funasr/runtime/onnxruntime/third_party/gflags/src/gflags_reporting.cc b/runtime/onnxruntime/third_party/gflags/src/gflags_reporting.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/src/gflags_reporting.cc rename to runtime/onnxruntime/third_party/gflags/src/gflags_reporting.cc diff --git a/funasr/runtime/onnxruntime/third_party/gflags/src/mutex.h b/runtime/onnxruntime/third_party/gflags/src/mutex.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/src/mutex.h rename to runtime/onnxruntime/third_party/gflags/src/mutex.h diff --git a/funasr/runtime/onnxruntime/third_party/gflags/src/util.h b/runtime/onnxruntime/third_party/gflags/src/util.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/src/util.h rename to runtime/onnxruntime/third_party/gflags/src/util.h diff --git a/funasr/runtime/onnxruntime/third_party/gflags/src/windows_port.cc b/runtime/onnxruntime/third_party/gflags/src/windows_port.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/src/windows_port.cc rename to runtime/onnxruntime/third_party/gflags/src/windows_port.cc diff --git a/funasr/runtime/onnxruntime/third_party/gflags/src/windows_port.h b/runtime/onnxruntime/third_party/gflags/src/windows_port.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/src/windows_port.h rename to runtime/onnxruntime/third_party/gflags/src/windows_port.h diff --git a/funasr/runtime/onnxruntime/third_party/gflags/test/CMakeLists.txt b/runtime/onnxruntime/third_party/gflags/test/CMakeLists.txt similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/test/CMakeLists.txt rename to runtime/onnxruntime/third_party/gflags/test/CMakeLists.txt diff --git a/funasr/runtime/onnxruntime/third_party/gflags/test/config/CMakeLists.txt b/runtime/onnxruntime/third_party/gflags/test/config/CMakeLists.txt similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/test/config/CMakeLists.txt rename to runtime/onnxruntime/third_party/gflags/test/config/CMakeLists.txt diff --git a/funasr/runtime/onnxruntime/third_party/gflags/test/config/main.cc b/runtime/onnxruntime/third_party/gflags/test/config/main.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/test/config/main.cc rename to runtime/onnxruntime/third_party/gflags/test/config/main.cc diff --git a/funasr/runtime/onnxruntime/third_party/gflags/test/flagfile.1 b/runtime/onnxruntime/third_party/gflags/test/flagfile.1 similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/test/flagfile.1 rename to runtime/onnxruntime/third_party/gflags/test/flagfile.1 diff --git a/funasr/runtime/onnxruntime/third_party/gflags/test/flagfile.2 b/runtime/onnxruntime/third_party/gflags/test/flagfile.2 similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/test/flagfile.2 rename to runtime/onnxruntime/third_party/gflags/test/flagfile.2 diff --git a/funasr/runtime/onnxruntime/third_party/gflags/test/flagfile.3 b/runtime/onnxruntime/third_party/gflags/test/flagfile.3 similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/test/flagfile.3 rename to runtime/onnxruntime/third_party/gflags/test/flagfile.3 diff --git a/funasr/runtime/onnxruntime/third_party/gflags/test/gflags_build.py.in b/runtime/onnxruntime/third_party/gflags/test/gflags_build.py.in similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/test/gflags_build.py.in rename to runtime/onnxruntime/third_party/gflags/test/gflags_build.py.in diff --git a/funasr/runtime/onnxruntime/third_party/gflags/test/gflags_declare_flags.cc b/runtime/onnxruntime/third_party/gflags/test/gflags_declare_flags.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/test/gflags_declare_flags.cc rename to runtime/onnxruntime/third_party/gflags/test/gflags_declare_flags.cc diff --git a/funasr/runtime/onnxruntime/third_party/gflags/test/gflags_declare_test.cc b/runtime/onnxruntime/third_party/gflags/test/gflags_declare_test.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/test/gflags_declare_test.cc rename to runtime/onnxruntime/third_party/gflags/test/gflags_declare_test.cc diff --git a/funasr/runtime/onnxruntime/third_party/gflags/test/gflags_strip_flags_test.cc b/runtime/onnxruntime/third_party/gflags/test/gflags_strip_flags_test.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/test/gflags_strip_flags_test.cc rename to runtime/onnxruntime/third_party/gflags/test/gflags_strip_flags_test.cc diff --git a/funasr/runtime/onnxruntime/third_party/gflags/test/gflags_strip_flags_test.cmake b/runtime/onnxruntime/third_party/gflags/test/gflags_strip_flags_test.cmake similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/test/gflags_strip_flags_test.cmake rename to runtime/onnxruntime/third_party/gflags/test/gflags_strip_flags_test.cmake diff --git a/funasr/runtime/onnxruntime/third_party/gflags/test/gflags_unittest.cc b/runtime/onnxruntime/third_party/gflags/test/gflags_unittest.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/test/gflags_unittest.cc rename to runtime/onnxruntime/third_party/gflags/test/gflags_unittest.cc diff --git a/funasr/runtime/onnxruntime/third_party/gflags/test/gflags_unittest_flagfile b/runtime/onnxruntime/third_party/gflags/test/gflags_unittest_flagfile similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/test/gflags_unittest_flagfile rename to runtime/onnxruntime/third_party/gflags/test/gflags_unittest_flagfile diff --git a/funasr/runtime/onnxruntime/third_party/gflags/test/nc/CMakeLists.txt b/runtime/onnxruntime/third_party/gflags/test/nc/CMakeLists.txt similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/test/nc/CMakeLists.txt rename to runtime/onnxruntime/third_party/gflags/test/nc/CMakeLists.txt diff --git a/funasr/runtime/onnxruntime/third_party/gflags/test/nc/gflags_nc.cc b/runtime/onnxruntime/third_party/gflags/test/nc/gflags_nc.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/gflags/test/nc/gflags_nc.cc rename to runtime/onnxruntime/third_party/gflags/test/nc/gflags_nc.cc diff --git a/funasr/runtime/onnxruntime/third_party/glog/CMakeLists.txt b/runtime/onnxruntime/third_party/glog/CMakeLists.txt similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/CMakeLists.txt rename to runtime/onnxruntime/third_party/glog/CMakeLists.txt diff --git a/funasr/runtime/onnxruntime/third_party/glog/cmake/DetermineGflagsNamespace.cmake b/runtime/onnxruntime/third_party/glog/cmake/DetermineGflagsNamespace.cmake similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/cmake/DetermineGflagsNamespace.cmake rename to runtime/onnxruntime/third_party/glog/cmake/DetermineGflagsNamespace.cmake diff --git a/funasr/runtime/onnxruntime/third_party/glog/cmake/FindUnwind.cmake b/runtime/onnxruntime/third_party/glog/cmake/FindUnwind.cmake similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/cmake/FindUnwind.cmake rename to runtime/onnxruntime/third_party/glog/cmake/FindUnwind.cmake diff --git a/funasr/runtime/onnxruntime/third_party/glog/cmake/GetCacheVariables.cmake b/runtime/onnxruntime/third_party/glog/cmake/GetCacheVariables.cmake similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/cmake/GetCacheVariables.cmake rename to runtime/onnxruntime/third_party/glog/cmake/GetCacheVariables.cmake diff --git a/funasr/runtime/onnxruntime/third_party/glog/cmake/RunCleanerTest1.cmake b/runtime/onnxruntime/third_party/glog/cmake/RunCleanerTest1.cmake similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/cmake/RunCleanerTest1.cmake rename to runtime/onnxruntime/third_party/glog/cmake/RunCleanerTest1.cmake diff --git a/funasr/runtime/onnxruntime/third_party/glog/cmake/RunCleanerTest2.cmake b/runtime/onnxruntime/third_party/glog/cmake/RunCleanerTest2.cmake similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/cmake/RunCleanerTest2.cmake rename to runtime/onnxruntime/third_party/glog/cmake/RunCleanerTest2.cmake diff --git a/funasr/runtime/onnxruntime/third_party/glog/cmake/RunCleanerTest3.cmake b/runtime/onnxruntime/third_party/glog/cmake/RunCleanerTest3.cmake similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/cmake/RunCleanerTest3.cmake rename to runtime/onnxruntime/third_party/glog/cmake/RunCleanerTest3.cmake diff --git a/funasr/runtime/onnxruntime/third_party/glog/cmake/TestInitPackageConfig.cmake b/runtime/onnxruntime/third_party/glog/cmake/TestInitPackageConfig.cmake similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/cmake/TestInitPackageConfig.cmake rename to runtime/onnxruntime/third_party/glog/cmake/TestInitPackageConfig.cmake diff --git a/funasr/runtime/onnxruntime/third_party/glog/cmake/TestPackageConfig.cmake b/runtime/onnxruntime/third_party/glog/cmake/TestPackageConfig.cmake similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/cmake/TestPackageConfig.cmake rename to runtime/onnxruntime/third_party/glog/cmake/TestPackageConfig.cmake diff --git a/funasr/runtime/onnxruntime/third_party/glog/glog-config.cmake.in b/runtime/onnxruntime/third_party/glog/glog-config.cmake.in similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/glog-config.cmake.in rename to runtime/onnxruntime/third_party/glog/glog-config.cmake.in diff --git a/funasr/runtime/onnxruntime/third_party/glog/glog-modules.cmake.in b/runtime/onnxruntime/third_party/glog/glog-modules.cmake.in similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/glog-modules.cmake.in rename to runtime/onnxruntime/third_party/glog/glog-modules.cmake.in diff --git a/funasr/runtime/onnxruntime/third_party/glog/libglog.pc.in b/runtime/onnxruntime/third_party/glog/libglog.pc.in similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/libglog.pc.in rename to runtime/onnxruntime/third_party/glog/libglog.pc.in diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/base/commandlineflags.h b/runtime/onnxruntime/third_party/glog/src/base/commandlineflags.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/base/commandlineflags.h rename to runtime/onnxruntime/third_party/glog/src/base/commandlineflags.h diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/base/googleinit.h b/runtime/onnxruntime/third_party/glog/src/base/googleinit.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/base/googleinit.h rename to runtime/onnxruntime/third_party/glog/src/base/googleinit.h diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/base/mutex.h b/runtime/onnxruntime/third_party/glog/src/base/mutex.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/base/mutex.h rename to runtime/onnxruntime/third_party/glog/src/base/mutex.h diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/config.h b/runtime/onnxruntime/third_party/glog/src/config.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/config.h rename to runtime/onnxruntime/third_party/glog/src/config.h diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/config.h.cmake.in b/runtime/onnxruntime/third_party/glog/src/config.h.cmake.in similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/config.h.cmake.in rename to runtime/onnxruntime/third_party/glog/src/config.h.cmake.in diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/demangle.cc b/runtime/onnxruntime/third_party/glog/src/demangle.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/demangle.cc rename to runtime/onnxruntime/third_party/glog/src/demangle.cc diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/demangle.h b/runtime/onnxruntime/third_party/glog/src/demangle.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/demangle.h rename to runtime/onnxruntime/third_party/glog/src/demangle.h diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/fuzz_demangle.cc b/runtime/onnxruntime/third_party/glog/src/fuzz_demangle.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/fuzz_demangle.cc rename to runtime/onnxruntime/third_party/glog/src/fuzz_demangle.cc diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/glog/export.h b/runtime/onnxruntime/third_party/glog/src/glog/export.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/glog/export.h rename to runtime/onnxruntime/third_party/glog/src/glog/export.h diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/glog/log_severity.h b/runtime/onnxruntime/third_party/glog/src/glog/log_severity.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/glog/log_severity.h rename to runtime/onnxruntime/third_party/glog/src/glog/log_severity.h diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/glog/logging.h b/runtime/onnxruntime/third_party/glog/src/glog/logging.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/glog/logging.h rename to runtime/onnxruntime/third_party/glog/src/glog/logging.h diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/glog/logging.h.in b/runtime/onnxruntime/third_party/glog/src/glog/logging.h.in similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/glog/logging.h.in rename to runtime/onnxruntime/third_party/glog/src/glog/logging.h.in diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/glog/platform.h b/runtime/onnxruntime/third_party/glog/src/glog/platform.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/glog/platform.h rename to runtime/onnxruntime/third_party/glog/src/glog/platform.h diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/glog/raw_logging.h b/runtime/onnxruntime/third_party/glog/src/glog/raw_logging.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/glog/raw_logging.h rename to runtime/onnxruntime/third_party/glog/src/glog/raw_logging.h diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/glog/raw_logging.h.in b/runtime/onnxruntime/third_party/glog/src/glog/raw_logging.h.in similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/glog/raw_logging.h.in rename to runtime/onnxruntime/third_party/glog/src/glog/raw_logging.h.in diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/glog/stl_logging.h b/runtime/onnxruntime/third_party/glog/src/glog/stl_logging.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/glog/stl_logging.h rename to runtime/onnxruntime/third_party/glog/src/glog/stl_logging.h diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/glog/stl_logging.h.in b/runtime/onnxruntime/third_party/glog/src/glog/stl_logging.h.in similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/glog/stl_logging.h.in rename to runtime/onnxruntime/third_party/glog/src/glog/stl_logging.h.in diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/glog/vlog_is_on.h b/runtime/onnxruntime/third_party/glog/src/glog/vlog_is_on.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/glog/vlog_is_on.h rename to runtime/onnxruntime/third_party/glog/src/glog/vlog_is_on.h diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/glog/vlog_is_on.h.in b/runtime/onnxruntime/third_party/glog/src/glog/vlog_is_on.h.in similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/glog/vlog_is_on.h.in rename to runtime/onnxruntime/third_party/glog/src/glog/vlog_is_on.h.in diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/logging.cc b/runtime/onnxruntime/third_party/glog/src/logging.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/logging.cc rename to runtime/onnxruntime/third_party/glog/src/logging.cc diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/mock-log.h b/runtime/onnxruntime/third_party/glog/src/mock-log.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/mock-log.h rename to runtime/onnxruntime/third_party/glog/src/mock-log.h diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/raw_logging.cc b/runtime/onnxruntime/third_party/glog/src/raw_logging.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/raw_logging.cc rename to runtime/onnxruntime/third_party/glog/src/raw_logging.cc diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/signalhandler.cc b/runtime/onnxruntime/third_party/glog/src/signalhandler.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/signalhandler.cc rename to runtime/onnxruntime/third_party/glog/src/signalhandler.cc diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/stacktrace.h b/runtime/onnxruntime/third_party/glog/src/stacktrace.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/stacktrace.h rename to runtime/onnxruntime/third_party/glog/src/stacktrace.h diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/stacktrace_generic-inl.h b/runtime/onnxruntime/third_party/glog/src/stacktrace_generic-inl.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/stacktrace_generic-inl.h rename to runtime/onnxruntime/third_party/glog/src/stacktrace_generic-inl.h diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/stacktrace_libunwind-inl.h b/runtime/onnxruntime/third_party/glog/src/stacktrace_libunwind-inl.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/stacktrace_libunwind-inl.h rename to runtime/onnxruntime/third_party/glog/src/stacktrace_libunwind-inl.h diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/stacktrace_powerpc-inl.h b/runtime/onnxruntime/third_party/glog/src/stacktrace_powerpc-inl.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/stacktrace_powerpc-inl.h rename to runtime/onnxruntime/third_party/glog/src/stacktrace_powerpc-inl.h diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/stacktrace_unittest.cc b/runtime/onnxruntime/third_party/glog/src/stacktrace_unittest.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/stacktrace_unittest.cc rename to runtime/onnxruntime/third_party/glog/src/stacktrace_unittest.cc diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/stacktrace_unwind-inl.h b/runtime/onnxruntime/third_party/glog/src/stacktrace_unwind-inl.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/stacktrace_unwind-inl.h rename to runtime/onnxruntime/third_party/glog/src/stacktrace_unwind-inl.h diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/stacktrace_windows-inl.h b/runtime/onnxruntime/third_party/glog/src/stacktrace_windows-inl.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/stacktrace_windows-inl.h rename to runtime/onnxruntime/third_party/glog/src/stacktrace_windows-inl.h diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/stacktrace_x86-inl.h b/runtime/onnxruntime/third_party/glog/src/stacktrace_x86-inl.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/stacktrace_x86-inl.h rename to runtime/onnxruntime/third_party/glog/src/stacktrace_x86-inl.h diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/symbolize.cc b/runtime/onnxruntime/third_party/glog/src/symbolize.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/symbolize.cc rename to runtime/onnxruntime/third_party/glog/src/symbolize.cc diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/symbolize.h b/runtime/onnxruntime/third_party/glog/src/symbolize.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/symbolize.h rename to runtime/onnxruntime/third_party/glog/src/symbolize.h diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/utilities.cc b/runtime/onnxruntime/third_party/glog/src/utilities.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/utilities.cc rename to runtime/onnxruntime/third_party/glog/src/utilities.cc diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/utilities.h b/runtime/onnxruntime/third_party/glog/src/utilities.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/utilities.h rename to runtime/onnxruntime/third_party/glog/src/utilities.h diff --git a/funasr/runtime/onnxruntime/third_party/glog/src/vlog_is_on.cc b/runtime/onnxruntime/third_party/glog/src/vlog_is_on.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/glog/src/vlog_is_on.cc rename to runtime/onnxruntime/third_party/glog/src/vlog_is_on.cc diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/DictTrie.hpp b/runtime/onnxruntime/third_party/jieba/include/cppjieba/DictTrie.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/DictTrie.hpp rename to runtime/onnxruntime/third_party/jieba/include/cppjieba/DictTrie.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/FullSegment.hpp b/runtime/onnxruntime/third_party/jieba/include/cppjieba/FullSegment.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/FullSegment.hpp rename to runtime/onnxruntime/third_party/jieba/include/cppjieba/FullSegment.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/HMMModel.hpp b/runtime/onnxruntime/third_party/jieba/include/cppjieba/HMMModel.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/HMMModel.hpp rename to runtime/onnxruntime/third_party/jieba/include/cppjieba/HMMModel.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/HMMSegment.hpp b/runtime/onnxruntime/third_party/jieba/include/cppjieba/HMMSegment.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/HMMSegment.hpp rename to runtime/onnxruntime/third_party/jieba/include/cppjieba/HMMSegment.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/Jieba.hpp b/runtime/onnxruntime/third_party/jieba/include/cppjieba/Jieba.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/Jieba.hpp rename to runtime/onnxruntime/third_party/jieba/include/cppjieba/Jieba.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/KeywordExtractor.hpp b/runtime/onnxruntime/third_party/jieba/include/cppjieba/KeywordExtractor.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/KeywordExtractor.hpp rename to runtime/onnxruntime/third_party/jieba/include/cppjieba/KeywordExtractor.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/MPSegment.hpp b/runtime/onnxruntime/third_party/jieba/include/cppjieba/MPSegment.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/MPSegment.hpp rename to runtime/onnxruntime/third_party/jieba/include/cppjieba/MPSegment.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/MixSegment.hpp b/runtime/onnxruntime/third_party/jieba/include/cppjieba/MixSegment.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/MixSegment.hpp rename to runtime/onnxruntime/third_party/jieba/include/cppjieba/MixSegment.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/PosTagger.hpp b/runtime/onnxruntime/third_party/jieba/include/cppjieba/PosTagger.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/PosTagger.hpp rename to runtime/onnxruntime/third_party/jieba/include/cppjieba/PosTagger.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/PreFilter.hpp b/runtime/onnxruntime/third_party/jieba/include/cppjieba/PreFilter.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/PreFilter.hpp rename to runtime/onnxruntime/third_party/jieba/include/cppjieba/PreFilter.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/QuerySegment.hpp b/runtime/onnxruntime/third_party/jieba/include/cppjieba/QuerySegment.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/QuerySegment.hpp rename to runtime/onnxruntime/third_party/jieba/include/cppjieba/QuerySegment.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/SegmentBase.hpp b/runtime/onnxruntime/third_party/jieba/include/cppjieba/SegmentBase.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/SegmentBase.hpp rename to runtime/onnxruntime/third_party/jieba/include/cppjieba/SegmentBase.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/SegmentTagged.hpp b/runtime/onnxruntime/third_party/jieba/include/cppjieba/SegmentTagged.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/SegmentTagged.hpp rename to runtime/onnxruntime/third_party/jieba/include/cppjieba/SegmentTagged.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/TextRankExtractor.hpp b/runtime/onnxruntime/third_party/jieba/include/cppjieba/TextRankExtractor.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/TextRankExtractor.hpp rename to runtime/onnxruntime/third_party/jieba/include/cppjieba/TextRankExtractor.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/Trie.hpp b/runtime/onnxruntime/third_party/jieba/include/cppjieba/Trie.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/Trie.hpp rename to runtime/onnxruntime/third_party/jieba/include/cppjieba/Trie.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/Unicode.hpp b/runtime/onnxruntime/third_party/jieba/include/cppjieba/Unicode.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/cppjieba/Unicode.hpp rename to runtime/onnxruntime/third_party/jieba/include/cppjieba/Unicode.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/limonp/ArgvContext.hpp b/runtime/onnxruntime/third_party/jieba/include/limonp/ArgvContext.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/limonp/ArgvContext.hpp rename to runtime/onnxruntime/third_party/jieba/include/limonp/ArgvContext.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/limonp/BlockingQueue.hpp b/runtime/onnxruntime/third_party/jieba/include/limonp/BlockingQueue.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/limonp/BlockingQueue.hpp rename to runtime/onnxruntime/third_party/jieba/include/limonp/BlockingQueue.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/limonp/BoundedBlockingQueue.hpp b/runtime/onnxruntime/third_party/jieba/include/limonp/BoundedBlockingQueue.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/limonp/BoundedBlockingQueue.hpp rename to runtime/onnxruntime/third_party/jieba/include/limonp/BoundedBlockingQueue.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/limonp/BoundedQueue.hpp b/runtime/onnxruntime/third_party/jieba/include/limonp/BoundedQueue.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/limonp/BoundedQueue.hpp rename to runtime/onnxruntime/third_party/jieba/include/limonp/BoundedQueue.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/limonp/Closure.hpp b/runtime/onnxruntime/third_party/jieba/include/limonp/Closure.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/limonp/Closure.hpp rename to runtime/onnxruntime/third_party/jieba/include/limonp/Closure.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/limonp/Colors.hpp b/runtime/onnxruntime/third_party/jieba/include/limonp/Colors.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/limonp/Colors.hpp rename to runtime/onnxruntime/third_party/jieba/include/limonp/Colors.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/limonp/Condition.hpp b/runtime/onnxruntime/third_party/jieba/include/limonp/Condition.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/limonp/Condition.hpp rename to runtime/onnxruntime/third_party/jieba/include/limonp/Condition.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/limonp/Config.hpp b/runtime/onnxruntime/third_party/jieba/include/limonp/Config.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/limonp/Config.hpp rename to runtime/onnxruntime/third_party/jieba/include/limonp/Config.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/limonp/FileLock.hpp b/runtime/onnxruntime/third_party/jieba/include/limonp/FileLock.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/limonp/FileLock.hpp rename to runtime/onnxruntime/third_party/jieba/include/limonp/FileLock.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/limonp/ForcePublic.hpp b/runtime/onnxruntime/third_party/jieba/include/limonp/ForcePublic.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/limonp/ForcePublic.hpp rename to runtime/onnxruntime/third_party/jieba/include/limonp/ForcePublic.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/limonp/LocalVector.hpp b/runtime/onnxruntime/third_party/jieba/include/limonp/LocalVector.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/limonp/LocalVector.hpp rename to runtime/onnxruntime/third_party/jieba/include/limonp/LocalVector.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/limonp/Logging.hpp b/runtime/onnxruntime/third_party/jieba/include/limonp/Logging.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/limonp/Logging.hpp rename to runtime/onnxruntime/third_party/jieba/include/limonp/Logging.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/limonp/Md5.hpp b/runtime/onnxruntime/third_party/jieba/include/limonp/Md5.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/limonp/Md5.hpp rename to runtime/onnxruntime/third_party/jieba/include/limonp/Md5.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/limonp/MutexLock.hpp b/runtime/onnxruntime/third_party/jieba/include/limonp/MutexLock.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/limonp/MutexLock.hpp rename to runtime/onnxruntime/third_party/jieba/include/limonp/MutexLock.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/limonp/NonCopyable.hpp b/runtime/onnxruntime/third_party/jieba/include/limonp/NonCopyable.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/limonp/NonCopyable.hpp rename to runtime/onnxruntime/third_party/jieba/include/limonp/NonCopyable.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/limonp/StdExtension.hpp b/runtime/onnxruntime/third_party/jieba/include/limonp/StdExtension.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/limonp/StdExtension.hpp rename to runtime/onnxruntime/third_party/jieba/include/limonp/StdExtension.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/limonp/StringUtil.hpp b/runtime/onnxruntime/third_party/jieba/include/limonp/StringUtil.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/limonp/StringUtil.hpp rename to runtime/onnxruntime/third_party/jieba/include/limonp/StringUtil.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/limonp/Thread.hpp b/runtime/onnxruntime/third_party/jieba/include/limonp/Thread.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/limonp/Thread.hpp rename to runtime/onnxruntime/third_party/jieba/include/limonp/Thread.hpp diff --git a/funasr/runtime/onnxruntime/third_party/jieba/include/limonp/ThreadPool.hpp b/runtime/onnxruntime/third_party/jieba/include/limonp/ThreadPool.hpp similarity index 100% rename from funasr/runtime/onnxruntime/third_party/jieba/include/limonp/ThreadPool.hpp rename to runtime/onnxruntime/third_party/jieba/include/limonp/ThreadPool.hpp diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/CMakeLists.txt b/runtime/onnxruntime/third_party/kaldi-native-fbank/CMakeLists.txt similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/CMakeLists.txt rename to runtime/onnxruntime/third_party/kaldi-native-fbank/CMakeLists.txt diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/cmake/Modules/FetchContent.cmake b/runtime/onnxruntime/third_party/kaldi-native-fbank/cmake/Modules/FetchContent.cmake similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/cmake/Modules/FetchContent.cmake rename to runtime/onnxruntime/third_party/kaldi-native-fbank/cmake/Modules/FetchContent.cmake diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/cmake/Modules/FetchContent/CMakeLists.cmake.in b/runtime/onnxruntime/third_party/kaldi-native-fbank/cmake/Modules/FetchContent/CMakeLists.cmake.in similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/cmake/Modules/FetchContent/CMakeLists.cmake.in rename to runtime/onnxruntime/third_party/kaldi-native-fbank/cmake/Modules/FetchContent/CMakeLists.cmake.in diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/cmake/Modules/README.md b/runtime/onnxruntime/third_party/kaldi-native-fbank/cmake/Modules/README.md similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/cmake/Modules/README.md rename to runtime/onnxruntime/third_party/kaldi-native-fbank/cmake/Modules/README.md diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/cmake/cmake_extension.py b/runtime/onnxruntime/third_party/kaldi-native-fbank/cmake/cmake_extension.py similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/cmake/cmake_extension.py rename to runtime/onnxruntime/third_party/kaldi-native-fbank/cmake/cmake_extension.py diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/cmake/googletest.cmake b/runtime/onnxruntime/third_party/kaldi-native-fbank/cmake/googletest.cmake similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/cmake/googletest.cmake rename to runtime/onnxruntime/third_party/kaldi-native-fbank/cmake/googletest.cmake diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/cmake/pybind11.cmake b/runtime/onnxruntime/third_party/kaldi-native-fbank/cmake/pybind11.cmake similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/cmake/pybind11.cmake rename to runtime/onnxruntime/third_party/kaldi-native-fbank/cmake/pybind11.cmake diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/CMakeLists.txt b/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/CMakeLists.txt similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/CMakeLists.txt rename to runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/CMakeLists.txt diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/CMakeLists.txt b/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/CMakeLists.txt similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/CMakeLists.txt rename to runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/CMakeLists.txt diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/CMakeLists.txt.bak b/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/CMakeLists.txt.bak similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/CMakeLists.txt.bak rename to runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/CMakeLists.txt.bak diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-fbank.cc b/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-fbank.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-fbank.cc rename to runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-fbank.cc diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-fbank.h b/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-fbank.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-fbank.h rename to runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-fbank.h diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-functions.cc b/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-functions.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-functions.cc rename to runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-functions.cc diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-functions.h b/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-functions.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-functions.h rename to runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-functions.h diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-window.cc b/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-window.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-window.cc rename to runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-window.cc diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-window.h b/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-window.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-window.h rename to runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/feature-window.h diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/fftsg.c b/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/fftsg.c similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/fftsg.c rename to runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/fftsg.c diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/log.cc b/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/log.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/log.cc rename to runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/log.cc diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/log.h b/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/log.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/log.h rename to runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/log.h diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/mel-computations.cc b/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/mel-computations.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/mel-computations.cc rename to runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/mel-computations.cc diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/mel-computations.h b/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/mel-computations.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/mel-computations.h rename to runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/mel-computations.h diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/online-feature.cc b/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/online-feature.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/online-feature.cc rename to runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/online-feature.cc diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/online-feature.h b/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/online-feature.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/online-feature.h rename to runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/online-feature.h diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/rfft.cc b/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/rfft.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/rfft.cc rename to runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/rfft.cc diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/rfft.h b/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/rfft.h similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/rfft.h rename to runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/rfft.h diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/test-log.cc b/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/test-log.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/test-log.cc rename to runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/test-log.cc diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/test-online-fbank.cc b/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/test-online-fbank.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/test-online-fbank.cc rename to runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/test-online-fbank.cc diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/test-online-feature.cc b/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/test-online-feature.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/test-online-feature.cc rename to runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/test-online-feature.cc diff --git a/funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/test-rfft.cc b/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/test-rfft.cc similarity index 100% rename from funasr/runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/test-rfft.cc rename to runtime/onnxruntime/third_party/kaldi-native-fbank/kaldi-native-fbank/csrc/test-rfft.cc diff --git a/runtime/onnxruntime/third_party/kaldi/CMakeLists.txt b/runtime/onnxruntime/third_party/kaldi/CMakeLists.txt new file mode 100644 index 000000000..1ff94d8af --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/CMakeLists.txt @@ -0,0 +1,51 @@ +cmake_minimum_required(VERSION 3.10 FATAL_ERROR) + +project(kaldi) + +add_library(kaldi-util + base/kaldi-error.cc + base/kaldi-math.cc + util/kaldi-io.cc + util/parse-options.cc + util/simple-io-funcs.cc + util/text-utils.cc +) +#target_link_libraries(kaldi-util PUBLIC utils) + +add_library(kaldi-decoder + lat/determinize-lattice-pruned.cc + lat/lattice-functions.cc + decoder/lattice-faster-decoder.cc + decoder/lattice-faster-online-decoder.cc +) +target_link_libraries(kaldi-decoder PUBLIC kaldi-util dl) + +include_directories(${CMAKE_SOURCE_DIR}/build/third_party/glog) +include_directories(${CMAKE_SOURCE_DIR}/third_party/glog/src) +include_directories(${CMAKE_SOURCE_DIR}/third_party/gflags/src/include) +if(TRUE) + # Arpa binary + add_executable(arpa2fst + lm/arpa-file-parser.cc + lm/arpa-lm-compiler.cc + lmbin/arpa2fst.cc + ) + target_link_libraries(arpa2fst PUBLIC kaldi-util fst dl) + + # FST tools binary + set(FST_BINS + fstaddselfloops + fstdeterminizestar + fstisstochastic + fstminimizeencoded + fsttablecompose + ) + + foreach(name IN LISTS FST_BINS) + add_executable(${name} + fstbin/${name}.cc + fstext/kaldi-fst-io.cc + ) + target_link_libraries(${name} PUBLIC kaldi-util fst dl) + endforeach() +endif() diff --git a/runtime/onnxruntime/third_party/kaldi/base/Makefile b/runtime/onnxruntime/third_party/kaldi/base/Makefile new file mode 100644 index 000000000..49af4f87f --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/base/Makefile @@ -0,0 +1,27 @@ +# Whenever make is run in this directory, call ./get_version.sh as the +# first thing. This script regenereates ./version.h if necessary, e.g. +# if it does not already exist or if the version number has changed. +LOG := $(shell ./get_version.sh; echo " $$?") +ifneq ($(strip $(LOG)), 0) + RC := $(lastword $(LOG)) + OUT := $(wordlist 1,$(shell echo $$(($(words $(LOG))-1))),$(LOG)) + ifeq ($(RC),0) + $(info $(OUT)) + else + $(error $(OUT)) + endif +endif + +all: + +include ../kaldi.mk + +TESTFILES = kaldi-math-test io-funcs-test kaldi-error-test timer-test + +OBJFILES = kaldi-math.o kaldi-error.o io-funcs.o kaldi-utils.o timer.o + +LIBNAME = kaldi-base + +ADDLIBS = + +include ../makefiles/default_rules.mk diff --git a/runtime/onnxruntime/third_party/kaldi/base/get_version.sh b/runtime/onnxruntime/third_party/kaldi/base/get_version.sh new file mode 100755 index 000000000..610bc2208 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/base/get_version.sh @@ -0,0 +1,94 @@ +#!/usr/bin/env bash + +# Copyright 2017 University of Southern California (Author: Dogan Can) + +# See ../../COPYING for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +# WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +# MERCHANTABLITY OR NON-INFRINGEMENT. +# See the Apache 2 License for the specific language governing permissions and +# limitations under the License. + + +# Kaldi versioning is loosely based on the semantic versioning scheme. This +# script tries to work out the version string from the partial version number +# specified in src/.version along with the recent git history. By convention +# src/.version specifies the first two components (MAJOR.MINOR) of the version +# number. The third component (PATCH) is determined by counting how many +# commits there are that are newer than than the last commit modifiying +# src/.version. If there are uncommitted changes in the src/ directory, then +# the version string is extended with a suffix (~N) specifiying the number of +# files with uncommitted changes. The last component of the version string is +# the abbreviated hash of the HEAD commit. If git history is not available or +# if the file src/.short_version exists, then the version string defaults to +# the number specified in src/.version. + +set -e + +# Change working directory to the directory where this script is located. +cd "$(dirname ${BASH_SOURCE[0]})" + +# Read the partial version number specified in the first line of src/.version. +version=$(head -1 ../.version) + +if [ -e ../.short_version ]; then + echo "$0: File src/.short_version exists." + echo "$0: Stopping the construction of full version number from git history." +elif ! [[ $version =~ ^[0-9][0-9]*.[0-9][0-9]*$ ]]; then + echo "$0: The version number \"$version\" specified in src/.version is not" \ + "in MAJOR.MINOR format." + echo "$0: Stopping the construction of full version number from git history." +elif ! which git >&/dev/null; then + echo "$0: Git is not installed." + echo "$0: Using the version number \"$version\" specified in src/.version." +elif [ "$(git rev-parse --is-inside-work-tree 2>/dev/null)" != true ]; then + echo "$0: Git history is not available." + echo "$0: Using the version number \"$version\" specified in src/.version." +else + # Figure out patch number. + version_commit=$(git log -1 --pretty=oneline ../.version | awk '{print $1}') + patch_number=$(git rev-list ${version_commit}..HEAD | wc -l | awk '{print $1}') + version="$version.$patch_number" + + # Check for uncommitted changes in src/. + uncommitted_changes=$(git diff-index HEAD -- .. | wc -l | awk '{print $1}') + if [ $uncommitted_changes -gt 0 ]; then + # Add suffix ~N if there are N files in src/ with uncommitted changes + version="$version~$uncommitted_changes" + fi + + # Figure out HEAD commit SHA-1. + head_commit=$(git log -1 --pretty=oneline | awk '{print $1}') + head_commit_short=$(git log -1 --oneline --abbrev=4 | awk '{print $1}') + version="$version-${head_commit_short}" +fi + +# Empty version number is not allowed. +if [ -z "$version" ]; then + version="?" +fi + +# Write version info to a temporary file. +temp=$(mktemp /tmp/temp.XXXXXX) +trap 'rm -f "$temp"' EXIT +echo "// This file was automatically created by ./get_version.sh." > $temp +echo "// It is only included by ./kaldi-error.cc." >> $temp +echo "#define KALDI_VERSION \"$version\"" >> $temp +if [ -n "$head_commit" ]; then + echo "#define KALDI_GIT_HEAD \"$head_commit\"" >> $temp +fi + +# Overwrite ./version.h with the temporary file if they are different. +if ! cmp -s $temp version.h; then + cp $temp version.h + chmod 644 version.h +fi +exit 0 diff --git a/runtime/onnxruntime/third_party/kaldi/base/io-funcs-inl.h b/runtime/onnxruntime/third_party/kaldi/base/io-funcs-inl.h new file mode 100644 index 000000000..b703ef5ad --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/base/io-funcs-inl.h @@ -0,0 +1,327 @@ +// base/io-funcs-inl.h + +// Copyright 2009-2011 Microsoft Corporation; Saarland University; +// Jan Silovsky; Yanmin Qian; +// Johns Hopkins University (Author: Daniel Povey) +// 2016 Xiaohui Zhang + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#ifndef KALDI_BASE_IO_FUNCS_INL_H_ +#define KALDI_BASE_IO_FUNCS_INL_H_ 1 + +// Do not include this file directly. It is included by base/io-funcs.h + +#include +#include + +namespace kaldi { + +// Template that covers integers. +template void WriteBasicType(std::ostream &os, + bool binary, T t) { + // Compile time assertion that this is not called with a wrong type. + KALDI_ASSERT_IS_INTEGER_TYPE(T); + if (binary) { + char len_c = (std::numeric_limits::is_signed ? 1 : -1) + * static_cast(sizeof(t)); + os.put(len_c); + os.write(reinterpret_cast(&t), sizeof(t)); + } else { + if (sizeof(t) == 1) + os << static_cast(t) << " "; + else + os << t << " "; + } + if (os.fail()) { + KALDI_ERR << "Write failure in WriteBasicType."; + } +} + +// Template that covers integers. +template inline void ReadBasicType(std::istream &is, + bool binary, T *t) { + KALDI_PARANOID_ASSERT(t != NULL); + // Compile time assertion that this is not called with a wrong type. + KALDI_ASSERT_IS_INTEGER_TYPE(T); + if (binary) { + int len_c_in = is.get(); + if (len_c_in == -1) + KALDI_ERR << "ReadBasicType: encountered end of stream."; + char len_c = static_cast(len_c_in), len_c_expected + = (std::numeric_limits::is_signed ? 1 : -1) + * static_cast(sizeof(*t)); + if (len_c != len_c_expected) { + KALDI_ERR << "ReadBasicType: did not get expected integer type, " + << static_cast(len_c) + << " vs. " << static_cast(len_c_expected) + << ". You can change this code to successfully" + << " read it later, if needed."; + // insert code here to read "wrong" type. Might have a switch statement. + } + is.read(reinterpret_cast(t), sizeof(*t)); + } else { + if (sizeof(*t) == 1) { + int16 i; + is >> i; + *t = i; + } else { + is >> *t; + } + } + if (is.fail()) { + KALDI_ERR << "Read failure in ReadBasicType, file position is " + << is.tellg() << ", next char is " << is.peek(); + } +} + +// Template that covers integers. +template +inline void WriteIntegerPairVector(std::ostream &os, bool binary, + const std::vector > &v) { + // Compile time assertion that this is not called with a wrong type. + KALDI_ASSERT_IS_INTEGER_TYPE(T); + if (binary) { + char sz = sizeof(T); // this is currently just a check. + os.write(&sz, 1); + int32 vecsz = static_cast(v.size()); + KALDI_ASSERT((size_t)vecsz == v.size()); + os.write(reinterpret_cast(&vecsz), sizeof(vecsz)); + if (vecsz != 0) { + os.write(reinterpret_cast(&(v[0])), sizeof(T) * vecsz * 2); + } + } else { + // focus here is on prettiness of text form rather than + // efficiency of reading-in. + // reading-in is dominated by low-level operations anyway: + // for efficiency use binary. + os << "[ "; + typename std::vector >::const_iterator iter = v.begin(), + end = v.end(); + for (; iter != end; ++iter) { + if (sizeof(T) == 1) + os << static_cast(iter->first) << ',' + << static_cast(iter->second) << ' '; + else + os << iter->first << ',' + << iter->second << ' '; + } + os << "]\n"; + } + if (os.fail()) { + KALDI_ERR << "Write failure in WriteIntegerPairVector."; + } +} + +// Template that covers integers. +template +inline void ReadIntegerPairVector(std::istream &is, bool binary, + std::vector > *v) { + KALDI_ASSERT_IS_INTEGER_TYPE(T); + KALDI_ASSERT(v != NULL); + if (binary) { + int sz = is.peek(); + if (sz == sizeof(T)) { + is.get(); + } else { // this is currently just a check. + KALDI_ERR << "ReadIntegerPairVector: expected to see type of size " + << sizeof(T) << ", saw instead " << sz << ", at file position " + << is.tellg(); + } + int32 vecsz; + is.read(reinterpret_cast(&vecsz), sizeof(vecsz)); + if (is.fail() || vecsz < 0) goto bad; + v->resize(vecsz); + if (vecsz > 0) { + is.read(reinterpret_cast(&((*v)[0])), sizeof(T)*vecsz*2); + } + } else { + std::vector > tmp_v; // use temporary so v doesn't use extra memory + // due to resizing. + is >> std::ws; + if (is.peek() != static_cast('[')) { + KALDI_ERR << "ReadIntegerPairVector: expected to see [, saw " + << is.peek() << ", at file position " << is.tellg(); + } + is.get(); // consume the '['. + is >> std::ws; // consume whitespace. + while (is.peek() != static_cast(']')) { + if (sizeof(T) == 1) { // read/write chars as numbers. + int16 next_t1, next_t2; + is >> next_t1; + if (is.fail()) goto bad; + if (is.peek() != static_cast(',')) + KALDI_ERR << "ReadIntegerPairVector: expected to see ',', saw " + << is.peek() << ", at file position " << is.tellg(); + is.get(); // consume the ','. + is >> next_t2 >> std::ws; + if (is.fail()) goto bad; + else + tmp_v.push_back(std::make_pair((T)next_t1, (T)next_t2)); + } else { + T next_t1, next_t2; + is >> next_t1; + if (is.fail()) goto bad; + if (is.peek() != static_cast(',')) + KALDI_ERR << "ReadIntegerPairVector: expected to see ',', saw " + << is.peek() << ", at file position " << is.tellg(); + is.get(); // consume the ','. + is >> next_t2 >> std::ws; + if (is.fail()) goto bad; + else + tmp_v.push_back(std::pair(next_t1, next_t2)); + } + } + is.get(); // get the final ']'. + *v = tmp_v; // could use std::swap to use less temporary memory, but this + // uses less permanent memory. + } + if (!is.fail()) return; + bad: + KALDI_ERR << "ReadIntegerPairVector: read failure at file position " + << is.tellg(); +} + +template inline void WriteIntegerVector(std::ostream &os, bool binary, + const std::vector &v) { + // Compile time assertion that this is not called with a wrong type. + KALDI_ASSERT_IS_INTEGER_TYPE(T); + if (binary) { + char sz = sizeof(T); // this is currently just a check. + os.write(&sz, 1); + int32 vecsz = static_cast(v.size()); + KALDI_ASSERT((size_t)vecsz == v.size()); + os.write(reinterpret_cast(&vecsz), sizeof(vecsz)); + if (vecsz != 0) { + os.write(reinterpret_cast(&(v[0])), sizeof(T)*vecsz); + } + } else { + // focus here is on prettiness of text form rather than + // efficiency of reading-in. + // reading-in is dominated by low-level operations anyway: + // for efficiency use binary. + os << "[ "; + typename std::vector::const_iterator iter = v.begin(), end = v.end(); + for (; iter != end; ++iter) { + if (sizeof(T) == 1) + os << static_cast(*iter) << " "; + else + os << *iter << " "; + } + os << "]\n"; + } + if (os.fail()) { + KALDI_ERR << "Write failure in WriteIntegerVector."; + } +} + + +template inline void ReadIntegerVector(std::istream &is, + bool binary, + std::vector *v) { + KALDI_ASSERT_IS_INTEGER_TYPE(T); + KALDI_ASSERT(v != NULL); + if (binary) { + int sz = is.peek(); + if (sz == sizeof(T)) { + is.get(); + } else { // this is currently just a check. + KALDI_ERR << "ReadIntegerVector: expected to see type of size " + << sizeof(T) << ", saw instead " << sz << ", at file position " + << is.tellg(); + } + int32 vecsz; + is.read(reinterpret_cast(&vecsz), sizeof(vecsz)); + if (is.fail() || vecsz < 0) goto bad; + v->resize(vecsz); + if (vecsz > 0) { + is.read(reinterpret_cast(&((*v)[0])), sizeof(T)*vecsz); + } + } else { + std::vector tmp_v; // use temporary so v doesn't use extra memory + // due to resizing. + is >> std::ws; + if (is.peek() != static_cast('[')) { + KALDI_ERR << "ReadIntegerVector: expected to see [, saw " + << is.peek() << ", at file position " << is.tellg(); + } + is.get(); // consume the '['. + is >> std::ws; // consume whitespace. + while (is.peek() != static_cast(']')) { + if (sizeof(T) == 1) { // read/write chars as numbers. + int16 next_t; + is >> next_t >> std::ws; + if (is.fail()) goto bad; + else + tmp_v.push_back((T)next_t); + } else { + T next_t; + is >> next_t >> std::ws; + if (is.fail()) goto bad; + else + tmp_v.push_back(next_t); + } + } + is.get(); // get the final ']'. + *v = tmp_v; // could use std::swap to use less temporary memory, but this + // uses less permanent memory. + } + if (!is.fail()) return; + bad: + KALDI_ERR << "ReadIntegerVector: read failure at file position " + << is.tellg(); +} + + +// Initialize an opened stream for writing by writing an optional binary +// header and modifying the floating-point precision. +inline void InitKaldiOutputStream(std::ostream &os, bool binary) { + // This does not throw exceptions (does not check for errors). + if (binary) { + os.put('\0'); + os.put('B'); + } + // Note, in non-binary mode we may at some point want to mess with + // the precision a bit. + // 7 is a bit more than the precision of float.. + if (os.precision() < 7) + os.precision(7); +} + +/// Initialize an opened stream for reading by detecting the binary header and +// setting the "binary" value appropriately. +inline bool InitKaldiInputStream(std::istream &is, bool *binary) { + // Sets the 'binary' variable. + // Throws exception in the very unusual situation that stream + // starts with '\0' but not then 'B'. + + if (is.peek() == '\0') { // seems to be binary + is.get(); + if (is.peek() != 'B') { + return false; + } + is.get(); + *binary = true; + return true; + } else { + *binary = false; + return true; + } +} + +} // end namespace kaldi. + +#endif // KALDI_BASE_IO_FUNCS_INL_H_ diff --git a/runtime/onnxruntime/third_party/kaldi/base/io-funcs-test.cc b/runtime/onnxruntime/third_party/kaldi/base/io-funcs-test.cc new file mode 100644 index 000000000..dd05326d5 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/base/io-funcs-test.cc @@ -0,0 +1,160 @@ +// base/io-funcs-test.cc + +// Copyright 2009-2011 Microsoft Corporation + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. +#include "base/io-funcs.h" +#include "base/kaldi-math.h" + +namespace kaldi { + +void UnitTestIo(bool binary) { + { + const char *filename = "tmpf"; + std::ofstream outfile(filename, std::ios_base::out | std::ios_base::binary); + InitKaldiOutputStream(outfile, binary); + if (!binary) outfile << "\t"; + int64 i1 = Rand() % 10000; + WriteBasicType(outfile, binary, i1); + uint16 i2 = Rand() % 10000; + WriteBasicType(outfile, binary, i2); + if (!binary) outfile << "\t"; + char c = Rand(); + WriteBasicType(outfile, binary, c); + if (!binary && Rand()%2 == 0) outfile << " \n"; + std::vector vec1; + WriteIntegerVector(outfile, binary, vec1); + if (!binary && Rand()%2 == 0) outfile << " \n"; + std::vector vec2; + for (size_t i = 0; i < 10; i++) vec2.push_back(Rand()%100 - 10); + WriteIntegerVector(outfile, binary, vec2); + if (!binary) outfile << " \n"; + std::vector vec3; + + int32 size = RandInt(0, 10); + for (size_t i = 0; i < size; i++) vec3.push_back(Rand()%100); + WriteIntegerVector(outfile, binary, vec3); + std::vector > vec4; + WriteIntegerPairVector(outfile, binary, vec4); + if (!binary && Rand()%2 == 0) outfile << " \n"; + std::vector > vec5; + for (size_t i = 0; i < size; i++) vec5.push_back(std::make_pair(Rand()%100 - 10, Rand()%100 - 10)); + WriteIntegerPairVector(outfile, binary, vec5); + if (!binary) outfile << " \n"; + std::vector > vec6; + for (size_t i = 0; i < size; i++) vec6.push_back(std::make_pair(Rand()%100, Rand()%100)); + WriteIntegerPairVector(outfile, binary, vec6); + if (!binary && Rand()%2 == 0) outfile << " \n"; + const char *token1 = "Hi"; + WriteToken(outfile, binary, token1); + if (!binary) outfile << " \n"; + std::string token2 = "There."; + WriteToken(outfile, binary, token2); + if (!binary && Rand()%2 == 0) outfile << " \n"; + std::string token3 = "You."; + WriteToken(outfile, binary, token3); + if (!binary && Rand()%2 == 0) outfile << " "; + float f1 = RandUniform(); + WriteBasicType(outfile, binary, f1); + if (!binary && Rand()%2 == 0) outfile << "\t"; + float f2 = RandUniform(); + WriteBasicType(outfile, binary, f2); + double d1 = RandUniform(); + WriteBasicType(outfile, binary, d1); + if (!binary && Rand()%2 == 0) outfile << "\t"; + double d2 = RandUniform(); + WriteBasicType(outfile, binary, d2); + if (!binary && Rand()%2 == 0) outfile << "\t"; + outfile.close(); + + { + std::ifstream infile(filename, std::ios_base::in | std::ios_base::binary); + bool binary_in; + InitKaldiInputStream(infile, &binary_in); + int64 i1_in; + ReadBasicType(infile, binary_in, &i1_in); + KALDI_ASSERT(i1_in == i1); + uint16 i2_in; + ReadBasicType(infile, binary_in, &i2_in); + KALDI_ASSERT(i2_in == i2); + char c_in; + ReadBasicType(infile, binary_in, &c_in); + KALDI_ASSERT(c_in == c); + std::vector vec1_in; + ReadIntegerVector(infile, binary_in, &vec1_in); + KALDI_ASSERT(vec1_in == vec1); + std::vector vec2_in; + ReadIntegerVector(infile, binary_in, &vec2_in); + KALDI_ASSERT(vec2_in == vec2); + std::vector vec3_in; + ReadIntegerVector(infile, binary_in, &vec3_in); + KALDI_ASSERT(vec3_in == vec3); + std::vector > vec4_in; + ReadIntegerPairVector(infile, binary_in, &vec4_in); + KALDI_ASSERT(vec4_in == vec4); + std::vector > vec5_in; + ReadIntegerPairVector(infile, binary_in, &vec5_in); + KALDI_ASSERT(vec5_in == vec5); + std::vector > vec6_in; + ReadIntegerPairVector(infile, binary_in, &vec6_in); + KALDI_ASSERT(vec6_in == vec6); + std::string token1_in, token2_in; + KALDI_ASSERT(Peek(infile, binary_in) == static_cast(*token1)); + KALDI_ASSERT(PeekToken(infile, binary_in) == static_cast(*token1)); + // Note: + // the stuff with skipping over '<' is tested in ../util/kaldi-io-test.cc, + // since we need to make sure it works with pipes. + ReadToken(infile, binary_in, &token1_in); + KALDI_ASSERT(token1_in == std::string(token1)); + ReadToken(infile, binary_in, &token2_in); + KALDI_ASSERT(token2_in == std::string(token2)); + if (Rand() % 2 == 0) + ExpectToken(infile, binary_in, token3.c_str()); + else + ExpectToken(infile, binary_in, token3); + float f1_in; // same type. + ReadBasicType(infile, binary_in, &f1_in); + AssertEqual(f1_in, f1); + double f2_in; // wrong type. + ReadBasicType(infile, binary_in, &f2_in); + AssertEqual(f2_in, f2); + double d1_in; // same type. + ReadBasicType(infile, binary_in, &d1_in); + AssertEqual(d1_in, d1); + float d2_in; // wrong type. + ReadBasicType(infile, binary_in, &d2_in); + AssertEqual(d2_in, d2); + KALDI_ASSERT(Peek(infile, binary_in) == -1); + KALDI_ASSERT(PeekToken(infile, binary_in) == -1); + } + unlink(filename); + } +} + + + +} // end namespace kaldi. + +int main() { + using namespace kaldi; + for (size_t i = 0; i < 10; i++) { + UnitTestIo(false); + UnitTestIo(true); + } + KALDI_ASSERT(1); // just to check that KALDI_ASSERT does not fail for 1. + return 0; +} + diff --git a/runtime/onnxruntime/third_party/kaldi/base/io-funcs.cc b/runtime/onnxruntime/third_party/kaldi/base/io-funcs.cc new file mode 100644 index 000000000..150f74099 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/base/io-funcs.cc @@ -0,0 +1,218 @@ +// base/io-funcs.cc + +// Copyright 2009-2011 Microsoft Corporation; Saarland University + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#include "base/io-funcs.h" +#include "base/kaldi-math.h" + +namespace kaldi { + +template<> +void WriteBasicType(std::ostream &os, bool binary, bool b) { + os << (b ? "T":"F"); + if (!binary) os << " "; + if (os.fail()) + KALDI_ERR << "Write failure in WriteBasicType"; +} + +template<> +void ReadBasicType(std::istream &is, bool binary, bool *b) { + KALDI_PARANOID_ASSERT(b != NULL); + if (!binary) is >> std::ws; // eat up whitespace. + char c = is.peek(); + if (c == 'T') { + *b = true; + is.get(); + } else if (c == 'F') { + *b = false; + is.get(); + } else { + KALDI_ERR << "Read failure in ReadBasicType, file position is " + << is.tellg() << ", next char is " << CharToString(c); + } +} + +template<> +void WriteBasicType(std::ostream &os, bool binary, float f) { + if (binary) { + char c = sizeof(f); + os.put(c); + os.write(reinterpret_cast(&f), sizeof(f)); + } else { + os << f << " "; + } +} + +template<> +void WriteBasicType(std::ostream &os, bool binary, double f) { + if (binary) { + char c = sizeof(f); + os.put(c); + os.write(reinterpret_cast(&f), sizeof(f)); + } else { + os << f << " "; + } +} + +template<> +void ReadBasicType(std::istream &is, bool binary, float *f) { + KALDI_PARANOID_ASSERT(f != NULL); + if (binary) { + double d; + int c = is.peek(); + if (c == sizeof(*f)) { + is.get(); + is.read(reinterpret_cast(f), sizeof(*f)); + } else if (c == sizeof(d)) { + ReadBasicType(is, binary, &d); + *f = d; + } else { + KALDI_ERR << "ReadBasicType: expected float, saw " << is.peek() + << ", at file position " << is.tellg(); + } + } else { + is >> *f; + } + if (is.fail()) { + KALDI_ERR << "ReadBasicType: failed to read, at file position " + << is.tellg(); + } +} + +template<> +void ReadBasicType(std::istream &is, bool binary, double *d) { + KALDI_PARANOID_ASSERT(d != NULL); + if (binary) { + float f; + int c = is.peek(); + if (c == sizeof(*d)) { + is.get(); + is.read(reinterpret_cast(d), sizeof(*d)); + } else if (c == sizeof(f)) { + ReadBasicType(is, binary, &f); + *d = f; + } else { + KALDI_ERR << "ReadBasicType: expected float, saw " << is.peek() + << ", at file position " << is.tellg(); + } + } else { + is >> *d; + } + if (is.fail()) { + KALDI_ERR << "ReadBasicType: failed to read, at file position " + << is.tellg(); + } +} + +void CheckToken(const char *token) { + if (*token == '\0') + KALDI_ERR << "Token is empty (not a valid token)"; + const char *orig_token = token; + while (*token != '\0') { + if (::isspace(*token)) + KALDI_ERR << "Token is not a valid token (contains space): '" + << orig_token << "'"; + token++; + } +} + +void WriteToken(std::ostream &os, bool binary, const char *token) { + // binary mode is ignored; + // we use space as termination character in either case. + KALDI_ASSERT(token != NULL); + CheckToken(token); // make sure it's valid (can be read back) + os << token << " "; + if (os.fail()) { + KALDI_ERR << "Write failure in WriteToken."; + } +} + +int Peek(std::istream &is, bool binary) { + if (!binary) is >> std::ws; // eat up whitespace. + return is.peek(); +} + +void WriteToken(std::ostream &os, bool binary, const std::string & token) { + WriteToken(os, binary, token.c_str()); +} + +void ReadToken(std::istream &is, bool binary, std::string *str) { + KALDI_ASSERT(str != NULL); + if (!binary) is >> std::ws; // consume whitespace. + is >> *str; + if (is.fail()) { + KALDI_ERR << "ReadToken, failed to read token at file position " + << is.tellg(); + } + if (!isspace(is.peek())) { + KALDI_ERR << "ReadToken, expected space after token, saw instead " + << CharToString(static_cast(is.peek())) + << ", at file position " << is.tellg(); + } + is.get(); // consume the space. +} + +int PeekToken(std::istream &is, bool binary) { + if (!binary) is >> std::ws; // consume whitespace. + bool read_bracket; + if (static_cast(is.peek()) == '<') { + read_bracket = true; + is.get(); + } else { + read_bracket = false; + } + int ans = is.peek(); + if (read_bracket) { + if (!is.unget()) { + // Clear the bad bit. This code can be (and is in fact) reached, since the + // C++ standard does not guarantee that a call to unget() must succeed. + is.clear(); + } + } + return ans; +} + + +void ExpectToken(std::istream &is, bool binary, const char *token) { + int pos_at_start = is.tellg(); + KALDI_ASSERT(token != NULL); + CheckToken(token); // make sure it's valid (can be read back) + if (!binary) is >> std::ws; // consume whitespace. + std::string str; + is >> str; + is.get(); // consume the space. + if (is.fail()) { + KALDI_ERR << "Failed to read token [started at file position " + << pos_at_start << "], expected " << token; + } + // The second half of the '&&' expression below is so that if we're expecting + // "", we will accept "Foo>" instead. This is so that the model-reading + // code will tolerate errors in PeekToken where is.unget() failed; search for + // is.clear() in PeekToken() for an explanation. + if (strcmp(str.c_str(), token) != 0 && + !(token[0] == '<' && strcmp(str.c_str(), token + 1) == 0)) { + KALDI_ERR << "Expected token \"" << token << "\", got instead \"" + << str <<"\"."; + } +} + +void ExpectToken(std::istream &is, bool binary, const std::string &token) { + ExpectToken(is, binary, token.c_str()); +} + +} // end namespace kaldi diff --git a/runtime/onnxruntime/third_party/kaldi/base/io-funcs.h b/runtime/onnxruntime/third_party/kaldi/base/io-funcs.h new file mode 100644 index 000000000..895f661ec --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/base/io-funcs.h @@ -0,0 +1,245 @@ +// base/io-funcs.h + +// Copyright 2009-2011 Microsoft Corporation; Saarland University; +// Jan Silovsky; Yanmin Qian +// 2016 Xiaohui Zhang + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#ifndef KALDI_BASE_IO_FUNCS_H_ +#define KALDI_BASE_IO_FUNCS_H_ + +// This header only contains some relatively low-level I/O functions. +// The full Kaldi I/O declarations are in ../util/kaldi-io.h +// and ../util/kaldi-table.h +// They were put in util/ in order to avoid making the Matrix library +// dependent on them. + +#include +#include +#include + +#include "base/kaldi-common.h" +#include "base/io-funcs-inl.h" + +namespace kaldi { + + + +/* + This comment describes the Kaldi approach to I/O. All objects can be written + and read in two modes: binary and text. In addition we want to make the I/O + work if we redefine the typedef "BaseFloat" between floats and doubles. + We also want to have control over whitespace in text mode without affecting + the meaning of the file, for pretty-printing purposes. + + Errors are handled by throwing a KaldiFatalError exception. + + For integer and floating-point types (and boolean values): + + WriteBasicType(std::ostream &, bool binary, const T&); + ReadBasicType(std::istream &, bool binary, T*); + + and we expect these functions to be defined in such a way that they work when + the type T changes between float and double, so you can read float into double + and vice versa]. Note that for efficiency and space-saving reasons, the Vector + and Matrix classes do not use these functions [but they preserve the type + interchangeability in their own way] + + For a class (or struct) C: + class C { + .. + Write(std::ostream &, bool binary, [possibly extra optional args for specific classes]) const; + Read(std::istream &, bool binary, [possibly extra optional args for specific classes]); + .. + } + NOTE: The only actual optional args we used are the "add" arguments in + Vector/Matrix classes, which specify whether we should sum the data already + in the class with the data being read. + + For types which are typedef's involving stl classes, I/O is as follows: + typedef std::vector > MyTypedefName; + + The user should define something like: + + WriteMyTypedefName(std::ostream &, bool binary, const MyTypedefName &t); + ReadMyTypedefName(std::ostream &, bool binary, MyTypedefName *t); + + The user would have to write these functions. + + For a type std::vector: + + void WriteIntegerVector(std::ostream &os, bool binary, const std::vector &v); + void ReadIntegerVector(std::istream &is, bool binary, std::vector *v); + + For other types, e.g. vectors of pairs, the user should create a routine of the + type WriteMyTypedefName. This is to avoid introducing confusing templated functions; + we could easily create templated functions to handle most of these cases but they + would have to share the same name. + + It also often happens that the user needs to write/read special tokens as part + of a file. These might be class headers, or separators/identifiers in the class. + We provide special functions for manipulating these. These special tokens must + be nonempty and must not contain any whitespace. + + void WriteToken(std::ostream &os, bool binary, const char*); + void WriteToken(std::ostream &os, bool binary, const std::string & token); + int Peek(std::istream &is, bool binary); + void ReadToken(std::istream &is, bool binary, std::string *str); + void PeekToken(std::istream &is, bool binary, std::string *str); + + WriteToken writes the token and one space (whether in binary or text mode). + + Peek returns the first character of the next token, by consuming whitespace + (in text mode) and then returning the peek() character. It returns -1 at EOF; + it doesn't throw. It's useful if a class can have various forms based on + typedefs and virtual classes, and wants to know which version to read. + + ReadToken allows the caller to obtain the next token. PeekToken works just + like ReadToken, but seeks back to the beginning of the token. A subsequent + call to ReadToken will read the same token again. This is useful when + different object types are written to the same file; using PeekToken one can + decide which of the objects to read. + + There is currently no special functionality for writing/reading strings (where the strings + contain data rather than "special tokens" that are whitespace-free and nonempty). This is + because Kaldi is structured in such a way that strings don't appear, except as OpenFst symbol + table entries (and these have their own format). + + + NOTE: you should not call ReadIntegerType and WriteIntegerType with types, + such as int and size_t, that are machine-independent -- at least not + if you want your file formats to port between machines. Use int32 and + int64 where necessary. There is no way to detect this using compile-time + assertions because C++ only keeps track of the internal representation of + the type. +*/ + +/// \addtogroup io_funcs_basic +/// @{ + + +/// WriteBasicType is the name of the write function for bool, integer types, +/// and floating-point types. They all throw on error. +template void WriteBasicType(std::ostream &os, bool binary, T t); + +/// ReadBasicType is the name of the read function for bool, integer types, +/// and floating-point types. They all throw on error. +template void ReadBasicType(std::istream &is, bool binary, T *t); + + +// Declare specialization for bool. +template<> +void WriteBasicType(std::ostream &os, bool binary, bool b); + +template <> +void ReadBasicType(std::istream &is, bool binary, bool *b); + +// Declare specializations for float and double. +template<> +void WriteBasicType(std::ostream &os, bool binary, float f); + +template<> +void WriteBasicType(std::ostream &os, bool binary, double f); + +template<> +void ReadBasicType(std::istream &is, bool binary, float *f); + +template<> +void ReadBasicType(std::istream &is, bool binary, double *f); + +// Define ReadBasicType that accepts an "add" parameter to add to +// the destination. Caution: if used in Read functions, be careful +// to initialize the parameters concerned to zero in the default +// constructor. +template +inline void ReadBasicType(std::istream &is, bool binary, T *t, bool add) { + if (!add) { + ReadBasicType(is, binary, t); + } else { + T tmp = T(0); + ReadBasicType(is, binary, &tmp); + *t += tmp; + } +} + +/// Function for writing STL vectors of integer types. +template inline void WriteIntegerVector(std::ostream &os, bool binary, + const std::vector &v); + +/// Function for reading STL vector of integer types. +template inline void ReadIntegerVector(std::istream &is, bool binary, + std::vector *v); + +/// Function for writing STL vectors of pairs of integer types. +template +inline void WriteIntegerPairVector(std::ostream &os, bool binary, + const std::vector > &v); + +/// Function for reading STL vector of pairs of integer types. +template +inline void ReadIntegerPairVector(std::istream &is, bool binary, + std::vector > *v); + +/// The WriteToken functions are for writing nonempty sequences of non-space +/// characters. They are not for general strings. +void WriteToken(std::ostream &os, bool binary, const char *token); +void WriteToken(std::ostream &os, bool binary, const std::string & token); + +/// Peek consumes whitespace (if binary == false) and then returns the peek() +/// value of the stream. +int Peek(std::istream &is, bool binary); + +/// ReadToken gets the next token and puts it in str (exception on failure). If +/// PeekToken() had been previously called, it is possible that the stream had +/// failed to unget the starting '<' character. In this case ReadToken() returns +/// the token string without the leading '<'. You must be prepared to handle +/// this case. ExpectToken() handles this internally, and is not affected. +void ReadToken(std::istream &is, bool binary, std::string *token); + +/// PeekToken will return the first character of the next token, or -1 if end of +/// file. It's the same as Peek(), except if the first character is '<' it will +/// skip over it and will return the next character. It will attempt to unget +/// the '<' so the stream is where it was before you did PeekToken(), however, +/// this is not guaranteed (see ReadToken()). +int PeekToken(std::istream &is, bool binary); + +/// ExpectToken tries to read in the given token, and throws an exception +/// on failure. +void ExpectToken(std::istream &is, bool binary, const char *token); +void ExpectToken(std::istream &is, bool binary, const std::string & token); + +/// ExpectPretty attempts to read the text in "token", but only in non-binary +/// mode. Throws exception on failure. It expects an exact match except that +/// arbitrary whitespace matches arbitrary whitespace. +void ExpectPretty(std::istream &is, bool binary, const char *token); +void ExpectPretty(std::istream &is, bool binary, const std::string & token); + +/// @} end "addtogroup io_funcs_basic" + + +/// InitKaldiOutputStream initializes an opened stream for writing by writing an +/// optional binary header and modifying the floating-point precision; it will +/// typically not be called by users directly. +inline void InitKaldiOutputStream(std::ostream &os, bool binary); + +/// InitKaldiInputStream initializes an opened stream for reading by detecting +/// the binary header and setting the "binary" value appropriately; +/// It will typically not be called by users directly. +inline bool InitKaldiInputStream(std::istream &is, bool *binary); + +} // end namespace kaldi. +#endif // KALDI_BASE_IO_FUNCS_H_ diff --git a/runtime/onnxruntime/third_party/kaldi/base/kaldi-common.h b/runtime/onnxruntime/third_party/kaldi/base/kaldi-common.h new file mode 100644 index 000000000..264565d18 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/base/kaldi-common.h @@ -0,0 +1,41 @@ +// base/kaldi-common.h + +// Copyright 2009-2011 Microsoft Corporation + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#ifndef KALDI_BASE_KALDI_COMMON_H_ +#define KALDI_BASE_KALDI_COMMON_H_ 1 + +#include +#include +#include // C string stuff like strcpy +#include +#include +#include +#include +#include +#include +#include + +#include "base/kaldi-utils.h" +#include "base/kaldi-error.h" +#include "base/kaldi-types.h" +#include "base/io-funcs.h" +#include "base/kaldi-math.h" +#include "base/timer.h" + +#endif // KALDI_BASE_KALDI_COMMON_H_ diff --git a/runtime/onnxruntime/third_party/kaldi/base/kaldi-error-test.cc b/runtime/onnxruntime/third_party/kaldi/base/kaldi-error-test.cc new file mode 100644 index 000000000..31440edf3 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/base/kaldi-error-test.cc @@ -0,0 +1,82 @@ +// base/kaldi-error-test.cc + +// Copyright 2009-2011 Microsoft Corporation + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#include "base/kaldi-common.h" + +// testing that we get the stack trace. +namespace kaldi { + +void MyFunction2() { KALDI_ERR << "Ignore this error"; } + +void MyFunction1() { MyFunction2(); } + +void UnitTestError() { + { + std::cerr << "Ignore next error:\n"; + MyFunction1(); + } +} + +void VerifySymbolRange(const std::string &trace, const bool want_found, + const std::string &want_symbol) { + size_t begin, end; + const bool found = internal::LocateSymbolRange(trace, &begin, &end); + if (found != want_found) { + KALDI_ERR << "Found mismatch, got " << found << " want " << want_found; + } + if (!found) { + return; + } + const std::string symbol = trace.substr(begin, end - begin); + if (symbol != want_symbol) { + KALDI_ERR << "Symbol mismatch, got " << symbol << " want " << want_symbol; + } +} + +void TestLocateSymbolRange() { + VerifySymbolRange("", false, ""); + VerifySymbolRange( + R"TRACE(./kaldi-error-test(_ZN5kaldi13UnitTestErrorEv+0xb) [0x804965d])TRACE", + true, "_ZN5kaldi13UnitTestErrorEv"); + // It is ok thread_start is not found because it is a C symbol. + VerifySymbolRange( + R"TRACE(31 libsystem_pthread.dylib 0x00007fff6fe4e40d thread_start + 13)TRACE", + false, ""); + VerifySymbolRange( + R"TRACE(0 server 0x000000010f67614d _ZNK5kaldi13MessageLogger10LogMessageEv + 813)TRACE", + true, "_ZNK5kaldi13MessageLogger10LogMessageEv"); + VerifySymbolRange( + R"TRACE(29 libsystem_pthread.dylib 0x00007fff6fe4f2eb _pthread_body + 126)TRACE", + true, "_pthread_body"); +} + +} // namespace kaldi + +int main() { + kaldi::TestLocateSymbolRange(); + + kaldi::SetProgramName("/foo/bar/kaldi-error-test"); + try { + kaldi::UnitTestError(); + KALDI_ASSERT(0); // should not happen. + exit(1); + } catch (kaldi::KaldiFatalError &e) { + std::cout << "The error we generated was: '" << e.KaldiMessage() << "'\n"; + } +} diff --git a/runtime/onnxruntime/third_party/kaldi/base/kaldi-error.cc b/runtime/onnxruntime/third_party/kaldi/base/kaldi-error.cc new file mode 100644 index 000000000..2970bbeaa --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/base/kaldi-error.cc @@ -0,0 +1,248 @@ +// base/kaldi-error.cc + +// Copyright 2019 LAIX (Yi Sun) +// Copyright 2019 SmartAction LLC (kkm) +// Copyright 2016 Brno University of Technology (author: Karel Vesely) +// Copyright 2009-2011 Microsoft Corporation; Lukas Burget; Ondrej Glembek + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#ifdef HAVE_EXECINFO_H +#include // To get stack trace in error messages. +// If this #include fails there is an error in the Makefile, it does not +// support your platform well. Make sure HAVE_EXECINFO_H is undefined, +// and the code will compile. +#ifdef HAVE_CXXABI_H +#include // For name demangling. +// Useful to decode the stack trace, but only used if we have execinfo.h +#endif // HAVE_CXXABI_H +#endif // HAVE_EXECINFO_H + +#include "base/kaldi-common.h" +#include "base/kaldi-error.h" + +// KALDI_GIT_HEAD is useless currently in full repo +//#if !defined(KALDI_VERSION) +//#include "base/version.h" +//#endif + +namespace kaldi { + +/***** GLOBAL VARIABLES FOR LOGGING *****/ + +int32 g_kaldi_verbose_level = 0; +static std::string program_name; +static LogHandler log_handler = NULL; + +void SetProgramName(const char *basename) { + // Using the 'static std::string' for the program name is mostly harmless, + // because (a) Kaldi logging is undefined before main(), and (b) no stdc++ + // string implementation has been found in the wild that would not be just + // an empty string when zero-initialized but not yet constructed. + program_name = basename; +} + +/***** HELPER FUNCTIONS *****/ + +// Trim filename to at most 1 trailing directory long. Given a filename like +// "/a/b/c/d/e/f.cc", return "e/f.cc". Support both '/' and '\' as the path +// separator. +static const char *GetShortFileName(const char *path) { + if (path == nullptr) + return ""; + + const char *prev = path, *last = path; + while ((path = std::strpbrk(path, "\\/")) != nullptr) { + ++path; + prev = last; + last = path; + } + return prev; +} + +/***** STACK TRACE *****/ + +namespace internal { +bool LocateSymbolRange(const std::string &trace_name, size_t *begin, + size_t *end) { + // Find the first '_' with leading ' ' or '('. + *begin = std::string::npos; + for (size_t i = 1; i < trace_name.size(); i++) { + if (trace_name[i] != '_') { + continue; + } + if (trace_name[i - 1] == ' ' || trace_name[i - 1] == '(') { + *begin = i; + break; + } + } + if (*begin == std::string::npos) { + return false; + } + *end = trace_name.find_first_of(" +", *begin); + return *end != std::string::npos; +} +} // namespace internal + +#ifdef HAVE_EXECINFO_H +static std::string Demangle(std::string trace_name) { +#ifndef HAVE_CXXABI_H + return trace_name; +#else // HAVE_CXXABI_H + // Try demangle the symbol. We are trying to support the following formats + // produced by different platforms: + // + // Linux: + // ./kaldi-error-test(_ZN5kaldi13UnitTestErrorEv+0xb) [0x804965d] + // + // Mac: + // 0 server 0x000000010f67614d _ZNK5kaldi13MessageLogger10LogMessageEv + 813 + // + // We want to extract the name e.g., '_ZN5kaldi13UnitTestErrorEv' and + // demangle it info a readable name like kaldi::UnitTextError. + size_t begin, end; + if (!internal::LocateSymbolRange(trace_name, &begin, &end)) { + return trace_name; + } + std::string symbol = trace_name.substr(begin, end - begin); + int status; + char *demangled_name = abi::__cxa_demangle(symbol.c_str(), 0, 0, &status); + if (status == 0 && demangled_name != nullptr) { + symbol = demangled_name; + free(demangled_name); + } + return trace_name.substr(0, begin) + symbol + + trace_name.substr(end, std::string::npos); +#endif // HAVE_CXXABI_H +} +#endif // HAVE_EXECINFO_H + +static std::string KaldiGetStackTrace() { + std::string ans; +#ifdef HAVE_EXECINFO_H + const size_t KALDI_MAX_TRACE_SIZE = 50; + const size_t KALDI_MAX_TRACE_PRINT = 50; // Must be even. + // Buffer for the trace. + void *trace[KALDI_MAX_TRACE_SIZE]; + // Get the trace. + size_t size = backtrace(trace, KALDI_MAX_TRACE_SIZE); + // Get the trace symbols. + char **trace_symbol = backtrace_symbols(trace, size); + if (trace_symbol == NULL) + return ans; + + // Compose a human-readable backtrace string. + ans += "[ Stack-Trace: ]\n"; + if (size <= KALDI_MAX_TRACE_PRINT) { + for (size_t i = 0; i < size; i++) { + ans += Demangle(trace_symbol[i]) + "\n"; + } + } else { // Print out first+last (e.g.) 5. + for (size_t i = 0; i < KALDI_MAX_TRACE_PRINT / 2; i++) { + ans += Demangle(trace_symbol[i]) + "\n"; + } + ans += ".\n.\n.\n"; + for (size_t i = size - KALDI_MAX_TRACE_PRINT / 2; i < size; i++) { + ans += Demangle(trace_symbol[i]) + "\n"; + } + if (size == KALDI_MAX_TRACE_SIZE) + ans += ".\n.\n.\n"; // Stack was too long, probably a bug. + } + + // We must free the array of pointers allocated by backtrace_symbols(), + // but not the strings themselves. + free(trace_symbol); +#endif // HAVE_EXECINFO_H + return ans; +} + +/***** KALDI LOGGING *****/ + +MessageLogger::MessageLogger(LogMessageEnvelope::Severity severity, + const char *func, const char *file, int32 line) { + // Obviously, we assume the strings survive the destruction of this object. + envelope_.severity = severity; + envelope_.func = func; + envelope_.file = GetShortFileName(file); // Points inside 'file'. + envelope_.line = line; +} + +void MessageLogger::LogMessage() const { + // Send to the logging handler if provided. + if (log_handler != NULL) { + log_handler(envelope_, GetMessage().c_str()); + return; + } + + // Otherwise, use the default Kaldi logging. + // Build the log-message header. + std::stringstream full_message; + if (envelope_.severity > LogMessageEnvelope::kInfo) { + full_message << "VLOG[" << envelope_.severity << "] ("; + } else { + switch (envelope_.severity) { + case LogMessageEnvelope::kInfo: + //full_message << "LOG (\n"; + break; + case LogMessageEnvelope::kWarning: + //full_message << "WARNING (\n"; + break; + case LogMessageEnvelope::kAssertFailed: + full_message << "ASSERTION_FAILED (\n"; + break; + case LogMessageEnvelope::kError: + default: // If not the ERROR, it still an error! + full_message << "ERROR (\n"; + break; + } + } + // Add other info from the envelope and the message text. + //full_message << program_name.c_str() << "[" KALDI_VERSION "]" << ':' + // << envelope_.func << "():" << envelope_.file << ':' + // << envelope_.line << ") " << GetMessage().c_str(); + + // Add stack trace for errors and assertion failures, if available. + if (envelope_.severity < LogMessageEnvelope::kWarning) { + const std::string &stack_trace = KaldiGetStackTrace(); + if (!stack_trace.empty()) { + full_message << "\n\n" << stack_trace; + } + } + + // Print the complete message to stderr. + std::cerr << full_message.str(); +} + +/***** KALDI ASSERTS *****/ + +void KaldiAssertFailure_(const char *func, const char *file, int32 line, + const char *cond_str) { + MessageLogger::Log() = + MessageLogger(LogMessageEnvelope::kAssertFailed, func, file, line) + << "Assertion failed: (" << cond_str << ")"; + fflush(NULL); // Flush all pending buffers, abort() may not flush stderr. + std::abort(); +} + +/***** THIRD-PARTY LOG-HANDLER *****/ + +LogHandler SetLogHandler(LogHandler handler) { + LogHandler old_handler = log_handler; + log_handler = handler; + return old_handler; +} + +} // namespace kaldi diff --git a/runtime/onnxruntime/third_party/kaldi/base/kaldi-error.h b/runtime/onnxruntime/third_party/kaldi/base/kaldi-error.h new file mode 100644 index 000000000..b150a02d4 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/base/kaldi-error.h @@ -0,0 +1,231 @@ +// base/kaldi-error.h + +// Copyright 2019 LAIX (Yi Sun) +// Copyright 2019 SmartAction LLC (kkm) +// Copyright 2016 Brno University of Technology (author: Karel Vesely) +// Copyright 2009-2011 Microsoft Corporation; Ondrej Glembek; Lukas Burget; +// Saarland University + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#ifndef KALDI_BASE_KALDI_ERROR_H_ +#define KALDI_BASE_KALDI_ERROR_H_ 1 + +#include +#include +#include +#include +#include +#include + +#include "base/kaldi-types.h" +#include "base/kaldi-utils.h" +/* Important that this file does not depend on any other kaldi headers. */ + +#ifdef _MSC_VER +#define __func__ __FUNCTION__ +#endif + +namespace kaldi { + +/// \addtogroup error_group +/// @{ + +/***** PROGRAM NAME AND VERBOSITY LEVEL *****/ + +/// Called by ParseOptions to set base name (no directory) of the executing +/// program. The name is printed in logging code along with every message, +/// because in our scripts, we often mix together the stderr of many programs. +/// This function is very thread-unsafe. +void SetProgramName(const char *basename); + +/// This is set by util/parse-options.{h,cc} if you set --verbose=? option. +/// Do not use directly, prefer {Get,Set}VerboseLevel(). +extern int32 g_kaldi_verbose_level; + +/// Get verbosity level, usually set via command line '--verbose=' switch. +inline int32 GetVerboseLevel() { return g_kaldi_verbose_level; } + +/// This should be rarely used, except by programs using Kaldi as library; +/// command-line programs set the verbose level automatically from ParseOptions. +inline void SetVerboseLevel(int32 i) { g_kaldi_verbose_level = i; } + +/***** KALDI LOGGING *****/ + +/// Log message severity and source location info. +struct LogMessageEnvelope { + /// Message severity. In addition to these levels, positive values (1 to 6) + /// specify verbose logging level. Verbose messages are produced only when + /// SetVerboseLevel() has been called to set logging level to at least the + /// corresponding value. + enum Severity { + kAssertFailed = -3, //!< Assertion failure. abort() will be called. + kError = -2, //!< Fatal error. KaldiFatalError will be thrown. + kWarning = -1, //!< Indicates a recoverable but abnormal condition. + kInfo = 0, //!< Informational message. + }; + int severity; //!< A Severity value, or positive verbosity level. + const char *func; //!< Name of the function invoking the logging. + const char *file; //!< Source file name with up to 1 leading directory. + int32 line; // MessageLogger &operator<<(const T &val) { + ss_ << val; + return *this; + } + + // When assigned a MessageLogger, log its contents. + struct Log final { + void operator=(const MessageLogger &logger) { logger.LogMessage(); } + }; + + // When assigned a MessageLogger, log its contents and then throw + // a KaldiFatalError. + struct LogAndThrow final { + [[noreturn]] void operator=(const MessageLogger &logger) { + logger.LogMessage(); + throw KaldiFatalError(logger.GetMessage()); + } + }; + +private: + std::string GetMessage() const { return ss_.str(); } + void LogMessage() const; + + LogMessageEnvelope envelope_; + std::ostringstream ss_; +}; + +// Logging macros. +#define KALDI_ERR \ + ::kaldi::MessageLogger::LogAndThrow() = ::kaldi::MessageLogger( \ + ::kaldi::LogMessageEnvelope::kError, __func__, __FILE__, __LINE__) +#define KALDI_WARN \ + ::kaldi::MessageLogger::Log() = ::kaldi::MessageLogger( \ + ::kaldi::LogMessageEnvelope::kWarning, __func__, __FILE__, __LINE__) +#define KALDI_LOG \ + ::kaldi::MessageLogger::Log() = ::kaldi::MessageLogger( \ + ::kaldi::LogMessageEnvelope::kInfo, __func__, __FILE__, __LINE__) +#define KALDI_VLOG(v) \ + if ((v) <= ::kaldi::GetVerboseLevel()) \ + ::kaldi::MessageLogger::Log() = \ + ::kaldi::MessageLogger((::kaldi::LogMessageEnvelope::Severity)(v), \ + __func__, __FILE__, __LINE__) + +/***** KALDI ASSERTS *****/ + +[[noreturn]] void KaldiAssertFailure_(const char *func, const char *file, + int32 line, const char *cond_str); + +// Note on KALDI_ASSERT and KALDI_PARANOID_ASSERT: +// +// A single block {} around if /else does not work, because it causes +// syntax error (unmatched else block) in the following code: +// +// if (condition) +// KALDI_ASSERT(condition2); +// else +// SomethingElse(); +// +// do {} while(0) -- note there is no semicolon at the end! -- works nicely, +// and compilers will be able to optimize the loop away (as the condition +// is always false). +// +// Also see KALDI_COMPILE_TIME_ASSERT, defined in base/kaldi-utils.h, and +// KALDI_ASSERT_IS_INTEGER_TYPE and KALDI_ASSERT_IS_FLOATING_TYPE, also defined +// there. +#ifndef NDEBUG +#define KALDI_ASSERT(cond) \ + do { \ + if (cond) \ + (void)0; \ + else \ + ::kaldi::KaldiAssertFailure_(__func__, __FILE__, __LINE__, #cond); \ + } while (0) +#else +#define KALDI_ASSERT(cond) (void)0 +#endif + +// Some more expensive asserts only checked if this defined. +#ifdef KALDI_PARANOID +#define KALDI_PARANOID_ASSERT(cond) \ + do { \ + if (cond) \ + (void)0; \ + else \ + ::kaldi::KaldiAssertFailure_(__func__, __FILE__, __LINE__, #cond); \ + } while (0) +#else +#define KALDI_PARANOID_ASSERT(cond) (void)0 +#endif + +/***** THIRD-PARTY LOG-HANDLER *****/ + +/// Type of third-party logging function. +typedef void (*LogHandler)(const LogMessageEnvelope &envelope, + const char *message); + +/// Set logging handler. If called with a non-NULL function pointer, the +/// function pointed by it is called to send messages to a caller-provided log. +/// If called with a NULL pointer, restores default Kaldi error logging to +/// stderr. This function is obviously not thread safe; the log handler must be. +/// Returns a previously set logging handler pointer, or NULL. +LogHandler SetLogHandler(LogHandler); + +/// @} end "addtogroup error_group" + +//// Functions within internal is exported for testing only, do not use. +//namespace internal { +//bool LocateSymbolRange(const std::string &trace_name, size_t *begin, +// size_t *end); +//} // namespace internal +} // namespace kaldi + +#endif // KALDI_BASE_KALDI_ERROR_H_ diff --git a/runtime/onnxruntime/third_party/kaldi/base/kaldi-math-test.cc b/runtime/onnxruntime/third_party/kaldi/base/kaldi-math-test.cc new file mode 100644 index 000000000..8d6e6164e --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/base/kaldi-math-test.cc @@ -0,0 +1,334 @@ +// base/kaldi-math-test.cc +// +// Copyright 2009-2011 Microsoft Corporation; Yanmin Qian; Jan Silovsky + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. +#include "base/kaldi-math.h" +#include +#include "base/timer.h" + +namespace kaldi { + +template void UnitTestGcdLcmTpl() { + for (I a = 1; a < 15; a++) { // a is min gcd. + I b = (I)(Rand() % 10); + I c = (I)(Rand() % 10); + if (std::numeric_limits::is_signed) { + if (Rand() % 2 == 0) b = -b; + if (Rand() % 2 == 0) c = -c; + } + if (b == 0 && c == 0) continue; // gcd not defined for such numbers. + I g = Gcd(b*a, c*a); + KALDI_ASSERT(g >= a); + KALDI_ASSERT((b*a) % g == 0); + KALDI_ASSERT((c*a) % g == 0); + + // test least common multiple + if (b <= 0 || c <= 0) continue; // lcm not defined unless both positive. + I h = Lcm(b*a, c*a); + KALDI_ASSERT(h != 0 && (h % (b*a)) == 0 && + (h % (c*a)) == 0); + } +} + +void UnitTestRoundUpToNearestPowerOfTwo() { + KALDI_ASSERT(RoundUpToNearestPowerOfTwo(1) == 1); + KALDI_ASSERT(RoundUpToNearestPowerOfTwo(2) == 2); + KALDI_ASSERT(RoundUpToNearestPowerOfTwo(3) == 4); + KALDI_ASSERT(RoundUpToNearestPowerOfTwo(4) == 4); + KALDI_ASSERT(RoundUpToNearestPowerOfTwo(7) == 8); + KALDI_ASSERT(RoundUpToNearestPowerOfTwo(8) == 8); + KALDI_ASSERT(RoundUpToNearestPowerOfTwo(255) == 256); + KALDI_ASSERT(RoundUpToNearestPowerOfTwo(256) == 256); + KALDI_ASSERT(RoundUpToNearestPowerOfTwo(257) == 512); + KALDI_ASSERT(RoundUpToNearestPowerOfTwo(1073700000) == 1073741824); +} + +void UnitTestDivideRoundingDown() { + for (int32 i = 0; i < 100; i++) { + int32 a = RandInt(-100, 100); + int32 b = 0; + while (b == 0) + b = RandInt(-100, 100); + KALDI_ASSERT(DivideRoundingDown(a, b) == + std::floor(static_cast(a) / static_cast(b))); + } +} + +void UnitTestGcdLcm() { + UnitTestGcdLcmTpl(); + UnitTestGcdLcmTpl(); + UnitTestGcdLcmTpl(); +} + +void UnitTestRand() { + // Testing random-number generation. + std::cout << "Testing random-number generation. " + << "If there is an error this may not terminate.\n"; + std::cout << "If this does not terminate, look more closely. " + << "There might be a problem [but might not be]\n"; + for (int i = 1; i < 10; i++) { + { // test RandUniform. + std::cout << "Test RandUniform\n"; + KALDI_ASSERT(RandUniform() >= 0 && RandUniform() <= 1); + float sum = RandUniform()-0.5; + for (int j = 0; ; j++) { + sum += RandUniform()-0.5; + if (std::abs(sum) < 0.5*sqrt(static_cast(j))) break; + } + } + { // test RandGauss. + float sum = RandGauss(); + for (int j = 0; ; j++) { + sum += RandGauss(); + if (std::abs(sum) < 0.5*sqrt(static_cast(j))) break; + } + } + { // test RandGauss. + float sum = RandGauss(); + for (int j = 0; ; j++) { + float a, b; + RandGauss2(&a, &b); + if (i % 2 == 0) sum += a; + else + sum += b; + if (std::abs(sum) < 0.5*sqrt(static_cast(j))) break; + } + } + { // test poisson_Rand(). + KALDI_ASSERT(RandPoisson(3.0) >= 0); + KALDI_ASSERT(RandPoisson(0.0) == 0); + std::cout << "Test RandPoisson\n"; + float lambda = RandUniform() * 3.0; // between 0 and 3. + double sum = RandPoisson(lambda) - lambda; // expected value is zero. + for (int j = 0; ; j++) { + sum += RandPoisson(lambda) - lambda; + if (std::abs(sum) < 0.5*sqrt(static_cast(j))) break; + } + } + + { // test WithProb(). + for (int32 i = 0; i < 10; i++) { + KALDI_ASSERT((WithProb(0.0) == false) && (WithProb(1.0) == true)); + } + { + int32 tot = 0, n = 10000; + BaseFloat p = 0.5; + for (int32 i = 0; i < n; i++) + tot += WithProb(p); + KALDI_ASSERT(tot > (n * p * 0.8) && tot < (n * p * 1.2)); + } + { + int32 tot = 0, n = 10000; + BaseFloat p = 0.25; + for (int32 i = 0; i < n; i++) + tot += WithProb(p); + KALDI_ASSERT(tot > (n * p * 0.8) && tot < (n * p * 1.2)); + } + } + { // test RandInt(). + KALDI_ASSERT(RandInt(0, 3) >= 0 && RandInt(0, 3) <= 3); + + std::cout << "Test RandInt\n"; + int minint = Rand() % 200; + int maxint = minint + 1 + Rand() % 20; + + float sum = RandInt(minint, maxint) + 0.5*(minint+maxint); + for (int j = 0; ; j++) { + sum += RandInt(minint, maxint) - 0.5*(minint+maxint); + if (std::abs(static_cast(sum)) < + 0.5*sqrt(static_cast(j))*(maxint-minint)) break; + } + } + { // test RandPrune in basic way. + KALDI_ASSERT(RandPrune(1.1, 1.0) == 1.1); + KALDI_ASSERT(RandPrune(0.0, 0.0) == 0.0); + KALDI_ASSERT(RandPrune(-1.1, 1.0) == -1.1); + KALDI_ASSERT(RandPrune(0.0, 1.0) == 0.0); + KALDI_ASSERT(RandPrune(0.5, 1.0) >= 0.0); + KALDI_ASSERT(RandPrune(-0.5, 1.0) <= 0.0); + BaseFloat f = RandPrune(-0.5, 1.0); + KALDI_ASSERT(f == 0.0 || f == -1.0); + f = RandPrune(0.5, 1.0); + KALDI_ASSERT(f == 0.0 || f == 1.0); + } + } +} + +void UnitTestLogAddSub() { + for (int i = 0; i < 100; i++) { + double f1 = Rand() % 10000, f2 = Rand() % 20; + double add1 = Exp(LogAdd(Log(f1), Log(f2))); + double add2 = Exp(LogAdd(Log(f2), Log(f1))); + double add = f1 + f2, thresh = add*0.00001; + KALDI_ASSERT(std::abs(add-add1) < thresh && std::abs(add-add2) < thresh); + + + try { + double f2_check = Exp(LogSub(Log(add), Log(f1))), + thresh = (f2*0.01)+0.001; + KALDI_ASSERT(std::abs(f2_check-f2) < thresh); + } catch(...) { + KALDI_ASSERT(f2 == 0); // It will probably crash for f2=0. + } + } +} + +void UnitTestDefines() { // Yes, we even unit-test the preprocessor statements. + KALDI_ASSERT(Exp(kLogZeroFloat) == 0.0); + KALDI_ASSERT(Exp(kLogZeroDouble) == 0.0); + BaseFloat den = 0.0; + KALDI_ASSERT(KALDI_ISNAN(0.0 / den)); + KALDI_ASSERT(!KALDI_ISINF(0.0 / den)); + KALDI_ASSERT(!KALDI_ISFINITE(0.0 / den)); + KALDI_ASSERT(!KALDI_ISNAN(1.0 / den)); + KALDI_ASSERT(KALDI_ISINF(1.0 / den)); + KALDI_ASSERT(!KALDI_ISFINITE(1.0 / den)); + KALDI_ASSERT(KALDI_ISFINITE(0.0)); + KALDI_ASSERT(!KALDI_ISINF(0.0)); + KALDI_ASSERT(!KALDI_ISNAN(0.0)); + + std::cout << 1.0+DBL_EPSILON; + std::cout << 1.0 + 0.5*DBL_EPSILON; + KALDI_ASSERT(1.0 + DBL_EPSILON != 1.0 && 1.0 + (0.5*DBL_EPSILON) == 1.0 + && "If this test fails, you can probably just comment it out-- " + "may mean your CPU exceeds expected floating point precision"); + KALDI_ASSERT(1.0f + FLT_EPSILON != 1.0f && 1.0f + (0.5f*FLT_EPSILON) == 1.0f + && "If this test fails, you can probably just comment it out-- " + "may mean your CPU exceeds expected floating point precision"); + KALDI_ASSERT(std::abs(sin(M_PI)) < 1.0e-05 + && std::abs(cos(M_PI)+1.0) < 1.0e-05); + KALDI_ASSERT(std::abs(sin(M_2PI)) < 1.0e-05 + && std::abs(cos(M_2PI)-1.0) < 1.0e-05); + KALDI_ASSERT(std::abs(sin(Exp(M_LOG_2PI))) < 1.0e-05); + KALDI_ASSERT(std::abs(cos(Exp(M_LOG_2PI)) - 1.0) < 1.0e-05); +} + +void UnitTestAssertFunc() { // Testing Assert** *functions + for (int i = 1; i < 100; i++) { + float f1 = Rand() % 10000 + 1, f2 = Rand() % 20 + 1; + float tmp1 = f1 * f2; + float tmp2 = (1/f1 + 1/f2); + float add = f1 + f2; + float addeql = tmp1 * tmp2; + float thresh = 0.00001; + AssertEqual(add, addeql, thresh); // test AssertEqual() + } +} + +template void UnitTestFactorizeTpl() { + for (int p= 0; p < 100; p++) { + I m = Rand() % 100000; + if (m >= 1) { + std::vector factors; + Factorize(m, &factors); + I m2 = 1; + for (size_t i = 0; i < factors.size(); i++) { + m2 *= factors[i]; + if (i+1 < factors.size()) + KALDI_ASSERT(factors[i+1] >= factors[i]); // check sorted. + } + KALDI_ASSERT(m2 == m); // check correctness. + } + } +} + +void UnitTestFactorize() { + UnitTestFactorizeTpl(); + UnitTestFactorizeTpl(); + UnitTestFactorizeTpl(); +} + +void UnitTestApproxEqual() { + KALDI_ASSERT(ApproxEqual(1.0, 1.00001)); + KALDI_ASSERT(ApproxEqual(1.0, 1.00001, 0.001)); + KALDI_ASSERT(!ApproxEqual(1.0, 1.1)); + KALDI_ASSERT(!ApproxEqual(1.0, 1.01, 0.001)); + KALDI_ASSERT(!ApproxEqual(1.0, 0.0)); + KALDI_ASSERT(ApproxEqual(0.0, 0.0)); + KALDI_ASSERT(!ApproxEqual(0.0, 0.00001)); + KALDI_ASSERT(!ApproxEqual(std::numeric_limits::infinity(), + -std::numeric_limits::infinity())); + KALDI_ASSERT(ApproxEqual(std::numeric_limits::infinity(), + std::numeric_limits::infinity())); + KALDI_ASSERT(ApproxEqual(-std::numeric_limits::infinity(), + -std::numeric_limits::infinity())); + KALDI_ASSERT(!ApproxEqual(-std::numeric_limits::infinity(), + 0)); + KALDI_ASSERT(!ApproxEqual(-std::numeric_limits::infinity(), + 1)); +} + +template +void UnitTestExpSpeed() { + Real sum = 0.0; // compute the sum to avoid optimizing it away. + Real time = 0.01; // how long this should last. + int block_size = 10; + int num_ops = 0; + Timer tim; + while (tim.Elapsed() < time) { + for (int i = 0; i < block_size; i++) { + sum += Exp((Real)i); + } + num_ops += block_size; + } + KALDI_ASSERT(sum > 0.0); // make it harder for the compiler to optimize Exp + // away, as we have a conditional. + Real flops = 1.0e-06 * num_ops / tim.Elapsed(); + KALDI_LOG << "Megaflops doing Exp(" + << (sizeof(Real) == 4 ? "float" : "double") << ") is " << flops; +} + + +template +void UnitTestLogSpeed() { + Real sum = 0.0; // compute the sum to avoid optimizing it away. + Real time = 0.01; // how long this should last. + int block_size = 10; + int num_ops = 0; + Timer tim; + while (tim.Elapsed() < time) { + for (int i = 0; i < block_size; i++) { + sum += Log(static_cast(i + 1)); + } + num_ops += block_size; + } + KALDI_ASSERT(sum > 0.0); // make it harder for the compiler to optimize Log + // away, as we have a conditional. + Real flops = 1.0e-06 * num_ops / tim.Elapsed(); + KALDI_LOG << "Megaflops doing Log(" + << (sizeof(Real) == 4 ? "float" : "double") << ") is " << flops; +} + +} // end namespace kaldi. + +int main() { + using namespace kaldi; + UnitTestApproxEqual(); + UnitTestGcdLcm(); + UnitTestFactorize(); + UnitTestDefines(); + UnitTestLogAddSub(); + UnitTestRand(); + UnitTestAssertFunc(); + UnitTestRoundUpToNearestPowerOfTwo(); + UnitTestDivideRoundingDown(); + UnitTestExpSpeed(); + UnitTestExpSpeed(); + UnitTestLogSpeed(); + UnitTestLogSpeed(); +} + diff --git a/runtime/onnxruntime/third_party/kaldi/base/kaldi-math.cc b/runtime/onnxruntime/third_party/kaldi/base/kaldi-math.cc new file mode 100644 index 000000000..f60d6c0ac --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/base/kaldi-math.cc @@ -0,0 +1,162 @@ +// base/kaldi-math.cc + +// Copyright 2009-2011 Microsoft Corporation; Yanmin Qian; +// Saarland University; Jan Silovsky + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#include "base/kaldi-math.h" +#ifndef _MSC_VER +#include +#include +#endif +#include +#include + +namespace kaldi { +// These routines are tested in matrix/matrix-test.cc + +int32 RoundUpToNearestPowerOfTwo(int32 n) { + KALDI_ASSERT(n > 0); + n--; + n |= n >> 1; + n |= n >> 2; + n |= n >> 4; + n |= n >> 8; + n |= n >> 16; + return n+1; +} + +static std::mutex _RandMutex; + +int Rand(struct RandomState* state) { +#if !defined(_POSIX_THREAD_SAFE_FUNCTIONS) + // On Windows and Cygwin, just call Rand() + return rand(); +#else + if (state) { + return rand_r(&(state->seed)); + } else { + std::lock_guard lock(_RandMutex); + return rand(); + } +#endif +} + +RandomState::RandomState() { + // we initialize it as Rand() + 27437 instead of just Rand(), because on some + // systems, e.g. at the very least Mac OSX Yosemite and later, it seems to be + // the case that rand_r when initialized with rand() will give you the exact + // same sequence of numbers that rand() will give if you keep calling rand() + // after that initial call. This can cause problems with repeated sequences. + // For example if you initialize two RandomState structs one after the other + // without calling rand() in between, they would give you the same sequence + // offset by one (if we didn't have the "+ 27437" in the code). 27437 is just + // a randomly chosen prime number. + seed = unsigned(Rand()) + 27437; +} + +bool WithProb(BaseFloat prob, struct RandomState* state) { + KALDI_ASSERT(prob >= 0 && prob <= 1.1); // prob should be <= 1.0, + // but we allow slightly larger values that could arise from roundoff in + // previous calculations. + KALDI_COMPILE_TIME_ASSERT(RAND_MAX > 128 * 128); + if (prob == 0) return false; + else if (prob == 1.0) return true; + else if (prob * RAND_MAX < 128.0) { + // prob is very small but nonzero, and the "main algorithm" + // wouldn't work that well. So: with probability 1/128, we + // return WithProb (prob * 128), else return false. + if (Rand(state) < RAND_MAX / 128) { // with probability 128... + // Note: we know that prob * 128.0 < 1.0, because + // we asserted RAND_MAX > 128 * 128. + return WithProb(prob * 128.0); + } else { + return false; + } + } else { + return (Rand(state) < ((RAND_MAX + static_cast(1.0)) * prob)); + } +} + +int32 RandInt(int32 min_val, int32 max_val, struct RandomState* state) { + // This is not exact. + KALDI_ASSERT(max_val >= min_val); + if (max_val == min_val) return min_val; + +#ifdef _MSC_VER + // RAND_MAX is quite small on Windows -> may need to handle larger numbers. + if (RAND_MAX > (max_val-min_val)*8) { + // *8 to avoid large inaccuracies in probability, from the modulus... + return min_val + + ((unsigned int)Rand(state) % (unsigned int)(max_val+1-min_val)); + } else { + if ((unsigned int)(RAND_MAX*RAND_MAX) > + (unsigned int)((max_val+1-min_val)*8)) { + // *8 to avoid inaccuracies in probability, from the modulus... + return min_val + ( (unsigned int)( (Rand(state)+RAND_MAX*Rand(state))) + % (unsigned int)(max_val+1-min_val)); + } else { + KALDI_ERR << "rand_int failed because we do not support such large " + "random numbers. (Extend this function)."; + } + } +#else + return min_val + + (static_cast(Rand(state)) % static_cast(max_val+1-min_val)); +#endif +} + +// Returns poisson-distributed random number. +// Take care: this takes time proportional +// to lambda. Faster algorithms exist but are more complex. +int32 RandPoisson(float lambda, struct RandomState* state) { + // Knuth's algorithm. + KALDI_ASSERT(lambda >= 0); + float L = expf(-lambda), p = 1.0; + int32 k = 0; + do { + k++; + float u = RandUniform(state); + p *= u; + } while (p > L); + return k-1; +} + +void RandGauss2(float *a, float *b, RandomState *state) { + KALDI_ASSERT(a); + KALDI_ASSERT(b); + float u1 = RandUniform(state); + float u2 = RandUniform(state); + u1 = sqrtf(-2.0f * logf(u1)); + u2 = 2.0f * M_PI * u2; + *a = u1 * cosf(u2); + *b = u1 * sinf(u2); +} + +void RandGauss2(double *a, double *b, RandomState *state) { + KALDI_ASSERT(a); + KALDI_ASSERT(b); + float a_float, b_float; + // Just because we're using doubles doesn't mean we need super-high-quality + // random numbers, so we just use the floating-point version internally. + RandGauss2(&a_float, &b_float, state); + *a = a_float; + *b = b_float; +} + + +} // end namespace kaldi diff --git a/runtime/onnxruntime/third_party/kaldi/base/kaldi-math.h b/runtime/onnxruntime/third_party/kaldi/base/kaldi-math.h new file mode 100644 index 000000000..93c265ee9 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/base/kaldi-math.h @@ -0,0 +1,363 @@ +// base/kaldi-math.h + +// Copyright 2009-2011 Ondrej Glembek; Microsoft Corporation; Yanmin Qian; +// Jan Silovsky; Saarland University +// +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#ifndef KALDI_BASE_KALDI_MATH_H_ +#define KALDI_BASE_KALDI_MATH_H_ 1 + +#ifdef _MSC_VER +#include +#endif + +#include +#include +#include + +#include "base/kaldi-types.h" +#include "base/kaldi-common.h" + + +#ifndef DBL_EPSILON +#define DBL_EPSILON 2.2204460492503131e-16 +#endif +#ifndef FLT_EPSILON +#define FLT_EPSILON 1.19209290e-7f +#endif + +#ifndef M_PI +#define M_PI 3.1415926535897932384626433832795 +#endif + +#ifndef M_SQRT2 +#define M_SQRT2 1.4142135623730950488016887 +#endif + +#ifndef M_2PI +#define M_2PI 6.283185307179586476925286766559005 +#endif + +#ifndef M_SQRT1_2 +#define M_SQRT1_2 0.7071067811865475244008443621048490 +#endif + +#ifndef M_LOG_2PI +#define M_LOG_2PI 1.8378770664093454835606594728112 +#endif + +#ifndef M_LN2 +#define M_LN2 0.693147180559945309417232121458 +#endif + +#ifndef M_LN10 +#define M_LN10 2.302585092994045684017991454684 +#endif + + +#define KALDI_ISNAN std::isnan +#define KALDI_ISINF std::isinf +#define KALDI_ISFINITE(x) std::isfinite(x) + +#if !defined(KALDI_SQR) +# define KALDI_SQR(x) ((x) * (x)) +#endif + +namespace kaldi { + +#if !defined(_MSC_VER) || (_MSC_VER >= 1900) +inline double Exp(double x) { return exp(x); } +#ifndef KALDI_NO_EXPF +inline float Exp(float x) { return expf(x); } +#else +inline float Exp(float x) { return exp(static_cast(x)); } +#endif // KALDI_NO_EXPF +#else +inline double Exp(double x) { return exp(x); } +#if !defined(__INTEL_COMPILER) && _MSC_VER == 1800 && defined(_M_X64) +// Microsoft CL v18.0 buggy 64-bit implementation of +// expf() incorrectly returns -inf for exp(-inf). +inline float Exp(float x) { return exp(static_cast(x)); } +#else +inline float Exp(float x) { return expf(x); } +#endif // !defined(__INTEL_COMPILER) && _MSC_VER == 1800 && defined(_M_X64) +#endif // !defined(_MSC_VER) || (_MSC_VER >= 1900) + +inline double Log(double x) { return log(x); } +inline float Log(float x) { return logf(x); } + +#if !defined(_MSC_VER) || (_MSC_VER >= 1700) +inline double Log1p(double x) { return log1p(x); } +inline float Log1p(float x) { return log1pf(x); } +#else +inline double Log1p(double x) { + const double cutoff = 1.0e-08; + if (x < cutoff) + return x - 0.5 * x * x; + else + return Log(1.0 + x); +} + +inline float Log1p(float x) { + const float cutoff = 1.0e-07; + if (x < cutoff) + return x - 0.5 * x * x; + else + return Log(1.0 + x); +} +#endif + +static const double kMinLogDiffDouble = Log(DBL_EPSILON); // negative! +static const float kMinLogDiffFloat = Log(FLT_EPSILON); // negative! + +// -infinity +const float kLogZeroFloat = -std::numeric_limits::infinity(); +const double kLogZeroDouble = -std::numeric_limits::infinity(); +const BaseFloat kLogZeroBaseFloat = -std::numeric_limits::infinity(); + +// Returns a random integer between 0 and RAND_MAX, inclusive +int Rand(struct RandomState* state = NULL); + +// State for thread-safe random number generator +struct RandomState { + RandomState(); + unsigned seed; +}; + +// Returns a random integer between first and last inclusive. +int32 RandInt(int32 first, int32 last, struct RandomState* state = NULL); + +// Returns true with probability "prob", +bool WithProb(BaseFloat prob, struct RandomState* state = NULL); +// with 0 <= prob <= 1 [we check this]. +// Internally calls Rand(). This function is carefully implemented so +// that it should work even if prob is very small. + +/// Returns a random number strictly between 0 and 1. +inline float RandUniform(struct RandomState* state = NULL) { + return static_cast((Rand(state) + 1.0) / (RAND_MAX+2.0)); +} + +inline float RandGauss(struct RandomState* state = NULL) { + return static_cast(sqrtf (-2 * Log(RandUniform(state))) + * cosf(2*M_PI*RandUniform(state))); +} + +// Returns poisson-distributed random number. Uses Knuth's algorithm. +// Take care: this takes time proportional +// to lambda. Faster algorithms exist but are more complex. +int32 RandPoisson(float lambda, struct RandomState* state = NULL); + +// Returns a pair of gaussian random numbers. Uses Box-Muller transform +void RandGauss2(float *a, float *b, RandomState *state = NULL); +void RandGauss2(double *a, double *b, RandomState *state = NULL); + +// Also see Vector::RandCategorical(). + +// This is a randomized pruning mechanism that preserves expectations, +// that we typically use to prune posteriors. +template +inline Float RandPrune(Float post, BaseFloat prune_thresh, + struct RandomState* state = NULL) { + KALDI_ASSERT(prune_thresh >= 0.0); + if (post == 0.0 || std::abs(post) >= prune_thresh) + return post; + return (post >= 0 ? 1.0 : -1.0) * + (RandUniform(state) <= fabs(post)/prune_thresh ? prune_thresh : 0.0); +} + +// returns log(exp(x) + exp(y)). +inline double LogAdd(double x, double y) { + double diff; + + if (x < y) { + diff = x - y; + x = y; + } else { + diff = y - x; + } + // diff is negative. x is now the larger one. + + if (diff >= kMinLogDiffDouble) { + double res; + res = x + Log1p(Exp(diff)); + return res; + } else { + return x; // return the larger one. + } +} + + +// returns log(exp(x) + exp(y)). +inline float LogAdd(float x, float y) { + float diff; + + if (x < y) { + diff = x - y; + x = y; + } else { + diff = y - x; + } + // diff is negative. x is now the larger one. + + if (diff >= kMinLogDiffFloat) { + float res; + res = x + Log1p(Exp(diff)); + return res; + } else { + return x; // return the larger one. + } +} + + +// returns log(exp(x) - exp(y)). +inline double LogSub(double x, double y) { + if (y >= x) { // Throws exception if y>=x. + if (y == x) + return kLogZeroDouble; + else + KALDI_ERR << "Cannot subtract a larger from a smaller number."; + } + + double diff = y - x; // Will be negative. + double res = x + Log(1.0 - Exp(diff)); + + // res might be NAN if diff ~0.0, and 1.0-exp(diff) == 0 to machine precision + if (KALDI_ISNAN(res)) + return kLogZeroDouble; + return res; +} + + +// returns log(exp(x) - exp(y)). +inline float LogSub(float x, float y) { + if (y >= x) { // Throws exception if y>=x. + if (y == x) + return kLogZeroDouble; + else + KALDI_ERR << "Cannot subtract a larger from a smaller number."; + } + + float diff = y - x; // Will be negative. + float res = x + Log(1.0f - Exp(diff)); + + // res might be NAN if diff ~0.0, and 1.0-exp(diff) == 0 to machine precision + if (KALDI_ISNAN(res)) + return kLogZeroFloat; + return res; +} + +/// return abs(a - b) <= relative_tolerance * (abs(a)+abs(b)). +static inline bool ApproxEqual(float a, float b, + float relative_tolerance = 0.001) { + // a==b handles infinities. + if (a == b) return true; + float diff = std::abs(a-b); + if (diff == std::numeric_limits::infinity() + || diff != diff) return false; // diff is +inf or nan. + return (diff <= relative_tolerance*(std::abs(a)+std::abs(b))); +} + +/// assert abs(a - b) <= relative_tolerance * (abs(a)+abs(b)) +static inline void AssertEqual(float a, float b, + float relative_tolerance = 0.001) { + // a==b handles infinities. + KALDI_ASSERT(ApproxEqual(a, b, relative_tolerance)); +} + + +// RoundUpToNearestPowerOfTwo does the obvious thing. It crashes if n <= 0. +int32 RoundUpToNearestPowerOfTwo(int32 n); + +/// Returns a / b, rounding towards negative infinity in all cases. +static inline int32 DivideRoundingDown(int32 a, int32 b) { + KALDI_ASSERT(b != 0); + if (a * b >= 0) + return a / b; + else if (a < 0) + return (a - b + 1) / b; + else + return (a - b - 1) / b; +} + +template I Gcd(I m, I n) { + if (m == 0 || n == 0) { + if (m == 0 && n == 0) { // gcd not defined, as all integers are divisors. + KALDI_ERR << "Undefined GCD since m = 0, n = 0."; + } + return (m == 0 ? (n > 0 ? n : -n) : ( m > 0 ? m : -m)); + // return absolute value of whichever is nonzero + } + // could use compile-time assertion + // but involves messing with complex template stuff. + KALDI_ASSERT(std::numeric_limits::is_integer); + while (1) { + m %= n; + if (m == 0) return (n > 0 ? n : -n); + n %= m; + if (n == 0) return (m > 0 ? m : -m); + } +} + +/// Returns the least common multiple of two integers. Will +/// crash unless the inputs are positive. +template I Lcm(I m, I n) { + KALDI_ASSERT(m > 0 && n > 0); + I gcd = Gcd(m, n); + return gcd * (m/gcd) * (n/gcd); +} + + +template void Factorize(I m, std::vector *factors) { + // Splits a number into its prime factors, in sorted order from + // least to greatest, with duplication. A very inefficient + // algorithm, which is mainly intended for use in the + // mixed-radix FFT computation (where we assume most factors + // are small). + KALDI_ASSERT(factors != NULL); + KALDI_ASSERT(m >= 1); // Doesn't work for zero or negative numbers. + factors->clear(); + I small_factors[10] = { 2, 3, 5, 7, 11, 13, 17, 19, 23, 29 }; + + // First try small factors. + for (I i = 0; i < 10; i++) { + if (m == 1) return; // We're done. + while (m % small_factors[i] == 0) { + m /= small_factors[i]; + factors->push_back(small_factors[i]); + } + } + // Next try all odd numbers starting from 31. + for (I j = 31;; j += 2) { + if (m == 1) return; + while (m % j == 0) { + m /= j; + factors->push_back(j); + } + } +} + +inline double Hypot(double x, double y) { return hypot(x, y); } +inline float Hypot(float x, float y) { return hypotf(x, y); } + + + + +} // namespace kaldi + + +#endif // KALDI_BASE_KALDI_MATH_H_ diff --git a/runtime/onnxruntime/third_party/kaldi/base/kaldi-types.h b/runtime/onnxruntime/third_party/kaldi/base/kaldi-types.h new file mode 100644 index 000000000..7ebf4f853 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/base/kaldi-types.h @@ -0,0 +1,75 @@ +// base/kaldi-types.h + +// Copyright 2009-2011 Microsoft Corporation; Saarland University; +// Jan Silovsky; Yanmin Qian + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#ifndef KALDI_BASE_KALDI_TYPES_H_ +#define KALDI_BASE_KALDI_TYPES_H_ 1 + +namespace kaldi { +// TYPEDEFS .................................................................. +#if (KALDI_DOUBLEPRECISION != 0) +typedef double BaseFloat; +#else +typedef float BaseFloat; +#endif +} + +#ifdef _MSC_VER +#include +#define ssize_t SSIZE_T +#endif + +// we can do this a different way if some platform +// we find in the future lacks stdint.h +#include + +// for discussion on what to do if you need compile kaldi +// without OpenFST, see the bottom of this this file +#include + +namespace kaldi { + using ::int16; + using ::int32; + using ::int64; + using ::uint16; + using ::uint32; + using ::uint64; + typedef float float32; + typedef double double64; +} // end namespace kaldi + +// In a theoretical case you decide compile Kaldi without the OpenFST +// comment the previous namespace statement and uncomment the following +/* +namespace kaldi { + typedef int8_t int8; + typedef int16_t int16; + typedef int32_t int32; + typedef int64_t int64; + + typedef uint8_t uint8; + typedef uint16_t uint16; + typedef uint32_t uint32; + typedef uint64_t uint64; + typedef float float32; + typedef double double64; +} // end namespace kaldi +*/ + +#endif // KALDI_BASE_KALDI_TYPES_H_ diff --git a/runtime/onnxruntime/third_party/kaldi/base/kaldi-utils.cc b/runtime/onnxruntime/third_party/kaldi/base/kaldi-utils.cc new file mode 100644 index 000000000..0c0f1d4f3 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/base/kaldi-utils.cc @@ -0,0 +1,46 @@ +// base/kaldi-utils.cc +// Copyright 2009-2011 Karel Vesely; Yanmin Qian; Microsoft Corporation + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#include "base/kaldi-utils.h" + +#include +#include +#include +#include + + +namespace kaldi { + +std::string CharToString(const char &c) { + char buf[20]; + if (std::isprint(c)) + std::snprintf(buf, sizeof(buf), "\'%c\'", c); + else + std::snprintf(buf, sizeof(buf), "[character %d]", static_cast(c)); + return buf; +} + +void Sleep(double sec) { + // duration_cast<> rounds down, add 0.5 to compensate. + auto dur_nanos = std::chrono::duration(sec * 1E9 + 0.5); + auto dur_syshires = std::chrono::duration_cast< + typename std::chrono::high_resolution_clock::duration>(dur_nanos); + std::this_thread::sleep_for(dur_syshires); +} + +} // end namespace kaldi diff --git a/runtime/onnxruntime/third_party/kaldi/base/kaldi-utils.h b/runtime/onnxruntime/third_party/kaldi/base/kaldi-utils.h new file mode 100644 index 000000000..9f2c897ba --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/base/kaldi-utils.h @@ -0,0 +1,162 @@ +// base/kaldi-utils.h + +// Copyright 2009-2011 Ondrej Glembek; Microsoft Corporation; +// Saarland University; Karel Vesely; Yanmin Qian + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#ifndef KALDI_BASE_KALDI_UTILS_H_ +#define KALDI_BASE_KALDI_UTILS_H_ 1 + +#if _MSC_VER +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN 1 +#endif +#ifndef NOMINMAX +#define NOMINMAX 1 +#endif +#include +#endif + +#ifdef _MSC_VER +#include +#define unlink _unlink +#else +#include +#endif + +#include +#include + +#if defined(_MSC_VER) +#pragma warning(disable: 4244 4056 4305 4800 4267 4996 4756 4661) +#if _MSC_VER < 1400 +#define __restrict__ +#else +#define __restrict__ __restrict +#endif +#endif + +#if defined(_MSC_VER) +# define KALDI_MEMALIGN(align, size, pp_orig) \ + (*(pp_orig) = _aligned_malloc(size, align)) +# define KALDI_MEMALIGN_FREE(x) _aligned_free(x) +#elif defined(__CYGWIN__) +# define KALDI_MEMALIGN(align, size, pp_orig) \ + (*(pp_orig) = aligned_alloc(align, size)) +# define KALDI_MEMALIGN_FREE(x) free(x) +#else +# define KALDI_MEMALIGN(align, size, pp_orig) \ + (!posix_memalign(pp_orig, align, size) ? *(pp_orig) : NULL) +# define KALDI_MEMALIGN_FREE(x) free(x) +#endif + +#ifdef __ICC +#pragma warning(disable: 383) // ICPC remark we don't want. +#pragma warning(disable: 810) // ICPC remark we don't want. +#pragma warning(disable: 981) // ICPC remark we don't want. +#pragma warning(disable: 1418) // ICPC remark we don't want. +#pragma warning(disable: 444) // ICPC remark we don't want. +#pragma warning(disable: 869) // ICPC remark we don't want. +#pragma warning(disable: 1287) // ICPC remark we don't want. +#pragma warning(disable: 279) // ICPC remark we don't want. +#pragma warning(disable: 981) // ICPC remark we don't want. +#endif + + +namespace kaldi { + + +// CharToString prints the character in a human-readable form, for debugging. +std::string CharToString(const char &c); + + +inline int MachineIsLittleEndian() { + int check = 1; + return (*reinterpret_cast(&check) != 0); +} + +// This function kaldi::Sleep() provides a portable way +// to sleep for a possibly fractional +// number of seconds. On Windows it's only accurate to microseconds. +void Sleep(double seconds); +} + +#define KALDI_SWAP8(a) do { \ + int t = (reinterpret_cast(&a))[0];\ + (reinterpret_cast(&a))[0]=(reinterpret_cast(&a))[7];\ + (reinterpret_cast(&a))[7]=t;\ + t = (reinterpret_cast(&a))[1];\ + (reinterpret_cast(&a))[1]=(reinterpret_cast(&a))[6];\ + (reinterpret_cast(&a))[6]=t;\ + t = (reinterpret_cast(&a))[2];\ + (reinterpret_cast(&a))[2]=(reinterpret_cast(&a))[5];\ + (reinterpret_cast(&a))[5]=t;\ + t = (reinterpret_cast(&a))[3];\ + (reinterpret_cast(&a))[3]=(reinterpret_cast(&a))[4];\ + (reinterpret_cast(&a))[4]=t;} while (0) +#define KALDI_SWAP4(a) do { \ + int t = (reinterpret_cast(&a))[0];\ + (reinterpret_cast(&a))[0]=(reinterpret_cast(&a))[3];\ + (reinterpret_cast(&a))[3]=t;\ + t = (reinterpret_cast(&a))[1];\ + (reinterpret_cast(&a))[1]=(reinterpret_cast(&a))[2];\ + (reinterpret_cast(&a))[2]=t;} while (0) +#define KALDI_SWAP2(a) do { \ + int t = (reinterpret_cast(&a))[0];\ + (reinterpret_cast(&a))[0]=(reinterpret_cast(&a))[1];\ + (reinterpret_cast(&a))[1]=t;} while (0) + + +///\brief Declare deleted copy constructor and copy assignment operator=(). +/// +/// Use this macro in the \e public part of a class declaration in a header +/// file, next to its constructors, so that uncopyability of the type is +/// clearly readable. Place a semicolon after it. +///\param type The exact enclosing class name. +#define KALDI_DISALLOW_COPY_AND_ASSIGN(type) \ + type(const type&) = delete; \ + type& operator=(const type&) = delete + +#if __cplusplus >= 201703L +#define KALDI_COMPILE_TIME_ASSERT static_assert +#else +#define KALDI_COMPILE_TIME_ASSERT(b) static_assert((b), #b) +#endif + +#define KALDI_ASSERT_IS_INTEGER_TYPE(I) \ + KALDI_COMPILE_TIME_ASSERT(std::numeric_limits::is_specialized \ + && std::numeric_limits::is_integer) + +#define KALDI_ASSERT_IS_FLOATING_TYPE(F) \ + KALDI_COMPILE_TIME_ASSERT(std::numeric_limits::is_specialized \ + && !std::numeric_limits::is_integer) + +#if defined(_MSC_VER) +#define KALDI_STRCASECMP _stricmp +#elif defined(__CYGWIN__) +#include +#define KALDI_STRCASECMP strcasecmp +#else +#define KALDI_STRCASECMP strcasecmp +#endif +#ifdef _MSC_VER +# define KALDI_STRTOLL(cur_cstr, end_cstr) _strtoi64(cur_cstr, end_cstr, 10); +#else +# define KALDI_STRTOLL(cur_cstr, end_cstr) strtoll(cur_cstr, end_cstr, 10); +#endif + +#endif // KALDI_BASE_KALDI_UTILS_H_ diff --git a/runtime/onnxruntime/third_party/kaldi/base/timer-test.cc b/runtime/onnxruntime/third_party/kaldi/base/timer-test.cc new file mode 100644 index 000000000..86a20b486 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/base/timer-test.cc @@ -0,0 +1,45 @@ +// base/timer-test.cc + +// Copyright 2009-2011 Microsoft Corporation +// 2014 Johns Hopkins University (author: Daniel Povey) + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#include "base/timer.h" +#include "base/kaldi-common.h" +#include "base/kaldi-utils.h" + + +namespace kaldi { + +void TimerTest() { + float time_secs = 0.025 * (rand() % 10); + std::cout << "target is " << time_secs << "\n"; + Timer timer; + Sleep(time_secs); + BaseFloat f = timer.Elapsed(); + std::cout << "time is " << f << std::endl; + if (fabs(time_secs - f) > 0.05) + KALDI_ERR << "Timer fail: waited " << f << " seconds instead of " + << time_secs << " secs."; +} +} + + +int main() { + for (int i = 0; i < 4; i++) + kaldi::TimerTest(); +} diff --git a/runtime/onnxruntime/third_party/kaldi/base/timer.cc b/runtime/onnxruntime/third_party/kaldi/base/timer.cc new file mode 100644 index 000000000..ce4ef2927 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/base/timer.cc @@ -0,0 +1,85 @@ +// base/timer.cc + +// Copyright 2018 Johns Hopkins University (author: Daniel Povey) + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#include "base/timer.h" +#include "base/kaldi-error.h" +#include +#include +#include +#include + +namespace kaldi { + +class ProfileStats { + public: + void AccStats(const char *function_name, double elapsed) { + std::unordered_map::iterator + iter = map_.find(function_name); + if (iter == map_.end()) { + map_[function_name] = ProfileStatsEntry(function_name); + map_[function_name].total_time = elapsed; + } else { + iter->second.total_time += elapsed; + } + } + ~ProfileStats() { + // This map makes sure we agglomerate the time if there were any duplicate + // addresses of strings. + std::unordered_map total_time; + for (auto iter = map_.begin(); iter != map_.end(); iter++) + total_time[iter->second.name] += iter->second.total_time; + + ReverseSecondComparator comp; + std::vector > pairs(total_time.begin(), + total_time.end()); + std::sort(pairs.begin(), pairs.end(), comp); + for (size_t i = 0; i < pairs.size(); i++) { + KALDI_LOG << "Time taken in " << pairs[i].first << " is " + << std::fixed << std::setprecision(2) << pairs[i].second << "s."; + } + } + private: + + struct ProfileStatsEntry { + std::string name; + double total_time; + ProfileStatsEntry() { } + ProfileStatsEntry(const char *name): name(name) { } + }; + + struct ReverseSecondComparator { + bool operator () (const std::pair &a, + const std::pair &b) { + return a.second > b.second; + } + }; + + // Note: this map is keyed on the address of the string, there is no proper + // hash function. The assumption is that the strings are compile-time + // constants. + std::unordered_map map_; +}; + +ProfileStats g_profile_stats; + +Profiler::~Profiler() { + g_profile_stats.AccStats(name_, tim_.Elapsed()); +} + +} // namespace kaldi diff --git a/runtime/onnxruntime/third_party/kaldi/base/timer.h b/runtime/onnxruntime/third_party/kaldi/base/timer.h new file mode 100644 index 000000000..0e0337663 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/base/timer.h @@ -0,0 +1,115 @@ +// base/timer.h + +// Copyright 2009-2011 Ondrej Glembek; Microsoft Corporation + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. +#ifndef KALDI_BASE_TIMER_H_ +#define KALDI_BASE_TIMER_H_ + +#include "base/kaldi-utils.h" +#include "base/kaldi-error.h" + + +#if defined(_MSC_VER) || defined(MINGW) + +namespace kaldi { +class Timer { + public: + Timer() { Reset(); } + + // You can initialize with bool to control whether or not you want the time to + // be set when the object is created. + explicit Timer(bool set_timer) { if (set_timer) Reset(); } + + void Reset() { + QueryPerformanceCounter(&time_start_); + } + double Elapsed() const { + LARGE_INTEGER time_end; + LARGE_INTEGER freq; + QueryPerformanceCounter(&time_end); + + if (QueryPerformanceFrequency(&freq) == 0) { + // Hardware does not support this. + return 0.0; + } + return (static_cast(time_end.QuadPart) - + static_cast(time_start_.QuadPart)) / + (static_cast(freq.QuadPart)); + } + private: + LARGE_INTEGER time_start_; +}; + + +#else +#include +#include + +namespace kaldi { +class Timer { + public: + Timer() { Reset(); } + + // You can initialize with bool to control whether or not you want the time to + // be set when the object is created. + explicit Timer(bool set_timer) { if (set_timer) Reset(); } + + void Reset() { gettimeofday(&this->time_start_, &time_zone_); } + + /// Returns time in seconds. + double Elapsed() const { + struct timeval time_end; + struct timezone time_zone; + gettimeofday(&time_end, &time_zone); + double t1, t2; + t1 = static_cast(time_start_.tv_sec) + + static_cast(time_start_.tv_usec)/(1000*1000); + t2 = static_cast(time_end.tv_sec) + + static_cast(time_end.tv_usec)/(1000*1000); + return t2-t1; + } + + private: + struct timeval time_start_; + struct timezone time_zone_; +}; + +#endif + +class Profiler { + public: + // Caution: the 'const char' should always be a string constant; for speed, + // internally the profiling code uses the address of it as a lookup key. + Profiler(const char *function_name): name_(function_name) { } + ~Profiler(); + private: + Timer tim_; + const char *name_; +}; + +// To add timing info for a function, you just put +// KALDI_PROFILE; +// at the beginning of the function. Caution: this doesn't +// include the class name. +#define KALDI_PROFILE Profiler _profiler(__func__) + + + +} // namespace kaldi + + +#endif // KALDI_BASE_TIMER_H_ diff --git a/runtime/onnxruntime/third_party/kaldi/decoder/Makefile b/runtime/onnxruntime/third_party/kaldi/decoder/Makefile new file mode 100644 index 000000000..a814931f6 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/decoder/Makefile @@ -0,0 +1,20 @@ +all: + +EXTRA_CXXFLAGS = -Wno-sign-compare +include ../kaldi.mk + +TESTFILES = + +OBJFILES = training-graph-compiler.o lattice-simple-decoder.o lattice-faster-decoder.o \ + lattice-faster-online-decoder.o simple-decoder.o faster-decoder.o \ + decoder-wrappers.o grammar-fst.o decodable-matrix.o \ + lattice-incremental-decoder.o lattice-incremental-online-decoder.o + +LIBNAME = kaldi-decoder + +ADDLIBS = ../lat/kaldi-lat.a ../fstext/kaldi-fstext.a ../hmm/kaldi-hmm.a \ + ../transform/kaldi-transform.a ../gmm/kaldi-gmm.a \ + ../tree/kaldi-tree.a ../util/kaldi-util.a ../matrix/kaldi-matrix.a \ + ../base/kaldi-base.a + +include ../makefiles/default_rules.mk diff --git a/runtime/onnxruntime/third_party/kaldi/decoder/biglm-faster-decoder.h b/runtime/onnxruntime/third_party/kaldi/decoder/biglm-faster-decoder.h new file mode 100644 index 000000000..8e36deb8b --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/decoder/biglm-faster-decoder.h @@ -0,0 +1,502 @@ +// decoder/biglm-faster-decoder.h + +// Copyright 2009-2011 Microsoft Corporation, Gilles Boulianne + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#ifndef KALDI_DECODER_BIGLM_FASTER_DECODER_H_ +#define KALDI_DECODER_BIGLM_FASTER_DECODER_H_ + +#include "util/stl-utils.h" +#include "util/hash-list.h" +#include "fst/fstlib.h" +#include "itf/decodable-itf.h" +#include "lat/kaldi-lattice.h" // for CompactLatticeArc +#include "decoder/faster-decoder.h" // for options class +#include "fstext/deterministic-fst.h" + +namespace kaldi { + +struct BiglmFasterDecoderOptions: public FasterDecoderOptions { + BiglmFasterDecoderOptions() { + min_active = 200; + } +}; + +/** This is as FasterDecoder, but does online composition between + HCLG and the "difference language model", which is a deterministic + FST that represents the difference between the language model you want + and the language model you compiled HCLG with. The class + DeterministicOnDemandFst follows through the epsilons in G for you + (assuming G is a standard backoff language model) and makes it look + like a determinized FST. Actually, in practice, + DeterministicOnDemandFst operates in a mode where it composes two + G's together; one has negated likelihoods and works by removing the + LM probabilities that you made HCLG with, and one is the language model + you want to use. +*/ +class BiglmFasterDecoder { + public: + typedef fst::StdArc Arc; + typedef Arc::Label Label; + typedef Arc::StateId StateId; + // A PairId will be constructed as: (StateId in fst) + (StateId in lm_diff_fst) << 32; + typedef uint64 PairId; + typedef Arc::Weight Weight; + + // This constructor is the same as for FasterDecoder, except the second + // argument (lm_diff_fst) is new; it's an FST (actually, a + // DeterministicOnDemandFst) that represents the difference in LM scores + // between the LM we want and the LM the decoding-graph "fst" was built with. + // See e.g. gmm-decode-biglm-faster.cc for an example of how this is called. + // Basically, we are using fst o lm_diff_fst (where o is composition) + // as the decoding graph. Instead of having everything indexed by the state in + // "fst", we now index by the pair of states in (fst, lm_diff_fst). + // Whenever we cross a word, we need to propagate the state within + // lm_diff_fst. + BiglmFasterDecoder(const fst::Fst &fst, + const BiglmFasterDecoderOptions &opts, + fst::DeterministicOnDemandFst *lm_diff_fst): + fst_(fst), lm_diff_fst_(lm_diff_fst), opts_(opts), warned_noarc_(false) { + KALDI_ASSERT(opts_.hash_ratio >= 1.0); // less doesn't make much sense. + KALDI_ASSERT(opts_.max_active > 1); + KALDI_ASSERT(fst.Start() != fst::kNoStateId && + lm_diff_fst->Start() != fst::kNoStateId); + toks_.SetSize(1000); // just so on the first frame we do something reasonable. + } + + void SetOptions(const BiglmFasterDecoderOptions &opts) { opts_ = opts; } + + ~BiglmFasterDecoder() { + ClearToks(toks_.Clear()); + } + + void Decode(DecodableInterface *decodable) { + // clean up from last time: + ClearToks(toks_.Clear()); + PairId start_pair = ConstructPair(fst_.Start(), lm_diff_fst_->Start()); + Arc dummy_arc(0, 0, Weight::One(), fst_.Start()); // actually, the last element of + // the Arcs (fst_.Start(), here) is never needed. + toks_.Insert(start_pair, new Token(dummy_arc, NULL)); + ProcessNonemitting(std::numeric_limits::max()); + for (int32 frame = 0; !decodable->IsLastFrame(frame-1); frame++) { + BaseFloat weight_cutoff = ProcessEmitting(decodable, frame); + ProcessNonemitting(weight_cutoff); + } + } + + bool ReachedFinal() { + for (const Elem *e = toks_.GetList(); e != NULL; e = e->tail) { + PairId state_pair = e->key; + StateId state = PairToState(state_pair), + lm_state = PairToLmState(state_pair); + Weight this_weight = + Times(e->val->weight_, + Times(fst_.Final(state), lm_diff_fst_->Final(lm_state))); + if (this_weight != Weight::Zero()) + return true; + } + return false; + } + + bool GetBestPath(fst::MutableFst *fst_out, + bool use_final_probs = true) { + // GetBestPath gets the decoding output. If "use_final_probs" is true + // AND we reached a final state, it limits itself to final states; + // otherwise it gets the most likely token not taking into + // account final-probs. fst_out will be empty (Start() == kNoStateId) if + // nothing was available. It returns true if it got output (thus, fst_out + // will be nonempty). + fst_out->DeleteStates(); + Token *best_tok = NULL; + Weight best_final = Weight::Zero(); // set only if is_final == true. The + // final-prob corresponding to the best final token (i.e. the one with best + // weight best_weight, below). + bool is_final = ReachedFinal(); + if (!is_final) { + for (const Elem *e = toks_.GetList(); e != NULL; e = e->tail) + if (best_tok == NULL || *best_tok < *(e->val) ) + best_tok = e->val; + } else { + Weight best_weight = Weight::Zero(); + for (const Elem *e = toks_.GetList(); e != NULL; e = e->tail) { + Weight fst_final = fst_.Final(PairToState(e->key)), + lm_final = lm_diff_fst_->Final(PairToLmState(e->key)), + final = Times(fst_final, lm_final); + Weight this_weight = Times(e->val->weight_, final); + if (this_weight != Weight::Zero() && + this_weight.Value() < best_weight.Value()) { + best_weight = this_weight; + best_final = final; + best_tok = e->val; + } + } + } + if (best_tok == NULL) return false; // No output. + + std::vector arcs_reverse; // arcs in reverse order. + + for (Token *tok = best_tok; tok != NULL; tok = tok->prev_) { + BaseFloat tot_cost = tok->weight_.Value() - + (tok->prev_ ? tok->prev_->weight_.Value() : 0.0), + graph_cost = tok->arc_.weight.Value(), + ac_cost = tot_cost - graph_cost; + LatticeArc l_arc(tok->arc_.ilabel, + tok->arc_.olabel, + LatticeWeight(graph_cost, ac_cost), + tok->arc_.nextstate); + arcs_reverse.push_back(l_arc); + } + KALDI_ASSERT(arcs_reverse.back().nextstate == fst_.Start()); + arcs_reverse.pop_back(); // that was a "fake" token... gives no info. + + StateId cur_state = fst_out->AddState(); + fst_out->SetStart(cur_state); + for (ssize_t i = static_cast(arcs_reverse.size())-1; i >= 0; i--) { + LatticeArc arc = arcs_reverse[i]; + arc.nextstate = fst_out->AddState(); + fst_out->AddArc(cur_state, arc); + cur_state = arc.nextstate; + } + if (is_final && use_final_probs) { + fst_out->SetFinal(cur_state, LatticeWeight(best_final.Value(), 0.0)); + } else { + fst_out->SetFinal(cur_state, LatticeWeight::One()); + } + RemoveEpsLocal(fst_out); + return true; + } + + private: + inline PairId ConstructPair(StateId fst_state, StateId lm_state) { + return static_cast(fst_state) + (static_cast(lm_state) << 32); + } + + static inline StateId PairToState(PairId state_pair) { + return static_cast(static_cast(state_pair)); + } + static inline StateId PairToLmState(PairId state_pair) { + return static_cast(static_cast(state_pair >> 32)); + } + + class Token { + public: + Arc arc_; // contains only the graph part of the cost, + // including the part in "fst" (== HCLG) plus lm_diff_fst. + // We can work out the acoustic part from difference between + // "weight_" and prev->weight_. + Token *prev_; + int32 ref_count_; + Weight weight_; // weight up to current point. + inline Token(const Arc &arc, Weight &ac_weight, Token *prev): + arc_(arc), prev_(prev), ref_count_(1) { + if (prev) { + prev->ref_count_++; + weight_ = Times(Times(prev->weight_, arc.weight), ac_weight); + } else { + weight_ = Times(arc.weight, ac_weight); + } + } + inline Token(const Arc &arc, Token *prev): + arc_(arc), prev_(prev), ref_count_(1) { + if (prev) { + prev->ref_count_++; + weight_ = Times(prev->weight_, arc.weight); + } else { + weight_ = arc.weight; + } + } + inline bool operator < (const Token &other) { + return weight_.Value() > other.weight_.Value(); + // This makes sense for log + tropical semiring. + } + + inline ~Token() { + KALDI_ASSERT(ref_count_ == 1); + if (prev_ != NULL) TokenDelete(prev_); + } + inline static void TokenDelete(Token *tok) { + if (tok->ref_count_ == 1) { + delete tok; + } else { + tok->ref_count_--; + } + } + }; + typedef HashList::Elem Elem; + + + /// Gets the weight cutoff. Also counts the active tokens. + BaseFloat GetCutoff(Elem *list_head, size_t *tok_count, + BaseFloat *adaptive_beam, Elem **best_elem) { + BaseFloat best_weight = 1.0e+10; // positive == high cost == bad. + size_t count = 0; + if (opts_.max_active == std::numeric_limits::max() && + opts_.min_active == 0) { + for (Elem *e = list_head; e != NULL; e = e->tail, count++) { + BaseFloat w = static_cast(e->val->weight_.Value()); + if (w < best_weight) { + best_weight = w; + if (best_elem) *best_elem = e; + } + } + if (tok_count != NULL) *tok_count = count; + if (adaptive_beam != NULL) *adaptive_beam = opts_.beam; + return best_weight + opts_.beam; + } else { + tmp_array_.clear(); + for (Elem *e = list_head; e != NULL; e = e->tail, count++) { + BaseFloat w = e->val->weight_.Value(); + tmp_array_.push_back(w); + if (w < best_weight) { + best_weight = w; + if (best_elem) *best_elem = e; + } + } + if (tok_count != NULL) *tok_count = count; + + BaseFloat beam_cutoff = best_weight + opts_.beam, + min_active_cutoff = std::numeric_limits::infinity(), + max_active_cutoff = std::numeric_limits::infinity(); + + if (tmp_array_.size() > static_cast(opts_.max_active)) { + std::nth_element(tmp_array_.begin(), + tmp_array_.begin() + opts_.max_active, + tmp_array_.end()); + max_active_cutoff = tmp_array_[opts_.max_active]; + } + if (tmp_array_.size() > static_cast(opts_.min_active)) { + if (opts_.min_active == 0) min_active_cutoff = best_weight; + else { + std::nth_element(tmp_array_.begin(), + tmp_array_.begin() + opts_.min_active, + tmp_array_.size() > static_cast(opts_.max_active) ? + tmp_array_.begin() + opts_.max_active : + tmp_array_.end()); + min_active_cutoff = tmp_array_[opts_.min_active]; + } + } + + if (max_active_cutoff < beam_cutoff) { // max_active is tighter than beam. + if (adaptive_beam) + *adaptive_beam = max_active_cutoff - best_weight + opts_.beam_delta; + return max_active_cutoff; + } else if (min_active_cutoff > beam_cutoff) { // min_active is looser than beam. + if (adaptive_beam) + *adaptive_beam = min_active_cutoff - best_weight + opts_.beam_delta; + return min_active_cutoff; + } else { + *adaptive_beam = opts_.beam; + return beam_cutoff; + } + } + } + + void PossiblyResizeHash(size_t num_toks) { + size_t new_sz = static_cast(static_cast(num_toks) + * opts_.hash_ratio); + if (new_sz > toks_.Size()) { + toks_.SetSize(new_sz); + } + } + + inline StateId PropagateLm(StateId lm_state, + Arc *arc) { // returns new LM state. + if (arc->olabel == 0) { + return lm_state; // no change in LM state if no word crossed. + } else { // Propagate in the LM-diff FST. + Arc lm_arc; + bool ans = lm_diff_fst_->GetArc(lm_state, arc->olabel, &lm_arc); + if (!ans) { // this case is unexpected for statistical LMs. + if (!warned_noarc_) { + warned_noarc_ = true; + KALDI_WARN << "No arc available in LM (unlikely to be correct " + "if a statistical language model); will not warn again"; + } + arc->weight = Weight::Zero(); + return lm_state; // doesn't really matter what we return here; will + // be pruned. + } else { + arc->weight = Times(arc->weight, lm_arc.weight); + arc->olabel = lm_arc.olabel; // probably will be the same. + return lm_arc.nextstate; // return the new LM state. + } + } + } + + // ProcessEmitting returns the likelihood cutoff used. + BaseFloat ProcessEmitting(DecodableInterface *decodable, int frame) { + Elem *last_toks = toks_.Clear(); + size_t tok_cnt; + BaseFloat adaptive_beam; + Elem *best_elem = NULL; + BaseFloat weight_cutoff = GetCutoff(last_toks, &tok_cnt, + &adaptive_beam, &best_elem); + PossiblyResizeHash(tok_cnt); // This makes sure the hash is always big enough. + + // This is the cutoff we use after adding in the log-likes (i.e. + // for the next frame). This is a bound on the cutoff we will use + // on the next frame. + BaseFloat next_weight_cutoff = 1.0e+10; + + // First process the best token to get a hopefully + // reasonably tight bound on the next cutoff. + if (best_elem) { + PairId state_pair = best_elem->key; + StateId state = PairToState(state_pair), + lm_state = PairToLmState(state_pair); + Token *tok = best_elem->val; + for (fst::ArcIterator > aiter(fst_, state); + !aiter.Done(); + aiter.Next()) { + Arc arc = aiter.Value(); + if (arc.ilabel != 0) { // we'd propagate.. + PropagateLm(lm_state, &arc); // may affect "arc.weight". + // We don't need the return value (the new LM state). + BaseFloat ac_cost = - decodable->LogLikelihood(frame, arc.ilabel), + new_weight = arc.weight.Value() + tok->weight_.Value() + ac_cost; + if (new_weight + adaptive_beam < next_weight_cutoff) + next_weight_cutoff = new_weight + adaptive_beam; + } + } + } + + // the tokens are now owned here, in last_toks, and the hash is empty. + // 'owned' is a complex thing here; the point is we need to call toks_.Delete(e) + // on each elem 'e' to let toks_ know we're done with them. + for (Elem *e = last_toks, *e_tail; e != NULL; e = e_tail) { // loop this way + // because we delete "e" as we go. + PairId state_pair = e->key; + StateId state = PairToState(state_pair), + lm_state = PairToLmState(state_pair); + Token *tok = e->val; + if (tok->weight_.Value() < weight_cutoff) { // not pruned. + KALDI_ASSERT(state == tok->arc_.nextstate); + for (fst::ArcIterator > aiter(fst_, state); + !aiter.Done(); + aiter.Next()) { + Arc arc = aiter.Value(); + if (arc.ilabel != 0) { // propagate. + StateId next_lm_state = PropagateLm(lm_state, &arc); + Weight ac_weight(-decodable->LogLikelihood(frame, arc.ilabel)); + BaseFloat new_weight = arc.weight.Value() + tok->weight_.Value() + + ac_weight.Value(); + if (new_weight < next_weight_cutoff) { // not pruned.. + PairId next_pair = ConstructPair(arc.nextstate, next_lm_state); + Token *new_tok = new Token(arc, ac_weight, tok); + Elem *e_found = toks_.Insert(next_pair, new_tok); + if (new_weight + adaptive_beam < next_weight_cutoff) + next_weight_cutoff = new_weight + adaptive_beam; + if (e_found->val != new_tok) { + if (*(e_found->val) < *new_tok) { + Token::TokenDelete(e_found->val); + e_found->val = new_tok; + } else { + Token::TokenDelete(new_tok); + } + } + } + } + } + } + e_tail = e->tail; + Token::TokenDelete(e->val); + toks_.Delete(e); + } + return next_weight_cutoff; + } + + // TODO: first time we go through this, could avoid using the queue. + void ProcessNonemitting(BaseFloat cutoff) { + // Processes nonemitting arcs for one frame. + KALDI_ASSERT(queue_.empty()); + for (const Elem *e = toks_.GetList(); e != NULL; e = e->tail) + queue_.push_back(e); + while (!queue_.empty()) { + const Elem *e = queue_.back(); + queue_.pop_back(); + PairId state_pair = e->key; + Token *tok = e->val; // would segfault if state not + // in toks_ but this can't happen. + if (tok->weight_.Value() > cutoff) { // Don't bother processing successors. + continue; + } + KALDI_ASSERT(tok != NULL); + StateId state = PairToState(state_pair), + lm_state = PairToLmState(state_pair); + for (fst::ArcIterator > aiter(fst_, state); + !aiter.Done(); + aiter.Next()) { + const Arc &arc_ref = aiter.Value(); + if (arc_ref.ilabel == 0) { // propagate nonemitting only... + Arc arc(arc_ref); + StateId next_lm_state = PropagateLm(lm_state, &arc); + PairId next_pair = ConstructPair(arc.nextstate, next_lm_state); + Token *new_tok = new Token(arc, tok); + if (new_tok->weight_.Value() > cutoff) { // prune + Token::TokenDelete(new_tok); + } else { + Elem *e_found = toks_.Insert(next_pair, new_tok); + if (e_found->val == new_tok) { + queue_.push_back(e_found); + } else { + if ( *(e_found->val) < *new_tok ) { + Token::TokenDelete(e_found->val); + e_found->val = new_tok; + queue_.push_back(e_found); + } else { + Token::TokenDelete(new_tok); + } + } + } + } + } + } + } + + // HashList defined in ../util/hash-list.h. It actually allows us to maintain + // more than one list (e.g. for current and previous frames), but only one of + // them at a time can be indexed by PairId. + HashList toks_; + const fst::Fst &fst_; + fst::DeterministicOnDemandFst *lm_diff_fst_; + BiglmFasterDecoderOptions opts_; + bool warned_noarc_; + std::vector queue_; // temp variable used in ProcessNonemitting, + std::vector tmp_array_; // used in GetCutoff. + // make it class member to avoid internal new/delete. + + // It might seem unclear why we call ClearToks(toks_.Clear()). + // There are two separate cleanup tasks we need to do at when we start a new file. + // one is to delete the Token objects in the list; the other is to delete + // the Elem objects. toks_.Clear() just clears them from the hash and gives ownership + // to the caller, who then has to call toks_.Delete(e) for each one. It was designed + // this way for convenience in propagating tokens from one frame to the next. + void ClearToks(Elem *list) { + for (Elem *e = list, *e_tail; e != NULL; e = e_tail) { + Token::TokenDelete(e->val); + e_tail = e->tail; + toks_.Delete(e); + } + } + KALDI_DISALLOW_COPY_AND_ASSIGN(BiglmFasterDecoder); +}; + + +} // end namespace kaldi. + + +#endif diff --git a/runtime/onnxruntime/third_party/kaldi/decoder/decodable-mapped.h b/runtime/onnxruntime/third_party/kaldi/decoder/decodable-mapped.h new file mode 100644 index 000000000..daebfb50d --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/decoder/decodable-mapped.h @@ -0,0 +1,69 @@ +// decoder/decodable-mapped.h + +// Copyright 2009-2011 Saarland University; Microsoft Corporation; +// Lukas Burget + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#ifndef KALDI_DECODER_DECODABLE_MAPPED_H_ +#define KALDI_DECODER_DECODABLE_MAPPED_H_ + +#include + +#include "base/kaldi-common.h" +#include "itf/decodable-itf.h" + +namespace kaldi { + +// The DecodableMapped object is initialized by a normal decodable object, +// and a vector that maps indices. The "pdf index" into this decodable object +// is the index into the vector, and the value it finds there is used +// to index into the base decodable object. + +class DecodableMapped: public DecodableInterface { + public: + DecodableMapped(const std::vector &index_map, DecodableInterface *d): + index_map_(index_map), decodable_(d) { } + + // Note, frames are numbered from zero. But state_index is numbered + // from one (this routine is called by FSTs). + virtual BaseFloat LogLikelihood(int32 frame, int32 state_index) { + KALDI_ASSERT(static_cast(state_index) < index_map_.size()); + return decodable_->LogLikelihood(frame, index_map_[state_index]); + } + + // note: indices are assumed to be numbered from one, so + // NumIndices() will be the same as the largest index. + virtual int32 NumIndices() const { return static_cast(index_map_.size()) - 1; } + + virtual bool IsLastFrame(int32 frame) const { + // We require all the decodables have the same #frames. We don't check this though. + return decodable_->IsLastFrame(frame); + } + + private: + std::vector index_map_; + DecodableInterface *decodable_; + + KALDI_DISALLOW_COPY_AND_ASSIGN(DecodableMapped); +}; + + + +} // namespace kaldi + +#endif // KALDI_DECODER_DECODABLE_MAPPED_H_ + diff --git a/runtime/onnxruntime/third_party/kaldi/decoder/decodable-matrix.cc b/runtime/onnxruntime/third_party/kaldi/decoder/decodable-matrix.cc new file mode 100644 index 000000000..5cbc0a595 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/decoder/decodable-matrix.cc @@ -0,0 +1,112 @@ +// decoder/decodable-matrix.cc + +// Copyright 2018 Johns Hopkins University (author: Daniel Povey) + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#include "decoder/decodable-matrix.h" + +namespace kaldi { + +DecodableMatrixMapped::DecodableMatrixMapped( + const TransitionInformation &tm, + const MatrixBase &likes, + int32 frame_offset): + trans_model_(tm), + tid_to_pdf_(trans_model_.TransitionIdToPdfArray()), + likes_(&likes), likes_to_delete_(NULL), + frame_offset_(frame_offset) { + stride_ = likes.Stride(); + raw_data_ = likes.Data() - (stride_ * frame_offset); + + if (likes.NumCols() != tm.NumPdfs()) + KALDI_ERR << "Mismatch, matrix has " + << likes.NumCols() << " cols but transition-model has " + << tm.NumPdfs() << " pdf-ids."; +} + +DecodableMatrixMapped::DecodableMatrixMapped( + const TransitionInformation &tm, const Matrix *likes, + int32 frame_offset): + trans_model_(tm), + tid_to_pdf_(trans_model_.TransitionIdToPdfArray()), + likes_(likes), likes_to_delete_(likes), + frame_offset_(frame_offset) { + stride_ = likes->Stride(); + raw_data_ = likes->Data() - (stride_ * frame_offset_); + if (likes->NumCols() != tm.NumPdfs()) + KALDI_ERR << "Mismatch, matrix has " + << likes->NumCols() << " cols but transition-model has " + << tm.NumPdfs() << " pdf-ids."; +} + + +BaseFloat DecodableMatrixMapped::LogLikelihood(int32 frame, int32 tid) { + KALDI_PARANOID_ASSERT(tid >= 1 && tid < tid_to_pdf_.size()); + int32 pdf_id = tid_to_pdf_[tid]; +#ifdef KALDI_PARANOID + return (*likes_)(frame - frame_offset_, pdf_id); +#else + return raw_data_[frame * stride_ + pdf_id]; +#endif +} + +int32 DecodableMatrixMapped::NumFramesReady() const { + return frame_offset_ + likes_->NumRows(); +} + +bool DecodableMatrixMapped::IsLastFrame(int32 frame) const { + KALDI_ASSERT(frame < NumFramesReady()); + return (frame == NumFramesReady() - 1); +} + +// Indices are one-based! This is for compatibility with OpenFst. +int32 DecodableMatrixMapped::NumIndices() const { + return trans_model_.NumTransitionIds(); +} + +DecodableMatrixMapped::~DecodableMatrixMapped() { + delete likes_to_delete_; +} + + +void DecodableMatrixMappedOffset::AcceptLoglikes( + Matrix *loglikes, int32 frames_to_discard) { + if (loglikes->NumRows() == 0) return; + KALDI_ASSERT(loglikes->NumCols() == trans_model_.NumPdfs()); + KALDI_ASSERT(frames_to_discard <= loglikes_.NumRows() && + frames_to_discard >= 0); + if (frames_to_discard == loglikes_.NumRows()) { + loglikes_.Swap(loglikes); + loglikes->Resize(0, 0); + } else { + int32 old_rows_kept = loglikes_.NumRows() - frames_to_discard, + new_num_rows = old_rows_kept + loglikes->NumRows(); + Matrix new_loglikes(new_num_rows, loglikes->NumCols()); + new_loglikes.RowRange(0, old_rows_kept).CopyFromMat( + loglikes_.RowRange(frames_to_discard, old_rows_kept)); + new_loglikes.RowRange(old_rows_kept, loglikes->NumRows()).CopyFromMat( + *loglikes); + loglikes_.Swap(&new_loglikes); + } + frame_offset_ += frames_to_discard; + stride_ = loglikes_.Stride(); + raw_data_ = loglikes_.Data() - (frame_offset_ * stride_); +} + + + +} // end namespace kaldi. diff --git a/runtime/onnxruntime/third_party/kaldi/decoder/decodable-matrix.h b/runtime/onnxruntime/third_party/kaldi/decoder/decodable-matrix.h new file mode 100644 index 000000000..94e141d87 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/decoder/decodable-matrix.h @@ -0,0 +1,253 @@ +// decoder/decodable-matrix.h + +// Copyright 2009-2011 Microsoft Corporation +// 2013 Johns Hopkins University (author: Daniel Povey) + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#ifndef KALDI_DECODER_DECODABLE_MATRIX_H_ +#define KALDI_DECODER_DECODABLE_MATRIX_H_ + +#include + +#include "base/kaldi-common.h" +#include "itf/decodable-itf.h" +#include "itf/transition-information.h" +#include "matrix/kaldi-matrix.h" + +namespace kaldi { + + +class DecodableMatrixScaledMapped: public DecodableInterface { + public: + // This constructor creates an object that will not delete "likes" when done. + DecodableMatrixScaledMapped(const TransitionInformation &tm, + const Matrix &likes, + BaseFloat scale): trans_model_(tm), likes_(&likes), + tid_to_pdf_(trans_model_.TransitionIdToPdfArray()), + scale_(scale), delete_likes_(false) { + if (likes.NumCols() != tm.NumPdfs()) + KALDI_ERR << "DecodableMatrixScaledMapped: mismatch, matrix has " + << likes.NumCols() << " cols but transition-model has " + << tm.NumPdfs() << " pdf-ids."; + } + + // This constructor creates an object that will delete "likes" + // when done. + DecodableMatrixScaledMapped(const TransitionInformation &tm, + BaseFloat scale, + const Matrix *likes): + trans_model_(tm), likes_(likes), + tid_to_pdf_(trans_model_.TransitionIdToPdfArray()), + scale_(scale), delete_likes_(true) { + if (likes->NumCols() != tm.NumPdfs()) + KALDI_ERR << "DecodableMatrixScaledMapped: mismatch, matrix has " + << likes->NumCols() << " cols but transition-model has " + << tm.NumPdfs() << " pdf-ids."; + } + + virtual int32 NumFramesReady() const { return likes_->NumRows(); } + + virtual bool IsLastFrame(int32 frame) const { + KALDI_ASSERT(frame < NumFramesReady()); + return (frame == NumFramesReady() - 1); + } + + // Note, frames are numbered from zero. + virtual BaseFloat LogLikelihood(int32 frame, int32 tid) { + KALDI_PARANOID_ASSERT(tid >= 1 && tid < tid_to_pdf_.size()); + return scale_ * (*likes_)(frame, tid_to_pdf_[tid]); + } + + // Indices are one-based! This is for compatibility with OpenFst. + virtual int32 NumIndices() const { return trans_model_.NumTransitionIds(); } + + virtual ~DecodableMatrixScaledMapped() { + if (delete_likes_) delete likes_; + } + private: + const TransitionInformation &trans_model_; // for tid to pdf mapping + const Matrix *likes_; + const std::vector &tid_to_pdf_; + BaseFloat scale_; + bool delete_likes_; + KALDI_DISALLOW_COPY_AND_ASSIGN(DecodableMatrixScaledMapped); +}; + +/** + This is like DecodableMatrixScaledMapped, but it doesn't support an acoustic + scale, and it does support a frame offset, whereby you can state that the + first row of 'likes' is actually the n'th row of the matrix of available + log-likelihoods. It's useful if the neural net output comes in chunks for + different frame ranges. + + Note: DecodableMatrixMappedOffset solves the same problem in a slightly + different way, where you use the same decodable object. This one, unlike + DecodableMatrixMappedOffset, is compatible with when the loglikes are in a + SubMatrix. + */ +class DecodableMatrixMapped: public DecodableInterface { + public: + // This constructor creates an object that will not delete "likes" when done. + // the frame_offset is the frame the row 0 of 'likes' corresponds to, would be + // greater than one if this is not the first chunk of likelihoods. + DecodableMatrixMapped(const TransitionInformation &tm, + const MatrixBase &likes, + int32 frame_offset = 0); + + // This constructor creates an object that will delete "likes" + // when done. + DecodableMatrixMapped(const TransitionInformation &tm, + const Matrix *likes, + int32 frame_offset = 0); + + virtual int32 NumFramesReady() const; + + virtual bool IsLastFrame(int32 frame) const; + + virtual BaseFloat LogLikelihood(int32 frame, int32 tid); + + // Note: these indices are 1-based. + virtual int32 NumIndices() const; + + virtual ~DecodableMatrixMapped(); + + private: + const TransitionInformation &trans_model_; // for tid to pdf mapping + const std::vector& tid_to_pdf_; + const MatrixBase *likes_; + const Matrix *likes_to_delete_; + int32 frame_offset_; + + // raw_data_ and stride_ are a kind of fast look-aside for 'likes_', to be + // used when KALDI_PARANOID is false. + const BaseFloat *raw_data_; + int32 stride_; + + KALDI_DISALLOW_COPY_AND_ASSIGN(DecodableMatrixMapped); +}; + + +/** + This decodable class returns log-likes stored in a matrix; it supports + repeatedly writing to the matrix and setting a time-offset representing the + frame-index of the first row of the matrix. It's intended for use in + multi-threaded decoding; mutex and semaphores are not included. External + code will call SetLoglikes() each time more log-likelihods are available. + If you try to access a log-likelihood that's no longer available because + the frame index is less than the current offset, it is of course an error. + + See also DecodableMatrixMapped, which supports the same type of thing but + with a different interface where you are expected to re-construct the + object each time you want to decode. +*/ +class DecodableMatrixMappedOffset: public DecodableInterface { + public: + DecodableMatrixMappedOffset(const TransitionInformation &tm): + trans_model_(tm), tid_to_pdf_(trans_model_.TransitionIdToPdfArray()), + frame_offset_(0), input_is_finished_(false) { } + + // this is not part of the generic Decodable interface. + int32 FirstAvailableFrame() const { return frame_offset_; } + + // Logically, this function appends 'loglikes' (interpreted as newly available + // frames) to the log-likelihoods stored in the class. + // + // This function is destructive of the input "loglikes" because it may + // under some circumstances do a shallow copy using Swap(). This function + // appends loglikes to any existing likelihoods you've previously supplied. + void AcceptLoglikes(Matrix *loglikes, + int32 frames_to_discard); + + void InputIsFinished() { input_is_finished_ = true; } + + virtual int32 NumFramesReady() const { + return loglikes_.NumRows() + frame_offset_; + } + + virtual bool IsLastFrame(int32 frame) const { + KALDI_ASSERT(frame < NumFramesReady()); + return (frame == NumFramesReady() - 1 && input_is_finished_); + } + + virtual BaseFloat LogLikelihood(int32 frame, int32 tid) { + KALDI_PARANOID_ASSERT(tid >= 1 && tid < tid_to_pdf_.size()); + int32 pdf_id = tid_to_pdf_[tid]; +#ifdef KALDI_PARANOID + return loglikes_(frame - frame_offset_, pdf_id); +#else + // This does no checking, so will be faster. + return raw_data_[frame * stride_ + pdf_id]; +#endif + } + + virtual int32 NumIndices() const { return trans_model_.NumTransitionIds(); } + + // nothing special to do in destructor. + virtual ~DecodableMatrixMappedOffset() { } + private: + const TransitionInformation &trans_model_; // for tid to pdf mapping + const std::vector& tid_to_pdf_; + Matrix loglikes_; + int32 frame_offset_; + bool input_is_finished_; + + // 'raw_data_' and 'stride_' are intended as a fast look-aside which is an + // alternative to accessing data_. raw_data_ is a faked version of + // data_->Data() as if it started from frame zero rather than frame_offset_. + // This simplifies the code of LogLikelihood(), in cases where KALDI_PARANOID + // is not defined. + BaseFloat *raw_data_; + int32 stride_; + + KALDI_DISALLOW_COPY_AND_ASSIGN(DecodableMatrixMappedOffset); +}; + + +class DecodableMatrixScaled: public DecodableInterface { + public: + DecodableMatrixScaled(const Matrix &likes, + BaseFloat scale): + likes_(likes), scale_(scale) { } + + virtual int32 NumFramesReady() const { return likes_.NumRows(); } + + virtual bool IsLastFrame(int32 frame) const { + KALDI_ASSERT(frame < NumFramesReady()); + return (frame == NumFramesReady() - 1); + } + + // Note, frames are numbered from zero. + virtual BaseFloat LogLikelihood(int32 frame, int32 index) { + if (index > likes_.NumCols() || index <= 0 || + frame < 0 || frame >= likes_.NumRows()) + KALDI_ERR << "Invalid (frame, index - 1) = (" + << frame << ", " << index - 1 << ") for matrix of size " + << likes_.NumRows() << " x " << likes_.NumCols(); + return scale_ * likes_(frame, index - 1); + } + + // Indices are one-based! This is for compatibility with OpenFst. + virtual int32 NumIndices() const { return likes_.NumCols(); } + + private: + const Matrix &likes_; + BaseFloat scale_; + KALDI_DISALLOW_COPY_AND_ASSIGN(DecodableMatrixScaled); +}; +} // namespace kaldi + +#endif // KALDI_DECODER_DECODABLE_MATRIX_H_ diff --git a/runtime/onnxruntime/third_party/kaldi/decoder/decodable-sum.h b/runtime/onnxruntime/third_party/kaldi/decoder/decodable-sum.h new file mode 100644 index 000000000..e3ff60a6f --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/decoder/decodable-sum.h @@ -0,0 +1,109 @@ +// decoder/decodable-sum.h + +// Copyright 2009-2011 Saarland University; Microsoft Corporation; +// Lukas Burget, Pawel Swietojanski + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#ifndef KALDI_DECODER_DECODABLE_SUM_H_ +#define KALDI_DECODER_DECODABLE_SUM_H_ + +#include +#include + +#include "base/kaldi-common.h" +#include "itf/decodable-itf.h" + +namespace kaldi { + +// The DecodableSum object is a very simple object that just sums +// scores over a number of Decodable objects. They must all have +// the same dimensions. + +class DecodableSum: public DecodableInterface { + public: + // Does not take ownership of pointers! They are just + // pointers because they are non-const. + DecodableSum(DecodableInterface *d1, BaseFloat w1, + DecodableInterface *d2, BaseFloat w2) { + decodables_.push_back(std::make_pair(d1, w1)); + decodables_.push_back(std::make_pair(d2, w2)); + CheckSizes(); + } + + // Does not take ownership of pointers! + DecodableSum( + const std::vector > &decodables) : + decodables_(decodables) { CheckSizes(); } + + void CheckSizes() const { + KALDI_ASSERT(decodables_.size() >= 1 + && decodables_[0].first != NULL); + for (size_t i = 1; i < decodables_.size(); i++) + KALDI_ASSERT(decodables_[i].first != NULL && + decodables_[i].first->NumIndices() == + decodables_[0].first->NumIndices()); + } + + // Note, frames are numbered from zero. But state_index is numbered + // from one (this routine is called by FSTs). + virtual BaseFloat LogLikelihood(int32 frame, int32 state_index) { + BaseFloat sum = 0.0; + // int32 i=1; + for (std::vector >::iterator iter = decodables_.begin(); + iter != decodables_.end(); + ++iter) { + sum += iter->first->LogLikelihood(frame, state_index) * iter->second; + } + return sum; + } + + virtual int32 NumIndices() const { return decodables_[0].first->NumIndices(); } + + virtual bool IsLastFrame(int32 frame) const { + // We require all the decodables have the same #frames. We don't check this though. + return decodables_[0].first->IsLastFrame(frame); + } + + private: + std::vector > decodables_; + KALDI_DISALLOW_COPY_AND_ASSIGN(DecodableSum); +}; + +class DecodableSumScaled : public DecodableSum { + public: + DecodableSumScaled(DecodableInterface *d1, BaseFloat w1, + DecodableInterface *d2, BaseFloat w2, + BaseFloat scale) + : DecodableSum(d1, w1, d2, w2), scale_(scale) {} + + DecodableSumScaled(const std::vector > &decodables, + BaseFloat scale) + : DecodableSum(decodables), scale_(scale) {} + + virtual BaseFloat LogLikelihood(int32 frame, int32 state_index) { + return scale_ * DecodableSum::LogLikelihood(frame, state_index); + } + + private: + BaseFloat scale_; + KALDI_DISALLOW_COPY_AND_ASSIGN(DecodableSumScaled); +}; + +} // namespace kaldi + +#endif // KALDI_DECODER_DECODABLE_SUM_H_ + diff --git a/runtime/onnxruntime/third_party/kaldi/decoder/decoder-wrappers.cc b/runtime/onnxruntime/third_party/kaldi/decoder/decoder-wrappers.cc new file mode 100644 index 000000000..626a69387 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/decoder/decoder-wrappers.cc @@ -0,0 +1,665 @@ +// decoder/decoder-wrappers.cc + +// Copyright 2014 Johns Hopkins University (author: Daniel Povey) + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#include "decoder/decoder-wrappers.h" +#include "decoder/faster-decoder.h" +#include "decoder/lattice-faster-decoder.h" +#include "decoder/grammar-fst.h" +#include "lat/lattice-functions.h" + +namespace kaldi { + + + + + +DecodeUtteranceLatticeFasterClass::DecodeUtteranceLatticeFasterClass( + LatticeFasterDecoder *decoder, + DecodableInterface *decodable, + const TransitionInformation &trans_model, + const fst::SymbolTable *word_syms, + const std::string &utt, + BaseFloat acoustic_scale, + bool determinize, + bool allow_partial, + Int32VectorWriter *alignments_writer, + Int32VectorWriter *words_writer, + CompactLatticeWriter *compact_lattice_writer, + LatticeWriter *lattice_writer, + double *like_sum, // on success, adds likelihood to this. + int64 *frame_sum, // on success, adds #frames to this. + int32 *num_done, // on success (including partial decode), increments this. + int32 *num_err, // on failure, increments this. + int32 *num_partial): // If partial decode (final-state not reached), increments this. + decoder_(decoder), decodable_(decodable), trans_model_(&trans_model), + word_syms_(word_syms), utt_(utt), acoustic_scale_(acoustic_scale), + determinize_(determinize), allow_partial_(allow_partial), + alignments_writer_(alignments_writer), + words_writer_(words_writer), + compact_lattice_writer_(compact_lattice_writer), + lattice_writer_(lattice_writer), + like_sum_(like_sum), frame_sum_(frame_sum), + num_done_(num_done), num_err_(num_err), + num_partial_(num_partial), + computed_(false), success_(false), partial_(false), + clat_(NULL), lat_(NULL) { } + + +void DecodeUtteranceLatticeFasterClass::operator () () { + // Decoding and lattice determinization happens here. + computed_ = true; // Just means this function was called-- a check on the + // calling code. + success_ = true; + using fst::VectorFst; + if (!decoder_->Decode(decodable_)) { + KALDI_WARN << "Failed to decode utterance with id " << utt_; + success_ = false; + } + if (!decoder_->ReachedFinal()) { + if (allow_partial_) { + KALDI_WARN << "Outputting partial output for utterance " << utt_ + << " since no final-state reached\n"; + partial_ = true; + } else { + KALDI_WARN << "Not producing output for utterance " << utt_ + << " since no final-state reached and " + << "--allow-partial=false.\n"; + success_ = false; + } + } + if (!success_) return; + + // Get lattice, and do determinization if requested. + lat_ = new Lattice; + decoder_->GetRawLattice(lat_); + if (lat_->NumStates() == 0) + KALDI_ERR << "Unexpected problem getting lattice for utterance " << utt_; + fst::Connect(lat_); + if (determinize_) { + clat_ = new CompactLattice; + if (!DeterminizeLatticePhonePrunedWrapper( + *trans_model_, + lat_, + decoder_->GetOptions().lattice_beam, + clat_, + decoder_->GetOptions().det_opts)) + KALDI_WARN << "Determinization finished earlier than the beam for " + << "utterance " << utt_; + delete lat_; + lat_ = NULL; + // We'll write the lattice without acoustic scaling. + if (acoustic_scale_ != 0.0) + fst::ScaleLattice(fst::AcousticLatticeScale(1.0 / acoustic_scale_), clat_); + } else { + // We'll write the lattice without acoustic scaling. + if (acoustic_scale_ != 0.0) + fst::ScaleLattice(fst::AcousticLatticeScale(1.0 / acoustic_scale_), lat_); + } +} + +DecodeUtteranceLatticeFasterClass::~DecodeUtteranceLatticeFasterClass() { + if (!computed_) + KALDI_ERR << "Destructor called without operator (), error in calling code."; + + if (!success_) { + if (num_err_ != NULL) (*num_err_)++; + } else { // successful decode. + // Getting the one-best output is lightweight enough that we can do it in + // the destructor (easier than adding more variables to the class, and + // will rarely slow down the main thread.) + double likelihood; + LatticeWeight weight; + int32 num_frames; + { // First do some stuff with word-level traceback... + // This is basically for diagnostics. + fst::VectorFst decoded; + decoder_->GetBestPath(&decoded); + if (decoded.NumStates() == 0) { + // Shouldn't really reach this point as already checked success. + KALDI_ERR << "Failed to get traceback for utterance " << utt_; + } + std::vector alignment; + std::vector words; + GetLinearSymbolSequence(decoded, &alignment, &words, &weight); + num_frames = alignment.size(); + if (words_writer_->IsOpen()) + words_writer_->Write(utt_, words); + if (alignments_writer_->IsOpen()) + alignments_writer_->Write(utt_, alignment); + if (word_syms_ != NULL) { + std::cerr << utt_ << ' '; + for (size_t i = 0; i < words.size(); i++) { + std::string s = word_syms_->Find(words[i]); + if (s == "") + KALDI_ERR << "Word-id " << words[i] << " not in symbol table."; + std::cerr << s << ' '; + } + std::cerr << '\n'; + } + likelihood = -(weight.Value1() + weight.Value2()); + } + + // Ouptut the lattices. + if (determinize_) { // CompactLattice output. + KALDI_ASSERT(compact_lattice_writer_ != NULL && clat_ != NULL); + if (clat_->NumStates() == 0) { + KALDI_WARN << "Empty lattice for utterance " << utt_; + } else { + compact_lattice_writer_->Write(utt_, *clat_); + } + delete clat_; + clat_ = NULL; + } else { + KALDI_ASSERT(lattice_writer_ != NULL && lat_ != NULL); + if (lat_->NumStates() == 0) { + KALDI_WARN << "Empty lattice for utterance " << utt_; + } else { + lattice_writer_->Write(utt_, *lat_); + } + delete lat_; + lat_ = NULL; + } + + // Print out logging information. + KALDI_LOG << "Log-like per frame for utterance " << utt_ << " is " + << (likelihood / num_frames) << " over " + << num_frames << " frames."; + KALDI_VLOG(2) << "Cost for utterance " << utt_ << " is " + << weight.Value1() << " + " << weight.Value2(); + + // Now output the various diagnostic variables. + if (like_sum_ != NULL) *like_sum_ += likelihood; + if (frame_sum_ != NULL) *frame_sum_ += num_frames; + if (num_done_ != NULL) (*num_done_)++; + if (partial_ && num_partial_ != NULL) (*num_partial_)++; + } + // We were given ownership of these two objects that were passed in in + // the initializer. + delete decoder_; + delete decodable_; +} + +template +bool DecodeUtteranceLatticeIncremental( + LatticeIncrementalDecoderTpl &decoder, // not const but is really an input. + DecodableInterface &decodable, // not const but is really an input. + const TransitionInformation &trans_model, + const fst::SymbolTable *word_syms, + std::string utt, + double acoustic_scale, + bool determinize, + bool allow_partial, + Int32VectorWriter *alignment_writer, + Int32VectorWriter *words_writer, + CompactLatticeWriter *compact_lattice_writer, + LatticeWriter *lattice_writer, + double *like_ptr) { // puts utterance's like in like_ptr on success. + using fst::VectorFst; + if (!decoder.Decode(&decodable)) { + KALDI_WARN << "Failed to decode utterance with id " << utt; + return false; + } + if (!decoder.ReachedFinal()) { + if (allow_partial) { + KALDI_WARN << "Outputting partial output for utterance " << utt + << " since no final-state reached\n"; + } else { + KALDI_WARN << "Not producing output for utterance " << utt + << " since no final-state reached and " + << "--allow-partial=false.\n"; + return false; + } + } + + // Get lattice + CompactLattice clat = decoder.GetLattice(decoder.NumFramesDecoded(), true); + if (clat.NumStates() == 0) + KALDI_ERR << "Unexpected problem getting lattice for utterance " << utt; + + double likelihood; + LatticeWeight weight; + int32 num_frames; + { // First do some stuff with word-level traceback... + CompactLattice decoded_clat; + CompactLatticeShortestPath(clat, &decoded_clat); + Lattice decoded; + fst::ConvertLattice(decoded_clat, &decoded); + + if (decoded.Start() == fst::kNoStateId) + // Shouldn't really reach this point as already checked success. + KALDI_ERR << "Failed to get traceback for utterance " << utt; + + std::vector alignment; + std::vector words; + GetLinearSymbolSequence(decoded, &alignment, &words, &weight); + num_frames = alignment.size(); + KALDI_ASSERT(num_frames == decoder.NumFramesDecoded()); + if (words_writer->IsOpen()) + words_writer->Write(utt, words); + if (alignment_writer->IsOpen()) + alignment_writer->Write(utt, alignment); + if (word_syms != NULL) { + std::cerr << utt << ' '; + for (size_t i = 0; i < words.size(); i++) { + std::string s = word_syms->Find(words[i]); + if (s == "") + KALDI_ERR << "Word-id " << words[i] << " not in symbol table."; + std::cerr << s << ' '; + } + std::cerr << '\n'; + } + likelihood = -(weight.Value1() + weight.Value2()); + } + + // We'll write the lattice without acoustic scaling. + if (acoustic_scale != 0.0) + fst::ScaleLattice(fst::AcousticLatticeScale(1.0 / acoustic_scale), &clat); + Connect(&clat); + compact_lattice_writer->Write(utt, clat); + KALDI_LOG << "Log-like per frame for utterance " << utt << " is " + << (likelihood / num_frames) << " over " + << num_frames << " frames."; + KALDI_VLOG(2) << "Cost for utterance " << utt << " is " + << weight.Value1() << " + " << weight.Value2(); + *like_ptr = likelihood; + return true; +} + + +// Takes care of output. Returns true on success. +template +bool DecodeUtteranceLatticeFaster( + LatticeFasterDecoderTpl &decoder, // not const but is really an input. + DecodableInterface &decodable, // not const but is really an input. + const TransitionInformation &trans_model, + const fst::SymbolTable *word_syms, + std::string utt, + double acoustic_scale, + bool determinize, + bool allow_partial, + Int32VectorWriter *alignment_writer, + Int32VectorWriter *words_writer, + CompactLatticeWriter *compact_lattice_writer, + LatticeWriter *lattice_writer, + double *like_ptr) { // puts utterance's like in like_ptr on success. + using fst::VectorFst; + + if (!decoder.Decode(&decodable)) { + KALDI_WARN << "Failed to decode utterance with id " << utt; + return false; + } + if (!decoder.ReachedFinal()) { + if (allow_partial) { + KALDI_WARN << "Outputting partial output for utterance " << utt + << " since no final-state reached\n"; + } else { + KALDI_WARN << "Not producing output for utterance " << utt + << " since no final-state reached and " + << "--allow-partial=false.\n"; + return false; + } + } + + double likelihood; + LatticeWeight weight; + int32 num_frames; + { // First do some stuff with word-level traceback... + VectorFst decoded; + if (!decoder.GetBestPath(&decoded)) + // Shouldn't really reach this point as already checked success. + KALDI_ERR << "Failed to get traceback for utterance " << utt; + + std::vector alignment; + std::vector words; + GetLinearSymbolSequence(decoded, &alignment, &words, &weight); + num_frames = alignment.size(); + if (words_writer->IsOpen()) + words_writer->Write(utt, words); + if (alignment_writer->IsOpen()) + alignment_writer->Write(utt, alignment); + if (word_syms != NULL) { + std::cerr << utt << ' '; + for (size_t i = 0; i < words.size(); i++) { + std::string s = word_syms->Find(words[i]); + if (s == "") + KALDI_ERR << "Word-id " << words[i] << " not in symbol table."; + std::cerr << s << ' '; + } + std::cerr << '\n'; + } + likelihood = -(weight.Value1() + weight.Value2()); + } + + // Get lattice, and do determinization if requested. + Lattice lat; + decoder.GetRawLattice(&lat); + if (lat.NumStates() == 0) + KALDI_ERR << "Unexpected problem getting lattice for utterance " << utt; + fst::Connect(&lat); + if (determinize) { + CompactLattice clat; + if (!DeterminizeLatticePhonePrunedWrapper( + trans_model, + &lat, + decoder.GetOptions().lattice_beam, + &clat, + decoder.GetOptions().det_opts)) + KALDI_WARN << "Determinization finished earlier than the beam for " + << "utterance " << utt; + // We'll write the lattice without acoustic scaling. + if (acoustic_scale != 0.0) + fst::ScaleLattice(fst::AcousticLatticeScale(1.0 / acoustic_scale), &clat); + compact_lattice_writer->Write(utt, clat); + } else { + // We'll write the lattice without acoustic scaling. + if (acoustic_scale != 0.0) + fst::ScaleLattice(fst::AcousticLatticeScale(1.0 / acoustic_scale), &lat); + lattice_writer->Write(utt, lat); + } + KALDI_LOG << "Log-like per frame for utterance " << utt << " is " + << (likelihood / num_frames) << " over " + << num_frames << " frames."; + KALDI_VLOG(2) << "Cost for utterance " << utt << " is " + << weight.Value1() << " + " << weight.Value2(); + *like_ptr = likelihood; + return true; +} + +// Instantiate the template above for the two required FST types. +template bool DecodeUtteranceLatticeIncremental( + LatticeIncrementalDecoderTpl > &decoder, + DecodableInterface &decodable, + const TransitionInformation &trans_model, + const fst::SymbolTable *word_syms, + std::string utt, + double acoustic_scale, + bool determinize, + bool allow_partial, + Int32VectorWriter *alignment_writer, + Int32VectorWriter *words_writer, + CompactLatticeWriter *compact_lattice_writer, + LatticeWriter *lattice_writer, + double *like_ptr); + +template bool DecodeUtteranceLatticeIncremental( + LatticeIncrementalDecoderTpl &decoder, + DecodableInterface &decodable, + const TransitionInformation &trans_model, + const fst::SymbolTable *word_syms, + std::string utt, + double acoustic_scale, + bool determinize, + bool allow_partial, + Int32VectorWriter *alignment_writer, + Int32VectorWriter *words_writer, + CompactLatticeWriter *compact_lattice_writer, + LatticeWriter *lattice_writer, + double *like_ptr); + +template bool DecodeUtteranceLatticeFaster( + LatticeFasterDecoderTpl > &decoder, + DecodableInterface &decodable, + const TransitionInformation &trans_model, + const fst::SymbolTable *word_syms, + std::string utt, + double acoustic_scale, + bool determinize, + bool allow_partial, + Int32VectorWriter *alignment_writer, + Int32VectorWriter *words_writer, + CompactLatticeWriter *compact_lattice_writer, + LatticeWriter *lattice_writer, + double *like_ptr); + +template bool DecodeUtteranceLatticeFaster( + LatticeFasterDecoderTpl &decoder, + DecodableInterface &decodable, + const TransitionInformation &trans_model, + const fst::SymbolTable *word_syms, + std::string utt, + double acoustic_scale, + bool determinize, + bool allow_partial, + Int32VectorWriter *alignment_writer, + Int32VectorWriter *words_writer, + CompactLatticeWriter *compact_lattice_writer, + LatticeWriter *lattice_writer, + double *like_ptr); + + +// Takes care of output. Returns true on success. +bool DecodeUtteranceLatticeSimple( + LatticeSimpleDecoder &decoder, // not const but is really an input. + DecodableInterface &decodable, // not const but is really an input. + const TransitionInformation &trans_model, + const fst::SymbolTable *word_syms, + std::string utt, + double acoustic_scale, + bool determinize, + bool allow_partial, + Int32VectorWriter *alignment_writer, + Int32VectorWriter *words_writer, + CompactLatticeWriter *compact_lattice_writer, + LatticeWriter *lattice_writer, + double *like_ptr) { // puts utterance's like in like_ptr on success. + using fst::VectorFst; + + if (!decoder.Decode(&decodable)) { + KALDI_WARN << "Failed to decode utterance with id " << utt; + return false; + } + if (!decoder.ReachedFinal()) { + if (allow_partial) { + KALDI_WARN << "Outputting partial output for utterance " << utt + << " since no final-state reached\n"; + } else { + KALDI_WARN << "Not producing output for utterance " << utt + << " since no final-state reached and " + << "--allow-partial=false.\n"; + return false; + } + } + + double likelihood; + LatticeWeight weight = LatticeWeight::Zero(); + int32 num_frames; + { // First do some stuff with word-level traceback... + VectorFst decoded; + if (!decoder.GetBestPath(&decoded)) + // Shouldn't really reach this point as already checked success. + KALDI_ERR << "Failed to get traceback for utterance " << utt; + + std::vector alignment; + std::vector words; + GetLinearSymbolSequence(decoded, &alignment, &words, &weight); + num_frames = alignment.size(); + if (words_writer->IsOpen()) + words_writer->Write(utt, words); + if (alignment_writer->IsOpen()) + alignment_writer->Write(utt, alignment); + if (word_syms != NULL) { + std::cerr << utt << ' '; + for (size_t i = 0; i < words.size(); i++) { + std::string s = word_syms->Find(words[i]); + if (s == "") + KALDI_ERR << "Word-id " << words[i] << " not in symbol table."; + std::cerr << s << ' '; + } + std::cerr << '\n'; + } + likelihood = -(weight.Value1() + weight.Value2()); + } + + // Get lattice, and do determinization if requested. + Lattice lat; + if (!decoder.GetRawLattice(&lat)) + KALDI_ERR << "Unexpected problem getting lattice for utterance " << utt; + fst::Connect(&lat); + if (determinize) { + CompactLattice clat; + if (!DeterminizeLatticePhonePrunedWrapper( + trans_model, + &lat, + decoder.GetOptions().lattice_beam, + &clat, + decoder.GetOptions().det_opts)) + KALDI_WARN << "Determinization finished earlier than the beam for " + << "utterance " << utt; + // We'll write the lattice without acoustic scaling. + if (acoustic_scale != 0.0) + fst::ScaleLattice(fst::AcousticLatticeScale(1.0 / acoustic_scale), &clat); + compact_lattice_writer->Write(utt, clat); + } else { + // We'll write the lattice without acoustic scaling. + if (acoustic_scale != 0.0) + fst::ScaleLattice(fst::AcousticLatticeScale(1.0 / acoustic_scale), &lat); + lattice_writer->Write(utt, lat); + } + KALDI_LOG << "Log-like per frame for utterance " << utt << " is " + << (likelihood / num_frames) << " over " + << num_frames << " frames."; + KALDI_VLOG(2) << "Cost for utterance " << utt << " is " + << weight.Value1() << " + " << weight.Value2(); + *like_ptr = likelihood; + return true; +} + + +// see comment in header. +void ModifyGraphForCarefulAlignment( + fst::VectorFst *fst) { + typedef fst::StdArc Arc; + typedef Arc::StateId StateId; + typedef Arc::Label Label; + typedef Arc::Weight Weight; + StateId num_states = fst->NumStates(); + if (num_states == 0) { + KALDI_WARN << "Empty FST input."; + return; + } + Weight zero = Weight::Zero(); + // fst_rhs will be the right hand side of the Concat operation. + fst::VectorFst fst_rhs(*fst); + // first remove the final-probs from fst_rhs. + for (StateId state = 0; state < num_states; state++) + fst_rhs.SetFinal(state, zero); + StateId pre_initial = fst_rhs.AddState(); + Arc to_initial(0, 0, Weight::One(), fst_rhs.Start()); + fst_rhs.AddArc(pre_initial, to_initial); + fst_rhs.SetStart(pre_initial); + // make the pre_initial state final with probability one; + // this is equivalent to keeping the final-probs of the first + // FST when we do concat (otherwise they would get deleted). + fst_rhs.SetFinal(pre_initial, Weight::One()); + fst::VectorFst fst_concat; + fst::Concat(fst, fst_rhs); +} + + +void AlignUtteranceWrapper( + const AlignConfig &config, + const std::string &utt, + BaseFloat acoustic_scale, // affects scores written to scores_writer, if + // present + fst::VectorFst *fst, // non-const in case config.careful == + // true. + DecodableInterface *decodable, // not const but is really an input. + Int32VectorWriter *alignment_writer, + BaseFloatWriter *scores_writer, + int32 *num_done, + int32 *num_error, + int32 *num_retried, + double *tot_like, + int64 *frame_count, + BaseFloatVectorWriter *per_frame_acwt_writer) { + + if ((config.retry_beam != 0 && config.retry_beam <= config.beam) || + config.beam <= 0.0) { + KALDI_ERR << "Beams do not make sense: beam " << config.beam + << ", retry-beam " << config.retry_beam; + } + + if (fst->Start() == fst::kNoStateId) { + KALDI_WARN << "Empty decoding graph for " << utt; + if (num_error != NULL) (*num_error)++; + return; + } + + if (config.careful) + ModifyGraphForCarefulAlignment(fst); + + FasterDecoderOptions decode_opts; + decode_opts.beam = config.beam; + + FasterDecoder decoder(*fst, decode_opts); + decoder.Decode(decodable); + + bool ans = decoder.ReachedFinal(); // consider only final states. + + if (!ans && config.retry_beam != 0.0) { + if (num_retried != NULL) (*num_retried)++; + KALDI_WARN << "Retrying utterance " << utt << " with beam " + << config.retry_beam; + decode_opts.beam = config.retry_beam; + decoder.SetOptions(decode_opts); + decoder.Decode(decodable); + ans = decoder.ReachedFinal(); + } + + if (!ans) { // Still did not reach final state. + KALDI_WARN << "Did not successfully decode file " << utt << ", len = " + << decodable->NumFramesReady(); + if (num_error != NULL) (*num_error)++; + return; + } + + fst::VectorFst decoded; // linear FST. + decoder.GetBestPath(&decoded); + if (decoded.NumStates() == 0) { + KALDI_WARN << "Error getting best path from decoder (likely a bug)"; + if (num_error != NULL) (*num_error)++; + return; + } + + std::vector alignment; + std::vector words; + LatticeWeight weight; + + GetLinearSymbolSequence(decoded, &alignment, &words, &weight); + BaseFloat like = -(weight.Value1()+weight.Value2()) / acoustic_scale; + + if (num_done != NULL) (*num_done)++; + if (tot_like != NULL) (*tot_like) += like; + if (frame_count != NULL) (*frame_count) += decodable->NumFramesReady(); + + if (alignment_writer != NULL && alignment_writer->IsOpen()) + alignment_writer->Write(utt, alignment); + + if (scores_writer != NULL && scores_writer->IsOpen()) + scores_writer->Write(utt, -(weight.Value1()+weight.Value2())); + + Vector per_frame_loglikes; + if (per_frame_acwt_writer != NULL && per_frame_acwt_writer->IsOpen()) { + GetPerFrameAcousticCosts(decoded, &per_frame_loglikes); + per_frame_loglikes.Scale(-1 / acoustic_scale); + per_frame_acwt_writer->Write(utt, per_frame_loglikes); + } +} + +} // end namespace kaldi. diff --git a/runtime/onnxruntime/third_party/kaldi/decoder/decoder-wrappers.h b/runtime/onnxruntime/third_party/kaldi/decoder/decoder-wrappers.h new file mode 100644 index 000000000..edea6cf9a --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/decoder/decoder-wrappers.h @@ -0,0 +1,221 @@ +// decoder/decoder-wrappers.h + +// Copyright 2014 Johns Hopkins University (author: Daniel Povey) + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#ifndef KALDI_DECODER_DECODER_WRAPPERS_H_ +#define KALDI_DECODER_DECODER_WRAPPERS_H_ + +#include "itf/options-itf.h" +#include "decoder/lattice-faster-decoder.h" +#include "decoder/lattice-incremental-decoder.h" +#include "decoder/lattice-simple-decoder.h" + +// This header contains declarations from various convenience functions that are called +// from binary-level programs such as gmm-decode-faster.cc, gmm-align-compiled.cc, and +// so on. + +namespace kaldi { + + +struct AlignConfig { + BaseFloat beam; + BaseFloat retry_beam; + bool careful; + + AlignConfig(): beam(200.0), retry_beam(0.0), careful(false) { } + + void Register(OptionsItf *opts) { + opts->Register("beam", &beam, "Decoding beam used in alignment"); + opts->Register("retry-beam", &retry_beam, + "Decoding beam for second try at alignment"); + opts->Register("careful", &careful, + "If true, do 'careful' alignment, which is better at detecting " + "alignment failure (involves loop to start of decoding graph)."); + } +}; + + +/// AlignUtteranceWapper is a wrapper for alignment code used in training, that +/// is called from many different binaries, e.g. gmm-align, gmm-align-compiled, +/// sgmm-align, etc. The writers for alignments and words will only be written +/// to if they are open. The num_done, num_error, num_retried, tot_like and +/// frame_count pointers will (if non-NULL) be incremented or added to, not set, +/// by this function. +void AlignUtteranceWrapper( + const AlignConfig &config, + const std::string &utt, + BaseFloat acoustic_scale, // affects scores written to scores_writer, if + // present + fst::VectorFst *fst, // non-const in case config.careful == + // true, we add loop. + DecodableInterface *decodable, // not const but is really an input. + Int32VectorWriter *alignment_writer, + BaseFloatWriter *scores_writer, + int32 *num_done, + int32 *num_error, + int32 *num_retried, + double *tot_like, + int64 *frame_count, + BaseFloatVectorWriter *per_frame_acwt_writer = NULL); + + + +/// This function modifies the decoding graph for what we call "careful +/// alignment". The problem we are trying to solve is that if the decoding eats +/// up the words in the graph too fast, it can get stuck at the end, and produce +/// what looks like a valid alignment even though there was really a failure. +/// So what we want to do is to introduce, after the final-states of the graph, +/// a "blind alley" with no final-probs reachable, where the decoding can go to +/// get lost. Our basic idea is to append the decoding-graph to itself using +/// the fst Concat operation; but in order that there should be final-probs at the end of +/// the first but not the second FST, we modify the right-hand argument to the +/// Concat operation so that it has none of the original final-probs, and add +/// a "pre-initial" state that is final. +void ModifyGraphForCarefulAlignment( + fst::VectorFst *fst); + +/// TODO +template +bool DecodeUtteranceLatticeIncremental( + LatticeIncrementalDecoderTpl &decoder, // not const but is really an input. + DecodableInterface &decodable, // not const but is really an input. + const TransitionInformation &trans_model, + const fst::SymbolTable *word_syms, + std::string utt, + double acoustic_scale, + bool determinize, + bool allow_partial, + Int32VectorWriter *alignments_writer, + Int32VectorWriter *words_writer, + CompactLatticeWriter *compact_lattice_writer, + LatticeWriter *lattice_writer, + double *like_ptr); // puts utterance's likelihood in like_ptr on success. + + +/// This function DecodeUtteranceLatticeFaster is used in several decoders, and +/// we have moved it here. Note: this is really "binary-level" code as it +/// involves table readers and writers; we've just put it here as there is no +/// other obvious place to put it. If determinize == false, it writes to +/// lattice_writer, else to compact_lattice_writer. The writers for +/// alignments and words will only be written to if they are open. +/// +/// Caution: this will only link correctly if FST is either fst::Fst, +/// or fst::GrammarFst, as the template function is defined in the .cc file and +/// only instantiated for those two types. +template +bool DecodeUtteranceLatticeFaster( + LatticeFasterDecoderTpl &decoder, // not const but is really an input. + DecodableInterface &decodable, // not const but is really an input. + const TransitionInformation &trans_model, + const fst::SymbolTable *word_syms, + std::string utt, + double acoustic_scale, + bool determinize, + bool allow_partial, + Int32VectorWriter *alignments_writer, + Int32VectorWriter *words_writer, + CompactLatticeWriter *compact_lattice_writer, + LatticeWriter *lattice_writer, + double *like_ptr); // puts utterance's likelihood in like_ptr on success. + + +/// This class basically does the same job as the function +/// DecodeUtteranceLatticeFaster, but in a way that allows us +/// to build a multi-threaded command line program more easily. +/// The main computation takes place in operator (), and the output +/// happens in the destructor. +class DecodeUtteranceLatticeFasterClass { + public: + // Initializer sets various variables. + // NOTE: we "take ownership" of "decoder" and "decodable". These + // are deleted by the destructor. On error, "num_err" is incremented. + DecodeUtteranceLatticeFasterClass( + LatticeFasterDecoder *decoder, + DecodableInterface *decodable, + const TransitionInformation &trans_model, + const fst::SymbolTable *word_syms, + const std::string &utt, + BaseFloat acoustic_scale, + bool determinize, + bool allow_partial, + Int32VectorWriter *alignments_writer, + Int32VectorWriter *words_writer, + CompactLatticeWriter *compact_lattice_writer, + LatticeWriter *lattice_writer, + double *like_sum, // on success, adds likelihood to this. + int64 *frame_sum, // on success, adds #frames to this. + int32 *num_done, // on success (including partial decode), increments this. + int32 *num_err, // on failure, increments this. + int32 *num_partial); // If partial decode (final-state not reached), increments this. + void operator () (); // The decoding happens here. + ~DecodeUtteranceLatticeFasterClass(); // Output happens here. + private: + // The following variables correspond to inputs: + LatticeFasterDecoder *decoder_; + DecodableInterface *decodable_; + const TransitionInformation *trans_model_; + const fst::SymbolTable *word_syms_; + std::string utt_; + BaseFloat acoustic_scale_; + bool determinize_; + bool allow_partial_; + Int32VectorWriter *alignments_writer_; + Int32VectorWriter *words_writer_; + CompactLatticeWriter *compact_lattice_writer_; + LatticeWriter *lattice_writer_; + double *like_sum_; + int64 *frame_sum_; + int32 *num_done_; + int32 *num_err_; + int32 *num_partial_; + + // The following variables are stored by the computation. + bool computed_; // operator () was called. + bool success_; // decoding succeeded (possibly partial) + bool partial_; // decoding was partial. + CompactLattice *clat_; // Stored output, if determinize_ == true. + Lattice *lat_; // Stored output, if determinize_ == false. +}; + +// This function DecodeUtteranceLatticeSimple is used in several decoders, and +// we have moved it here. Note: this is really "binary-level" code as it +// involves table readers and writers; we've just put it here as there is no +// other obvious place to put it. If determinize == false, it writes to +// lattice_writer, else to compact_lattice_writer. The writers for +// alignments and words will only be written to if they are open. +bool DecodeUtteranceLatticeSimple( + LatticeSimpleDecoder &decoder, // not const but is really an input. + DecodableInterface &decodable, // not const but is really an input. + const TransitionInformation &trans_model, + const fst::SymbolTable *word_syms, + std::string utt, + double acoustic_scale, + bool determinize, + bool allow_partial, + Int32VectorWriter *alignments_writer, + Int32VectorWriter *words_writer, + CompactLatticeWriter *compact_lattice_writer, + LatticeWriter *lattice_writer, + double *like_ptr); // puts utterance's likelihood in like_ptr on success. + + + +} // end namespace kaldi. + + +#endif diff --git a/runtime/onnxruntime/third_party/kaldi/decoder/faster-decoder.cc b/runtime/onnxruntime/third_party/kaldi/decoder/faster-decoder.cc new file mode 100644 index 000000000..9e8775875 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/decoder/faster-decoder.cc @@ -0,0 +1,351 @@ +// decoder/faster-decoder.cc + +// Copyright 2009-2011 Microsoft Corporation +// 2012-2013 Johns Hopkins University (author: Daniel Povey) + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#include "decoder/faster-decoder.h" + +namespace kaldi { + + +FasterDecoder::FasterDecoder(const fst::Fst &fst, + const FasterDecoderOptions &opts): + fst_(fst), config_(opts), num_frames_decoded_(-1) { + KALDI_ASSERT(config_.hash_ratio >= 1.0); // less doesn't make much sense. + KALDI_ASSERT(config_.max_active > 1); + KALDI_ASSERT(config_.min_active >= 0 && config_.min_active < config_.max_active); + toks_.SetSize(1000); // just so on the first frame we do something reasonable. +} + + +void FasterDecoder::InitDecoding() { + // clean up from last time: + ClearToks(toks_.Clear()); + StateId start_state = fst_.Start(); + KALDI_ASSERT(start_state != fst::kNoStateId); + Arc dummy_arc(0, 0, Weight::One(), start_state); + toks_.Insert(start_state, new Token(dummy_arc, NULL)); + ProcessNonemitting(std::numeric_limits::max()); + num_frames_decoded_ = 0; +} + + +void FasterDecoder::Decode(DecodableInterface *decodable) { + InitDecoding(); + AdvanceDecoding(decodable); +} + +void FasterDecoder::AdvanceDecoding(DecodableInterface *decodable, + int32 max_num_frames) { + KALDI_ASSERT(num_frames_decoded_ >= 0 && + "You must call InitDecoding() before AdvanceDecoding()"); + int32 num_frames_ready = decodable->NumFramesReady(); + // num_frames_ready must be >= num_frames_decoded, or else + // the number of frames ready must have decreased (which doesn't + // make sense) or the decodable object changed between calls + // (which isn't allowed). + KALDI_ASSERT(num_frames_ready >= num_frames_decoded_); + int32 target_frames_decoded = num_frames_ready; + if (max_num_frames >= 0) + target_frames_decoded = std::min(target_frames_decoded, + num_frames_decoded_ + max_num_frames); + while (num_frames_decoded_ < target_frames_decoded) { + // note: ProcessEmitting() increments num_frames_decoded_ + double weight_cutoff = ProcessEmitting(decodable); + ProcessNonemitting(weight_cutoff); + } +} + + +bool FasterDecoder::ReachedFinal() const { + for (const Elem *e = toks_.GetList(); e != NULL; e = e->tail) { + if (e->val->cost_ != std::numeric_limits::infinity() && + fst_.Final(e->key) != Weight::Zero()) + return true; + } + return false; +} + +bool FasterDecoder::GetBestPath(fst::MutableFst *fst_out, + bool use_final_probs) { + // GetBestPath gets the decoding output. If "use_final_probs" is true + // AND we reached a final state, it limits itself to final states; + // otherwise it gets the most likely token not taking into + // account final-probs. fst_out will be empty (Start() == kNoStateId) if + // nothing was available. It returns true if it got output (thus, fst_out + // will be nonempty). + fst_out->DeleteStates(); + Token *best_tok = NULL; + bool is_final = ReachedFinal(); + if (!is_final) { + for (const Elem *e = toks_.GetList(); e != NULL; e = e->tail) + if (best_tok == NULL || *best_tok < *(e->val) ) + best_tok = e->val; + } else { + double infinity = std::numeric_limits::infinity(), + best_cost = infinity; + for (const Elem *e = toks_.GetList(); e != NULL; e = e->tail) { + double this_cost = e->val->cost_ + fst_.Final(e->key).Value(); + if (this_cost < best_cost && this_cost != infinity) { + best_cost = this_cost; + best_tok = e->val; + } + } + } + if (best_tok == NULL) return false; // No output. + + std::vector arcs_reverse; // arcs in reverse order. + + for (Token *tok = best_tok; tok != NULL; tok = tok->prev_) { + BaseFloat tot_cost = tok->cost_ - + (tok->prev_ ? tok->prev_->cost_ : 0.0), + graph_cost = tok->arc_.weight.Value(), + ac_cost = tot_cost - graph_cost; + LatticeArc l_arc(tok->arc_.ilabel, + tok->arc_.olabel, + LatticeWeight(graph_cost, ac_cost), + tok->arc_.nextstate); + arcs_reverse.push_back(l_arc); + } + KALDI_ASSERT(arcs_reverse.back().nextstate == fst_.Start()); + arcs_reverse.pop_back(); // that was a "fake" token... gives no info. + + StateId cur_state = fst_out->AddState(); + fst_out->SetStart(cur_state); + for (ssize_t i = static_cast(arcs_reverse.size())-1; i >= 0; i--) { + LatticeArc arc = arcs_reverse[i]; + arc.nextstate = fst_out->AddState(); + fst_out->AddArc(cur_state, arc); + cur_state = arc.nextstate; + } + if (is_final && use_final_probs) { + Weight final_weight = fst_.Final(best_tok->arc_.nextstate); + fst_out->SetFinal(cur_state, LatticeWeight(final_weight.Value(), 0.0)); + } else { + fst_out->SetFinal(cur_state, LatticeWeight::One()); + } + RemoveEpsLocal(fst_out); + return true; +} + + +// Gets the weight cutoff. Also counts the active tokens. +double FasterDecoder::GetCutoff(Elem *list_head, size_t *tok_count, + BaseFloat *adaptive_beam, Elem **best_elem) { + double best_cost = std::numeric_limits::infinity(); + size_t count = 0; + if (config_.max_active == std::numeric_limits::max() && + config_.min_active == 0) { + for (Elem *e = list_head; e != NULL; e = e->tail, count++) { + double w = e->val->cost_; + if (w < best_cost) { + best_cost = w; + if (best_elem) *best_elem = e; + } + } + if (tok_count != NULL) *tok_count = count; + if (adaptive_beam != NULL) *adaptive_beam = config_.beam; + return best_cost + config_.beam; + } else { + tmp_array_.clear(); + for (Elem *e = list_head; e != NULL; e = e->tail, count++) { + double w = e->val->cost_; + tmp_array_.push_back(w); + if (w < best_cost) { + best_cost = w; + if (best_elem) *best_elem = e; + } + } + if (tok_count != NULL) *tok_count = count; + double beam_cutoff = best_cost + config_.beam, + min_active_cutoff = std::numeric_limits::infinity(), + max_active_cutoff = std::numeric_limits::infinity(); + + if (tmp_array_.size() > static_cast(config_.max_active)) { + std::nth_element(tmp_array_.begin(), + tmp_array_.begin() + config_.max_active, + tmp_array_.end()); + max_active_cutoff = tmp_array_[config_.max_active]; + } + if (max_active_cutoff < beam_cutoff) { // max_active is tighter than beam. + if (adaptive_beam) + *adaptive_beam = max_active_cutoff - best_cost + config_.beam_delta; + return max_active_cutoff; + } + if (tmp_array_.size() > static_cast(config_.min_active)) { + if (config_.min_active == 0) min_active_cutoff = best_cost; + else { + std::nth_element(tmp_array_.begin(), + tmp_array_.begin() + config_.min_active, + tmp_array_.size() > static_cast(config_.max_active) ? + tmp_array_.begin() + config_.max_active : + tmp_array_.end()); + min_active_cutoff = tmp_array_[config_.min_active]; + } + } + if (min_active_cutoff > beam_cutoff) { // min_active is looser than beam. + if (adaptive_beam) + *adaptive_beam = min_active_cutoff - best_cost + config_.beam_delta; + return min_active_cutoff; + } else { + *adaptive_beam = config_.beam; + return beam_cutoff; + } + } +} + +void FasterDecoder::PossiblyResizeHash(size_t num_toks) { + size_t new_sz = static_cast(static_cast(num_toks) + * config_.hash_ratio); + if (new_sz > toks_.Size()) { + toks_.SetSize(new_sz); + } +} + +// ProcessEmitting returns the likelihood cutoff used. +double FasterDecoder::ProcessEmitting(DecodableInterface *decodable) { + int32 frame = num_frames_decoded_; + Elem *last_toks = toks_.Clear(); + size_t tok_cnt; + BaseFloat adaptive_beam; + Elem *best_elem = NULL; + double weight_cutoff = GetCutoff(last_toks, &tok_cnt, + &adaptive_beam, &best_elem); + KALDI_VLOG(3) << tok_cnt << " tokens active."; + PossiblyResizeHash(tok_cnt); // This makes sure the hash is always big enough. + + // This is the cutoff we use after adding in the log-likes (i.e. + // for the next frame). This is a bound on the cutoff we will use + // on the next frame. + double next_weight_cutoff = std::numeric_limits::infinity(); + + // First process the best token to get a hopefully + // reasonably tight bound on the next cutoff. + if (best_elem) { + StateId state = best_elem->key; + Token *tok = best_elem->val; + for (fst::ArcIterator > aiter(fst_, state); + !aiter.Done(); + aiter.Next()) { + const Arc &arc = aiter.Value(); + if (arc.ilabel != 0) { // we'd propagate.. + BaseFloat ac_cost = - decodable->LogLikelihood(frame, arc.ilabel); + double new_weight = arc.weight.Value() + tok->cost_ + ac_cost; + if (new_weight + adaptive_beam < next_weight_cutoff) + next_weight_cutoff = new_weight + adaptive_beam; + } + } + } + + // int32 n = 0, np = 0; + + // the tokens are now owned here, in last_toks, and the hash is empty. + // 'owned' is a complex thing here; the point is we need to call TokenDelete + // on each elem 'e' to let toks_ know we're done with them. + for (Elem *e = last_toks, *e_tail; e != NULL; e = e_tail) { // loop this way + // n++; + // because we delete "e" as we go. + StateId state = e->key; + Token *tok = e->val; + if (tok->cost_ < weight_cutoff) { // not pruned. + // np++; + KALDI_ASSERT(state == tok->arc_.nextstate); + for (fst::ArcIterator > aiter(fst_, state); + !aiter.Done(); + aiter.Next()) { + Arc arc = aiter.Value(); + if (arc.ilabel != 0) { // propagate.. + BaseFloat ac_cost = - decodable->LogLikelihood(frame, arc.ilabel); + double new_weight = arc.weight.Value() + tok->cost_ + ac_cost; + if (new_weight < next_weight_cutoff) { // not pruned.. + Token *new_tok = new Token(arc, ac_cost, tok); + Elem *e_found = toks_.Insert(arc.nextstate, new_tok); + if (new_weight + adaptive_beam < next_weight_cutoff) + next_weight_cutoff = new_weight + adaptive_beam; + if (e_found->val != new_tok) { + if (*(e_found->val) < *new_tok) { + Token::TokenDelete(e_found->val); + e_found->val = new_tok; + } else { + Token::TokenDelete(new_tok); + } + } + } + } + } + } + e_tail = e->tail; + Token::TokenDelete(e->val); + toks_.Delete(e); + } + num_frames_decoded_++; + return next_weight_cutoff; +} + +// TODO: first time we go through this, could avoid using the queue. +void FasterDecoder::ProcessNonemitting(double cutoff) { + // Processes nonemitting arcs for one frame. + KALDI_ASSERT(queue_.empty()); + for (const Elem *e = toks_.GetList(); e != NULL; e = e->tail) + queue_.push_back(e); + while (!queue_.empty()) { + const Elem* e = queue_.back(); + queue_.pop_back(); + StateId state = e->key; + Token *tok = e->val; // would segfault if state not + // in toks_ but this can't happen. + if (tok->cost_ > cutoff) { // Don't bother processing successors. + continue; + } + KALDI_ASSERT(tok != NULL && state == tok->arc_.nextstate); + for (fst::ArcIterator > aiter(fst_, state); + !aiter.Done(); + aiter.Next()) { + const Arc &arc = aiter.Value(); + if (arc.ilabel == 0) { // propagate nonemitting only... + Token *new_tok = new Token(arc, tok); + if (new_tok->cost_ > cutoff) { // prune + Token::TokenDelete(new_tok); + } else { + Elem *e_found = toks_.Insert(arc.nextstate, new_tok); + if (e_found->val == new_tok) { + queue_.push_back(e_found); + } else { + if (*(e_found->val) < *new_tok) { + Token::TokenDelete(e_found->val); + e_found->val = new_tok; + queue_.push_back(e_found); + } else { + Token::TokenDelete(new_tok); + } + } + } + } + } + } +} + +void FasterDecoder::ClearToks(Elem *list) { + for (Elem *e = list, *e_tail; e != NULL; e = e_tail) { + Token::TokenDelete(e->val); + e_tail = e->tail; + toks_.Delete(e); + } +} + +} // end namespace kaldi. diff --git a/runtime/onnxruntime/third_party/kaldi/decoder/faster-decoder.h b/runtime/onnxruntime/third_party/kaldi/decoder/faster-decoder.h new file mode 100644 index 000000000..88f78a615 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/decoder/faster-decoder.h @@ -0,0 +1,195 @@ +// decoder/faster-decoder.h + +// Copyright 2009-2011 Microsoft Corporation +// 2013 Johns Hopkins University (author: Daniel Povey) + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#ifndef KALDI_DECODER_FASTER_DECODER_H_ +#define KALDI_DECODER_FASTER_DECODER_H_ + +#include "util/stl-utils.h" +#include "itf/options-itf.h" +#include "util/hash-list.h" +#include "fst/fstlib.h" +#include "itf/decodable-itf.h" +#include "lat/kaldi-lattice.h" // for CompactLatticeArc + +namespace kaldi { + +struct FasterDecoderOptions { + BaseFloat beam; + int32 max_active; + int32 min_active; + BaseFloat beam_delta; + BaseFloat hash_ratio; + FasterDecoderOptions(): beam(16.0), + max_active(std::numeric_limits::max()), + min_active(20), // This decoder mostly used for + // alignment, use small default. + beam_delta(0.5), + hash_ratio(2.0) { } + void Register(OptionsItf *opts, bool full) { /// if "full", use obscure + /// options too. + /// Depends on program. + opts->Register("beam", &beam, "Decoding beam. Larger->slower, more accurate."); + opts->Register("max-active", &max_active, "Decoder max active states. Larger->slower; " + "more accurate"); + opts->Register("min-active", &min_active, + "Decoder min active states (don't prune if #active less than this)."); + if (full) { + opts->Register("beam-delta", &beam_delta, + "Increment used in decoder [obscure setting]"); + opts->Register("hash-ratio", &hash_ratio, + "Setting used in decoder to control hash behavior"); + } + } +}; + +class FasterDecoder { + public: + typedef fst::StdArc Arc; + typedef Arc::Label Label; + typedef Arc::StateId StateId; + typedef Arc::Weight Weight; + + FasterDecoder(const fst::Fst &fst, + const FasterDecoderOptions &config); + + void SetOptions(const FasterDecoderOptions &config) { config_ = config; } + + ~FasterDecoder() { ClearToks(toks_.Clear()); } + + void Decode(DecodableInterface *decodable); + + /// Returns true if a final state was active on the last frame. + bool ReachedFinal() const; + + /// GetBestPath gets the decoding traceback. If "use_final_probs" is true + /// AND we reached a final state, it limits itself to final states; + /// otherwise it gets the most likely token not taking into account + /// final-probs. Returns true if the output best path was not the empty + /// FST (will only return false in unusual circumstances where + /// no tokens survived). + bool GetBestPath(fst::MutableFst *fst_out, + bool use_final_probs = true); + + /// As a new alternative to Decode(), you can call InitDecoding + /// and then (possibly multiple times) AdvanceDecoding(). + void InitDecoding(); + + + /// This will decode until there are no more frames ready in the decodable + /// object, but if max_num_frames is >= 0 it will decode no more than + /// that many frames. + void AdvanceDecoding(DecodableInterface *decodable, + int32 max_num_frames = -1); + + /// Returns the number of frames already decoded. + int32 NumFramesDecoded() const { return num_frames_decoded_; } + + protected: + + class Token { + public: + Arc arc_; // contains only the graph part of the cost; + // we can work out the acoustic part from difference between + // "cost_" and prev->cost_. + Token *prev_; + int32 ref_count_; + // if you are looking for weight_ here, it was removed and now we just have + // cost_, which corresponds to ConvertToCost(weight_). + double cost_; + inline Token(const Arc &arc, BaseFloat ac_cost, Token *prev): + arc_(arc), prev_(prev), ref_count_(1) { + if (prev) { + prev->ref_count_++; + cost_ = prev->cost_ + arc.weight.Value() + ac_cost; + } else { + cost_ = arc.weight.Value() + ac_cost; + } + } + inline Token(const Arc &arc, Token *prev): + arc_(arc), prev_(prev), ref_count_(1) { + if (prev) { + prev->ref_count_++; + cost_ = prev->cost_ + arc.weight.Value(); + } else { + cost_ = arc.weight.Value(); + } + } + inline bool operator < (const Token &other) { + return cost_ > other.cost_; + } + + inline static void TokenDelete(Token *tok) { + while (--tok->ref_count_ == 0) { + Token *prev = tok->prev_; + delete tok; + if (prev == NULL) return; + else tok = prev; + } +#ifdef KALDI_PARANOID + KALDI_ASSERT(tok->ref_count_ > 0); +#endif + } + }; + typedef HashList::Elem Elem; + + + /// Gets the weight cutoff. Also counts the active tokens. + double GetCutoff(Elem *list_head, size_t *tok_count, + BaseFloat *adaptive_beam, Elem **best_elem); + + void PossiblyResizeHash(size_t num_toks); + + // ProcessEmitting returns the likelihood cutoff used. + // It decodes the frame num_frames_decoded_ of the decodable object + // and then increments num_frames_decoded_ + double ProcessEmitting(DecodableInterface *decodable); + + // TODO: first time we go through this, could avoid using the queue. + void ProcessNonemitting(double cutoff); + + // HashList defined in ../util/hash-list.h. It actually allows us to maintain + // more than one list (e.g. for current and previous frames), but only one of + // them at a time can be indexed by StateId. + HashList toks_; + const fst::Fst &fst_; + FasterDecoderOptions config_; + std::vector queue_; // temp variable used in ProcessNonemitting, + std::vector tmp_array_; // used in GetCutoff. + // make it class member to avoid internal new/delete. + + // Keep track of the number of frames decoded in the current file. + int32 num_frames_decoded_; + + // It might seem unclear why we call ClearToks(toks_.Clear()). + // There are two separate cleanup tasks we need to do at when we start a new file. + // one is to delete the Token objects in the list; the other is to delete + // the Elem objects. toks_.Clear() just clears them from the hash and gives ownership + // to the caller, who then has to call toks_.Delete(e) for each one. It was designed + // this way for convenience in propagating tokens from one frame to the next. + void ClearToks(Elem *list); + + KALDI_DISALLOW_COPY_AND_ASSIGN(FasterDecoder); +}; + + +} // end namespace kaldi. + + +#endif diff --git a/runtime/onnxruntime/third_party/kaldi/decoder/lattice-biglm-faster-decoder.h b/runtime/onnxruntime/third_party/kaldi/decoder/lattice-biglm-faster-decoder.h new file mode 100644 index 000000000..9f50739a0 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/decoder/lattice-biglm-faster-decoder.h @@ -0,0 +1,886 @@ +// decoder/lattice-biglm-faster-decoder.h + +// Copyright 2009-2011 Microsoft Corporation, Mirko Hannemann, +// Gilles Boulianne + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#ifndef KALDI_DECODER_LATTICE_BIGLM_FASTER_DECODER_H_ +#define KALDI_DECODER_LATTICE_BIGLM_FASTER_DECODER_H_ + + +#include "util/stl-utils.h" +#include "util/hash-list.h" +#include "fst/fstlib.h" +#include "itf/decodable-itf.h" +#include "fstext/fstext-lib.h" +#include "lat/kaldi-lattice.h" +#include "decoder/lattice-faster-decoder.h" // for options. + + +namespace kaldi { + +// The options are the same as for lattice-faster-decoder.h for now. +typedef LatticeFasterDecoderConfig LatticeBiglmFasterDecoderConfig; + +/** This is as LatticeFasterDecoder, but does online composition between + HCLG and the "difference language model", which is a deterministic + FST that represents the difference between the language model you want + and the language model you compiled HCLG with. The class + DeterministicOnDemandFst follows through the epsilons in G for you + (assuming G is a standard backoff language model) and makes it look + like a determinized FST. +*/ + +class LatticeBiglmFasterDecoder { + public: + typedef fst::StdArc Arc; + typedef Arc::Label Label; + typedef Arc::StateId StateId; + // A PairId will be constructed as: (StateId in fst) + (StateId in lm_diff_fst) << 32; + typedef uint64 PairId; + typedef Arc::Weight Weight; + // instantiate this class once for each thing you have to decode. + LatticeBiglmFasterDecoder( + const fst::Fst &fst, + const LatticeBiglmFasterDecoderConfig &config, + fst::DeterministicOnDemandFst *lm_diff_fst): + fst_(fst), lm_diff_fst_(lm_diff_fst), config_(config), + warned_noarc_(false), num_toks_(0) { + config.Check(); + KALDI_ASSERT(fst.Start() != fst::kNoStateId && + lm_diff_fst->Start() != fst::kNoStateId); + toks_.SetSize(1000); // just so on the first frame we do something reasonable. + } + void SetOptions(const LatticeBiglmFasterDecoderConfig &config) { config_ = config; } + LatticeBiglmFasterDecoderConfig GetOptions() { return config_; } + ~LatticeBiglmFasterDecoder() { + DeleteElems(toks_.Clear()); + ClearActiveTokens(); + } + + // Returns true if any kind of traceback is available (not necessarily from + // a final state). + bool Decode(DecodableInterface *decodable) { + // clean up from last time: + DeleteElems(toks_.Clear()); + ClearActiveTokens(); + warned_ = false; + final_active_ = false; + final_costs_.clear(); + num_toks_ = 0; + PairId start_pair = ConstructPair(fst_.Start(), lm_diff_fst_->Start()); + active_toks_.resize(1); + Token *start_tok = new Token(0.0, 0.0, NULL, NULL); + active_toks_[0].toks = start_tok; + toks_.Insert(start_pair, start_tok); + num_toks_++; + ProcessNonemitting(0); + + // We use 1-based indexing for frames in this decoder (if you view it in + // terms of features), but note that the decodable object uses zero-based + // numbering, which we have to correct for when we call it. + for (int32 frame = 1; !decodable->IsLastFrame(frame-2); frame++) { + active_toks_.resize(frame+1); // new column + + ProcessEmitting(decodable, frame); + + ProcessNonemitting(frame); + + if (decodable->IsLastFrame(frame-1)) + PruneActiveTokensFinal(frame); + else if (frame % config_.prune_interval == 0) + PruneActiveTokens(frame, config_.lattice_beam * 0.1); // use larger delta. + } + // Returns true if we have any kind of traceback available (not necessarily + // to the end state; query ReachedFinal() for that). + return !final_costs_.empty(); + } + + /// says whether a final-state was active on the last frame. If it was not, the + /// lattice (or traceback) will end with states that are not final-states. + bool ReachedFinal() const { return final_active_; } + + + // Outputs an FST corresponding to the single best path + // through the lattice. + bool GetBestPath(fst::MutableFst *ofst, + bool use_final_probs = true) const { + fst::VectorFst fst; + if (!GetRawLattice(&fst, use_final_probs)) return false; + // std::cout << "Raw lattice is:\n"; + // fst::FstPrinter fstprinter(fst, NULL, NULL, NULL, false, true); + // fstprinter.Print(&std::cout, "standard output"); + ShortestPath(fst, ofst); + return true; + } + + // Outputs an FST corresponding to the raw, state-level + // tracebacks. + bool GetRawLattice(fst::MutableFst *ofst, + bool use_final_probs = true) const { + typedef LatticeArc Arc; + typedef Arc::StateId StateId; + // A PairId will be constructed as: (StateId in fst) + (StateId in lm_diff_fst) << 32; + typedef uint64 PairId; + typedef Arc::Weight Weight; + typedef Arc::Label Label; + ofst->DeleteStates(); + // num-frames plus one (since frames are one-based, and we have + // an extra frame for the start-state). + int32 num_frames = active_toks_.size() - 1; + KALDI_ASSERT(num_frames > 0); + unordered_map tok_map(num_toks_/2 + 3); // bucket count + // First create all states. + for (int32 f = 0; f <= num_frames; f++) { + if (active_toks_[f].toks == NULL) { + KALDI_WARN << "GetRawLattice: no tokens active on frame " << f + << ": not producing lattice.\n"; + return false; + } + for (Token *tok = active_toks_[f].toks; tok != NULL; tok = tok->next) + tok_map[tok] = ofst->AddState(); + // The next statement sets the start state of the output FST. + // Because we always add new states to the head of the list + // active_toks_[f].toks, and the start state was the first one + // added, it will be the last one added to ofst. + if (f == 0 && ofst->NumStates() > 0) + ofst->SetStart(ofst->NumStates()-1); + } + KALDI_VLOG(3) << "init:" << num_toks_/2 + 3 << " buckets:" + << tok_map.bucket_count() << " load:" << tok_map.load_factor() + << " max:" << tok_map.max_load_factor(); + // Now create all arcs. + StateId cur_state = 0; // we rely on the fact that we numbered these + // consecutively (AddState() returns the numbers in order..) + for (int32 f = 0; f <= num_frames; f++) { + for (Token *tok = active_toks_[f].toks; tok != NULL; tok = tok->next, + cur_state++) { + for (ForwardLink *l = tok->links; + l != NULL; + l = l->next) { + unordered_map::const_iterator iter = + tok_map.find(l->next_tok); + StateId nextstate = iter->second; + KALDI_ASSERT(iter != tok_map.end()); + Arc arc(l->ilabel, l->olabel, + Weight(l->graph_cost, l->acoustic_cost), + nextstate); + ofst->AddArc(cur_state, arc); + } + if (f == num_frames) { + if (use_final_probs && !final_costs_.empty()) { + std::map::const_iterator iter = + final_costs_.find(tok); + if (iter != final_costs_.end()) + ofst->SetFinal(cur_state, LatticeWeight(iter->second, 0)); + } else { + ofst->SetFinal(cur_state, LatticeWeight::One()); + } + } + } + } + KALDI_ASSERT(cur_state == ofst->NumStates()); + return (cur_state != 0); + } + + // This function is now deprecated, since now we do determinization from + // outside the LatticeBiglmFasterDecoder class. + // Outputs an FST corresponding to the lattice-determinized + // lattice (one path per word sequence). + bool GetLattice(fst::MutableFst *ofst, + bool use_final_probs = true) const { + Lattice raw_fst; + if (!GetRawLattice(&raw_fst, use_final_probs)) return false; + Invert(&raw_fst); // make it so word labels are on the input. + if (!TopSort(&raw_fst)) // topological sort makes lattice-determinization more efficient + KALDI_WARN << "Topological sorting of state-level lattice failed " + "(probably your lexicon has empty words or your LM has epsilon cycles; this " + " is a bad idea.)"; + // (in phase where we get backward-costs). + fst::ILabelCompare ilabel_comp; + ArcSort(&raw_fst, ilabel_comp); // sort on ilabel; makes + // lattice-determinization more efficient. + + fst::DeterminizeLatticePrunedOptions lat_opts; + lat_opts.max_mem = config_.det_opts.max_mem; + + DeterminizeLatticePruned(raw_fst, config_.lattice_beam, ofst, lat_opts); + raw_fst.DeleteStates(); // Free memory-- raw_fst no longer needed. + Connect(ofst); // Remove unreachable states... there might be + // a small number of these, in some cases. + return true; + } + + private: + inline PairId ConstructPair(StateId fst_state, StateId lm_state) { + return static_cast(fst_state) + (static_cast(lm_state) << 32); + } + + static inline StateId PairToState(PairId state_pair) { + return static_cast(static_cast(state_pair)); + } + static inline StateId PairToLmState(PairId state_pair) { + return static_cast(static_cast(state_pair >> 32)); + } + + struct Token; + // ForwardLinks are the links from a token to a token on the next frame. + // or sometimes on the current frame (for input-epsilon links). + struct ForwardLink { + Token *next_tok; // the next token [or NULL if represents final-state] + Label ilabel; // ilabel on link. + Label olabel; // olabel on link. + BaseFloat graph_cost; // graph cost of traversing link (contains LM, etc.) + BaseFloat acoustic_cost; // acoustic cost (pre-scaled) of traversing link + ForwardLink *next; // next in singly-linked list of forward links from a + // token. + inline ForwardLink(Token *next_tok, Label ilabel, Label olabel, + BaseFloat graph_cost, BaseFloat acoustic_cost, + ForwardLink *next): + next_tok(next_tok), ilabel(ilabel), olabel(olabel), + graph_cost(graph_cost), acoustic_cost(acoustic_cost), + next(next) { } + }; + + // Token is what's resident in a particular state at a particular time. + // In this decoder a Token actually contains *forward* links. + // When first created, a Token just has the (total) cost. We add forward + // links to it when we process the next frame. + struct Token { + BaseFloat tot_cost; // would equal weight.Value()... cost up to this point. + BaseFloat extra_cost; // >= 0. After calling PruneForwardLinks, this equals + // the minimum difference between the cost of the best path, and the cost of + // this is on, and the cost of the absolute best path, under the assumption + // that any of the currently active states at the decoding front may + // eventually succeed (e.g. if you were to take the currently active states + // one by one and compute this difference, and then take the minimum). + + ForwardLink *links; // Head of singly linked list of ForwardLinks + + Token *next; // Next in list of tokens for this frame. + + inline Token(BaseFloat tot_cost, BaseFloat extra_cost, ForwardLink *links, + Token *next): tot_cost(tot_cost), extra_cost(extra_cost), + links(links), next(next) { } + inline void DeleteForwardLinks() { + ForwardLink *l = links, *m; + while (l != NULL) { + m = l->next; + delete l; + l = m; + } + links = NULL; + } + }; + + // head and tail of per-frame list of Tokens (list is in topological order), + // and something saying whether we ever pruned it using PruneForwardLinks. + struct TokenList { + Token *toks; + bool must_prune_forward_links; + bool must_prune_tokens; + TokenList(): toks(NULL), must_prune_forward_links(true), + must_prune_tokens(true) { } + }; + + typedef HashList::Elem Elem; + + void PossiblyResizeHash(size_t num_toks) { + size_t new_sz = static_cast(static_cast(num_toks) + * config_.hash_ratio); + if (new_sz > toks_.Size()) { + toks_.SetSize(new_sz); + } + } + + // FindOrAddToken either locates a token in hash of toks_, + // or if necessary inserts a new, empty token (i.e. with no forward links) + // for the current frame. [note: it's inserted if necessary into hash toks_ + // and also into the singly linked list of tokens active on this frame + // (whose head is at active_toks_[frame]). + inline Elem *FindOrAddToken(PairId state_pair, int32 frame, + BaseFloat tot_cost, bool emitting, bool *changed) { + // Returns the Token pointer. Sets "changed" (if non-NULL) to true + // if the token was newly created or the cost changed. + KALDI_ASSERT(frame < active_toks_.size()); + Token *&toks = active_toks_[frame].toks; + Elem *e_found = toks_.Insert(state_pair, NULL); + if (e_found->val == NULL) { // no such token presently. + const BaseFloat extra_cost = 0.0; + // tokens on the currently final frame have zero extra_cost + // as any of them could end up + // on the winning path. + Token *new_tok = new Token (tot_cost, extra_cost, NULL, toks); + // NULL: no forward links yet + toks = new_tok; + num_toks_++; + e_found->val = new_tok; + if (changed) *changed = true; + return e_found; + } else { + Token *tok = e_found->val; // There is an existing Token for this state. + if (tok->tot_cost > tot_cost) { // replace old token + tok->tot_cost = tot_cost; + // we don't allocate a new token, the old stays linked in active_toks_ + // we only replace the tot_cost + // in the current frame, there are no forward links (and no extra_cost) + // only in ProcessNonemitting we have to delete forward links + // in case we visit a state for the second time + // those forward links, that lead to this replaced token before: + // they remain and will hopefully be pruned later (PruneForwardLinks...) + if (changed) *changed = true; + } else { + if (changed) *changed = false; + } + return e_found; + } + } + + // prunes outgoing links for all tokens in active_toks_[frame] + // it's called by PruneActiveTokens + // all links, that have link_extra_cost > lattice_beam are pruned + void PruneForwardLinks(int32 frame, bool *extra_costs_changed, + bool *links_pruned, + BaseFloat delta) { + // delta is the amount by which the extra_costs must change + // If delta is larger, we'll tend to go back less far + // toward the beginning of the file. + // extra_costs_changed is set to true if extra_cost was changed for any token + // links_pruned is set to true if any link in any token was pruned + + *extra_costs_changed = false; + *links_pruned = false; + KALDI_ASSERT(frame >= 0 && frame < active_toks_.size()); + if (active_toks_[frame].toks == NULL ) { // empty list; should not happen. + if (!warned_) { + KALDI_WARN << "No tokens alive [doing pruning].. warning first " + "time only for each utterance\n"; + warned_ = true; + } + } + + // We have to iterate until there is no more change, because the links + // are not guaranteed to be in topological order. + bool changed = true; // difference new minus old extra cost >= delta ? + while (changed) { + changed = false; + for (Token *tok = active_toks_[frame].toks; tok != NULL; tok = tok->next) { + ForwardLink *link, *prev_link=NULL; + // will recompute tok_extra_cost for tok. + BaseFloat tok_extra_cost = std::numeric_limits::infinity(); + // tok_extra_cost is the best (min) of link_extra_cost of outgoing links + for (link = tok->links; link != NULL; ) { + // See if we need to excise this link... + Token *next_tok = link->next_tok; + BaseFloat link_extra_cost = next_tok->extra_cost + + ((tok->tot_cost + link->acoustic_cost + link->graph_cost) + - next_tok->tot_cost); // difference in brackets is >= 0 + // link_exta_cost is the difference in score between the best paths + // through link source state and through link destination state + KALDI_ASSERT(link_extra_cost == link_extra_cost); // check for NaN + if (link_extra_cost > config_.lattice_beam) { // excise link + ForwardLink *next_link = link->next; + if (prev_link != NULL) prev_link->next = next_link; + else tok->links = next_link; + delete link; + link = next_link; // advance link but leave prev_link the same. + *links_pruned = true; + } else { // keep the link and update the tok_extra_cost if needed. + if (link_extra_cost < 0.0) { // this is just a precaution. + if (link_extra_cost < -0.01) + KALDI_WARN << "Negative extra_cost: " << link_extra_cost; + link_extra_cost = 0.0; + } + if (link_extra_cost < tok_extra_cost) + tok_extra_cost = link_extra_cost; + prev_link = link; // move to next link + link = link->next; + } + } // for all outgoing links + if (fabs(tok_extra_cost - tok->extra_cost) > delta) + changed = true; // difference new minus old is bigger than delta + tok->extra_cost = tok_extra_cost; + // will be +infinity or <= lattice_beam_. + // infinity indicates, that no forward link survived pruning + } // for all Token on active_toks_[frame] + if (changed) *extra_costs_changed = true; + + // Note: it's theoretically possible that aggressive compiler + // optimizations could cause an infinite loop here for small delta and + // high-dynamic-range scores. + } // while changed + } + + // PruneForwardLinksFinal is a version of PruneForwardLinks that we call + // on the final frame. If there are final tokens active, it uses + // the final-probs for pruning, otherwise it treats all tokens as final. + void PruneForwardLinksFinal(int32 frame) { + KALDI_ASSERT(static_cast(frame+1) == active_toks_.size()); + if (active_toks_[frame].toks == NULL ) // empty list; should not happen. + KALDI_WARN << "No tokens alive at end of file\n"; + + // First go through, working out the best token (do it in parallel + // including final-probs and not including final-probs; we'll take + // the one with final-probs if it's valid). + const BaseFloat infinity = std::numeric_limits::infinity(); + BaseFloat best_cost_final = infinity, + best_cost_nofinal = infinity; + unordered_map tok_to_final_cost; + Elem *cur_toks = toks_.Clear(); // swapping prev_toks_ / cur_toks_ + for (Elem *e = cur_toks, *e_tail; e != NULL; e = e_tail) { + PairId state_pair = e->key; + StateId state = PairToState(state_pair), + lm_state = PairToLmState(state_pair); + Token *tok = e->val; + BaseFloat final_cost = fst_.Final(state).Value() + + lm_diff_fst_->Final(lm_state).Value(); + tok_to_final_cost[tok] = final_cost; + best_cost_final = std::min(best_cost_final, tok->tot_cost + final_cost); + best_cost_nofinal = std::min(best_cost_nofinal, tok->tot_cost); + e_tail = e->tail; + toks_.Delete(e); + } + final_active_ = (best_cost_final != infinity); + + // Now go through tokens on this frame, pruning forward links... may have + // to iterate a few times until there is no more change, because the list is + // not in topological order. + + bool changed = true; + BaseFloat delta = 1.0e-05; + while (changed) { + changed = false; + for (Token *tok = active_toks_[frame].toks; tok != NULL; tok = tok->next) { + ForwardLink *link, *prev_link=NULL; + // will recompute tok_extra_cost. It has a term in it that corresponds + // to the "final-prob", so instead of initializing tok_extra_cost to infinity + // below we set it to the difference between the (score+final_prob) of this token, + // and the best such (score+final_prob). + BaseFloat tok_extra_cost; + if (final_active_) { + BaseFloat final_cost = tok_to_final_cost[tok]; + tok_extra_cost = (tok->tot_cost + final_cost) - best_cost_final; + } else + tok_extra_cost = tok->tot_cost - best_cost_nofinal; + + for (link = tok->links; link != NULL; ) { + // See if we need to excise this link... + Token *next_tok = link->next_tok; + BaseFloat link_extra_cost = next_tok->extra_cost + + ((tok->tot_cost + link->acoustic_cost + link->graph_cost) + - next_tok->tot_cost); + if (link_extra_cost > config_.lattice_beam) { // excise link + ForwardLink *next_link = link->next; + if (prev_link != NULL) prev_link->next = next_link; + else tok->links = next_link; + delete link; + link = next_link; // advance link but leave prev_link the same. + } else { // keep the link and update the tok_extra_cost if needed. + if (link_extra_cost < 0.0) { // this is just a precaution. + if (link_extra_cost < -0.01) + KALDI_WARN << "Negative extra_cost: " << link_extra_cost; + link_extra_cost = 0.0; + } + if (link_extra_cost < tok_extra_cost) + tok_extra_cost = link_extra_cost; + prev_link = link; + link = link->next; + } + } + // prune away tokens worse than lattice_beam above best path. This step + // was not necessary in the non-final case because then, this case + // showed up as having no forward links. Here, the tok_extra_cost has + // an extra component relating to the final-prob. + if (tok_extra_cost > config_.lattice_beam) + tok_extra_cost = infinity; + // to be pruned in PruneTokensForFrame + + if (!ApproxEqual(tok->extra_cost, tok_extra_cost, delta)) + changed = true; + tok->extra_cost = tok_extra_cost; // will be +infinity or <= lattice_beam_. + } + } // while changed + + // Now put surviving Tokens in the final_costs_ hash, which is a class + // member (unlike tok_to_final_costs). + for (Token *tok = active_toks_[frame].toks; tok != NULL; tok = tok->next) { + if (tok->extra_cost != infinity) { + // If the token was not pruned away, + if (final_active_) { + BaseFloat final_cost = tok_to_final_cost[tok]; + if (final_cost != infinity) + final_costs_[tok] = final_cost; + } else { + final_costs_[tok] = 0; + } + } + } + } + + // Prune away any tokens on this frame that have no forward links. + // [we don't do this in PruneForwardLinks because it would give us + // a problem with dangling pointers]. + // It's called by PruneActiveTokens if any forward links have been pruned + void PruneTokensForFrame(int32 frame) { + KALDI_ASSERT(frame >= 0 && frame < active_toks_.size()); + Token *&toks = active_toks_[frame].toks; + if (toks == NULL) + KALDI_WARN << "No tokens alive [doing pruning]\n"; + Token *tok, *next_tok, *prev_tok = NULL; + for (tok = toks; tok != NULL; tok = next_tok) { + next_tok = tok->next; + if (tok->extra_cost == std::numeric_limits::infinity()) { + // token is unreachable from end of graph; (no forward links survived) + // excise tok from list and delete tok. + if (prev_tok != NULL) prev_tok->next = tok->next; + else toks = tok->next; + delete tok; + num_toks_--; + } else { // fetch next Token + prev_tok = tok; + } + } + } + + // Go backwards through still-alive tokens, pruning them. note: cur_frame is + // where hash toks_ are (so we do not want to mess with it because these tokens + // don't yet have forward pointers), but we do all previous frames, unless we + // know that we can safely ignore them because the frame after them was unchanged. + // delta controls when it considers a cost to have changed enough to continue + // going backward and propagating the change. + // for a larger delta, we will recurse less far back + void PruneActiveTokens(int32 cur_frame, BaseFloat delta) { + int32 num_toks_begin = num_toks_; + for (int32 frame = cur_frame-1; frame >= 0; frame--) { + // Reason why we need to prune forward links in this situation: + // (1) we have never pruned them (new TokenList) + // (2) we have not yet pruned the forward links to the next frame, + // after any of those tokens have changed their extra_cost. + if (active_toks_[frame].must_prune_forward_links) { + bool extra_costs_changed = false, links_pruned = false; + PruneForwardLinks(frame, &extra_costs_changed, &links_pruned, delta); + if (extra_costs_changed && frame > 0) // any token has changed extra_cost + active_toks_[frame-1].must_prune_forward_links = true; + if (links_pruned) // any link was pruned + active_toks_[frame].must_prune_tokens = true; + active_toks_[frame].must_prune_forward_links = false; // job done + } + if (frame+1 < cur_frame && // except for last frame (no forward links) + active_toks_[frame+1].must_prune_tokens) { + PruneTokensForFrame(frame+1); + active_toks_[frame+1].must_prune_tokens = false; + } + } + KALDI_VLOG(3) << "PruneActiveTokens: pruned tokens from " << num_toks_begin + << " to " << num_toks_; + } + + // Version of PruneActiveTokens that we call on the final frame. + // Takes into account the final-prob of tokens. + void PruneActiveTokensFinal(int32 cur_frame) { + // returns true if there were final states active + // else returns false and treats all states as final while doing the pruning + // (this can be useful if you want partial lattice output, + // although it can be dangerous, depending what you want the lattices for). + // final_active_ and final_probs_ (a hash) are set internally + // by PruneForwardLinksFinal + int32 num_toks_begin = num_toks_; + PruneForwardLinksFinal(cur_frame); // prune final frame (with final-probs) + // sets final_active_ and final_probs_ + for (int32 frame = cur_frame-1; frame >= 0; frame--) { + bool b1, b2; // values not used. + BaseFloat dontcare = 0.0; // delta of zero means we must always update + PruneForwardLinks(frame, &b1, &b2, dontcare); + PruneTokensForFrame(frame+1); + } + PruneTokensForFrame(0); + KALDI_VLOG(3) << "PruneActiveTokensFinal: pruned tokens from " << num_toks_begin + << " to " << num_toks_; + } + + /// Gets the weight cutoff. Also counts the active tokens. + BaseFloat GetCutoff(Elem *list_head, size_t *tok_count, + BaseFloat *adaptive_beam, Elem **best_elem) { + BaseFloat best_weight = std::numeric_limits::infinity(); + // positive == high cost == bad. + size_t count = 0; + if (config_.max_active == std::numeric_limits::max()) { + for (Elem *e = list_head; e != NULL; e = e->tail, count++) { + BaseFloat w = static_cast(e->val->tot_cost); + if (w < best_weight) { + best_weight = w; + if (best_elem) *best_elem = e; + } + } + if (tok_count != NULL) *tok_count = count; + if (adaptive_beam != NULL) *adaptive_beam = config_.beam; + return best_weight + config_.beam; + } else { + tmp_array_.clear(); + for (Elem *e = list_head; e != NULL; e = e->tail, count++) { + BaseFloat w = e->val->tot_cost; + tmp_array_.push_back(w); + if (w < best_weight) { + best_weight = w; + if (best_elem) *best_elem = e; + } + } + if (tok_count != NULL) *tok_count = count; + if (tmp_array_.size() <= static_cast(config_.max_active)) { + if (adaptive_beam) *adaptive_beam = config_.beam; + return best_weight + config_.beam; + } else { + // the lowest elements (lowest costs, highest likes) + // will be put in the left part of tmp_array. + std::nth_element(tmp_array_.begin(), + tmp_array_.begin()+config_.max_active, + tmp_array_.end()); + // return the tighter of the two beams. + BaseFloat ans = std::min(best_weight + config_.beam, + *(tmp_array_.begin()+config_.max_active)); + if (adaptive_beam) + *adaptive_beam = std::min(config_.beam, + ans - best_weight + config_.beam_delta); + return ans; + } + } + } + + inline StateId PropagateLm(StateId lm_state, + Arc *arc) { // returns new LM state. + if (arc->olabel == 0) { + return lm_state; // no change in LM state if no word crossed. + } else { // Propagate in the LM-diff FST. + Arc lm_arc; + bool ans = lm_diff_fst_->GetArc(lm_state, arc->olabel, &lm_arc); + if (!ans) { // this case is unexpected for statistical LMs. + if (!warned_noarc_) { + warned_noarc_ = true; + KALDI_WARN << "No arc available in LM (unlikely to be correct " + "if a statistical language model); will not warn again"; + } + arc->weight = Weight::Zero(); + return lm_state; // doesn't really matter what we return here; will + // be pruned. + } else { + arc->weight = Times(arc->weight, lm_arc.weight); + arc->olabel = lm_arc.olabel; // probably will be the same. + return lm_arc.nextstate; // return the new LM state. + } + } + } + + void ProcessEmitting(DecodableInterface *decodable, int32 frame) { + // Processes emitting arcs for one frame. Propagates from prev_toks_ to cur_toks_. + Elem *last_toks = toks_.Clear(); // swapping prev_toks_ / cur_toks_ + Elem *best_elem = NULL; + BaseFloat adaptive_beam; + size_t tok_cnt; + BaseFloat cur_cutoff = GetCutoff(last_toks, &tok_cnt, &adaptive_beam, &best_elem); + PossiblyResizeHash(tok_cnt); // This makes sure the hash is always big enough. + + BaseFloat next_cutoff = std::numeric_limits::infinity(); + // pruning "online" before having seen all tokens + + // First process the best token to get a hopefully + // reasonably tight bound on the next cutoff. + if (best_elem) { + PairId state_pair = best_elem->key; + StateId state = PairToState(state_pair), // state in "fst" + lm_state = PairToLmState(state_pair); + Token *tok = best_elem->val; + for (fst::ArcIterator > aiter(fst_, state); + !aiter.Done(); + aiter.Next()) { + Arc arc = aiter.Value(); + if (arc.ilabel != 0) { // propagate.. + PropagateLm(lm_state, &arc); // may affect "arc.weight". + // We don't need the return value (the new LM state). + arc.weight = Times(arc.weight, + Weight(-decodable->LogLikelihood(frame-1, arc.ilabel))); + BaseFloat new_weight = arc.weight.Value() + tok->tot_cost; + if (new_weight + adaptive_beam < next_cutoff) + next_cutoff = new_weight + adaptive_beam; + } + } + } + + // the tokens are now owned here, in last_toks, and the hash is empty. + // 'owned' is a complex thing here; the point is we need to call DeleteElem + // on each elem 'e' to let toks_ know we're done with them. + for (Elem *e = last_toks, *e_tail; e != NULL; e = e_tail) { + // loop this way because we delete "e" as we go. + PairId state_pair = e->key; + StateId state = PairToState(state_pair), + lm_state = PairToLmState(state_pair); + Token *tok = e->val; + if (tok->tot_cost <= cur_cutoff) { + for (fst::ArcIterator > aiter(fst_, state); + !aiter.Done(); + aiter.Next()) { + const Arc &arc_ref = aiter.Value(); + if (arc_ref.ilabel != 0) { // propagate.. + Arc arc(arc_ref); + StateId next_lm_state = PropagateLm(lm_state, &arc); + BaseFloat ac_cost = -decodable->LogLikelihood(frame-1, arc.ilabel), + graph_cost = arc.weight.Value(), + cur_cost = tok->tot_cost, + tot_cost = cur_cost + ac_cost + graph_cost; + if (tot_cost >= next_cutoff) continue; + else if (tot_cost + adaptive_beam < next_cutoff) + next_cutoff = tot_cost + adaptive_beam; // prune by best current token + PairId next_pair = ConstructPair(arc.nextstate, next_lm_state); + Elem *e_next = FindOrAddToken(next_pair, frame, tot_cost, true, NULL); + // true: emitting, NULL: no change indicator needed + + // Add ForwardLink from tok to next_tok (put on head of list tok->links) + tok->links = new ForwardLink(e_next->val, arc.ilabel, arc.olabel, + graph_cost, ac_cost, tok->links); + } + } // for all arcs + } + e_tail = e->tail; + toks_.Delete(e); // delete Elem + } + } + + void ProcessNonemitting(int32 frame) { + // note: "frame" is the same as emitting states just processed. + + // Processes nonemitting arcs for one frame. Propagates within toks_. + // Note-- this queue structure is is not very optimal as + // it may cause us to process states unnecessarily (e.g. more than once), + // but in the baseline code, turning this vector into a set to fix this + // problem did not improve overall speed. + + KALDI_ASSERT(queue_.empty()); + BaseFloat best_cost = std::numeric_limits::infinity(); + for (const Elem *e = toks_.GetList(); e != NULL; e = e->tail) { + queue_.push_back(e); + // for pruning with current best token + best_cost = std::min(best_cost, static_cast(e->val->tot_cost)); + } + if (queue_.empty()) { + if (!warned_) { + KALDI_ERR << "Error in ProcessNonemitting: no surviving tokens: frame is " + << frame; + warned_ = true; + } + } + BaseFloat cutoff = best_cost + config_.beam; + + while (!queue_.empty()) { + const Elem *e = queue_.back(); + queue_.pop_back(); + + PairId state_pair = e->key; + Token *tok = e->val; // would segfault if state not in + // toks_ but this can't happen. + BaseFloat cur_cost = tok->tot_cost; + if (cur_cost >= cutoff) // Don't bother processing successors. + continue; + StateId state = PairToState(state_pair), + lm_state = PairToLmState(state_pair); + // If "tok" has any existing forward links, delete them, + // because we're about to regenerate them. This is a kind + // of non-optimality (remember, this is the simple decoder), + // but since most states are emitting it's not a huge issue. + tok->DeleteForwardLinks(); // necessary when re-visiting + tok->links = NULL; + for (fst::ArcIterator > aiter(fst_, state); + !aiter.Done(); + aiter.Next()) { + const Arc &arc_ref = aiter.Value(); + if (arc_ref.ilabel == 0) { // propagate nonemitting only... + Arc arc(arc_ref); + StateId next_lm_state = PropagateLm(lm_state, &arc); + BaseFloat graph_cost = arc.weight.Value(), + tot_cost = cur_cost + graph_cost; + if (tot_cost < cutoff) { + bool changed; + PairId next_pair = ConstructPair(arc.nextstate, next_lm_state); + Elem *e_new = FindOrAddToken(next_pair, frame, tot_cost, + false, &changed); // false: non-emit + + tok->links = new ForwardLink(e_new->val, 0, arc.olabel, + graph_cost, 0, tok->links); + + // "changed" tells us whether the new token has a different + // cost from before, or is new [if so, add into queue]. + if (changed) queue_.push_back(e_new); + } + } + } // for all arcs + } // while queue not empty + } + + + // HashList defined in ../util/hash-list.h. It actually allows us to maintain + // more than one list (e.g. for current and previous frames), but only one of + // them at a time can be indexed by StateId. + HashList toks_; + std::vector active_toks_; // Lists of tokens, indexed by + // frame (members of TokenList are toks, must_prune_forward_links, + // must_prune_tokens). + std::vector queue_; // temp variable used in ProcessNonemitting, + std::vector tmp_array_; // used in GetCutoff. + // make it class member to avoid internal new/delete. + const fst::Fst &fst_; + fst::DeterministicOnDemandFst *lm_diff_fst_; + LatticeBiglmFasterDecoderConfig config_; + bool warned_noarc_; + int32 num_toks_; // current total #toks allocated... + bool warned_; + bool final_active_; // use this to say whether we found active final tokens + // on the last frame. + std::map final_costs_; // A cache of final-costs + // of tokens on the last frame-- it's just convenient to store it this way. + + // It might seem unclear why we call DeleteElems(toks_.Clear()). + // There are two separate cleanup tasks we need to do at when we start a new file. + // one is to delete the Token objects in the list; the other is to delete + // the Elem objects. toks_.Clear() just clears them from the hash and gives ownership + // to the caller, who then has to call toks_.Delete(e) for each one. It was designed + // this way for convenience in propagating tokens from one frame to the next. + void DeleteElems(Elem *list) { + for (Elem *e = list, *e_tail; e != NULL; e = e_tail) { + e_tail = e->tail; + toks_.Delete(e); + } + toks_.Clear(); + } + + void ClearActiveTokens() { // a cleanup routine, at utt end/begin + for (size_t i = 0; i < active_toks_.size(); i++) { + // Delete all tokens alive on this frame, and any forward + // links they may have. + for (Token *tok = active_toks_[i].toks; tok != NULL; ) { + tok->DeleteForwardLinks(); + Token *next_tok = tok->next; + delete tok; + num_toks_--; + tok = next_tok; + } + } + active_toks_.clear(); + KALDI_ASSERT(num_toks_ == 0); + } +}; + +} // end namespace kaldi. + +#endif diff --git a/runtime/onnxruntime/third_party/kaldi/decoder/lattice-faster-decoder.cc b/runtime/onnxruntime/third_party/kaldi/decoder/lattice-faster-decoder.cc new file mode 100644 index 000000000..6cd63d9aa --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/decoder/lattice-faster-decoder.cc @@ -0,0 +1,1061 @@ +// decoder/lattice-faster-decoder.cc + +// Copyright 2009-2012 Microsoft Corporation Mirko Hannemann +// 2013-2018 Johns Hopkins University (Author: Daniel Povey) +// 2014 Guoguo Chen +// 2018 Zhehuai Chen + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#include "decoder/lattice-faster-decoder.h" +#include "lat/lattice-functions.h" + +namespace kaldi { +int niter = 0; +// instantiate this class once for each thing you have to decode. +template +LatticeFasterDecoderTpl::LatticeFasterDecoderTpl( + const FST &fst, const LatticeFasterDecoderConfig &config) + : fst_(&fst), + delete_fst_(false), + config_(config), + num_toks_(0), + token_pool_(config.memory_pool_tokens_block_size), + forward_link_pool_(config.memory_pool_links_block_size) { + config.Check(); + toks_.SetSize(1000); // just so on the first frame we do something reasonable. +} + +template +LatticeFasterDecoderTpl::LatticeFasterDecoderTpl( + const LatticeFasterDecoderConfig &config, FST *fst) + : fst_(fst), + delete_fst_(true), + config_(config), + num_toks_(0), + token_pool_(config.memory_pool_tokens_block_size), + forward_link_pool_(config.memory_pool_links_block_size) { + config.Check(); + toks_.SetSize(1000); // just so on the first frame we do something reasonable. +} + +template +LatticeFasterDecoderTpl::~LatticeFasterDecoderTpl() { + DeleteElems(toks_.Clear()); + ClearActiveTokens(); + if (delete_fst_) delete fst_; +} + +template +void LatticeFasterDecoderTpl::InitDecoding() { + // clean up from last time: + DeleteElems(toks_.Clear()); + cost_offsets_.clear(); + ClearActiveTokens(); + warned_ = false; + num_toks_ = 0; + decoding_finalized_ = false; + final_costs_.clear(); + StateId start_state = fst_->Start(); + KALDI_ASSERT(start_state != fst::kNoStateId); + active_toks_.resize(1); + Token *start_tok = + new (token_pool_.Allocate()) Token(0.0, 0.0, NULL, NULL, NULL); + active_toks_[0].toks = start_tok; + toks_.Insert(start_state, start_tok); + num_toks_++; + ProcessNonemitting(config_.beam); +} + +// Returns true if any kind of traceback is available (not necessarily from +// a final state). It should only very rarely return false; this indicates +// an unusual search error. +template +bool LatticeFasterDecoderTpl::Decode(DecodableInterface *decodable) { + InitDecoding(); + // We use 1-based indexing for frames in this decoder (if you view it in + // terms of features), but note that the decodable object uses zero-based + // numbering, which we have to correct for when we call it. + AdvanceDecoding(decodable); + FinalizeDecoding(); + + // Returns true if we have any kind of traceback available (not necessarily + // to the end state; query ReachedFinal() for that). + return !active_toks_.empty() && active_toks_.back().toks != NULL; +} + + +// Outputs an FST corresponding to the single best path through the lattice. +template +bool LatticeFasterDecoderTpl::GetBestPath(Lattice *olat, + bool use_final_probs) const { + Lattice raw_lat; + GetRawLattice(&raw_lat, use_final_probs); + ShortestPath(raw_lat, olat); + return (olat->NumStates() != 0); +} + + +// Outputs an FST corresponding to the raw, state-level lattice +template +bool LatticeFasterDecoderTpl::GetRawLattice( + Lattice *ofst, + bool use_final_probs) const { + typedef LatticeArc Arc; + typedef Arc::StateId StateId; + typedef Arc::Weight Weight; + typedef Arc::Label Label; + + // Note: you can't use the old interface (Decode()) if you want to + // get the lattice with use_final_probs = false. You'd have to do + // InitDecoding() and then AdvanceDecoding(). + if (decoding_finalized_ && !use_final_probs) + KALDI_ERR << "You cannot call FinalizeDecoding() and then call " + << "GetRawLattice() with use_final_probs == false"; + + unordered_map final_costs_local; + + const unordered_map &final_costs = + (decoding_finalized_ ? final_costs_ : final_costs_local); + if (!decoding_finalized_ && use_final_probs) + ComputeFinalCosts(&final_costs_local, NULL, NULL); + + ofst->DeleteStates(); + // num-frames plus one (since frames are one-based, and we have + // an extra frame for the start-state). + int32 num_frames = active_toks_.size() - 1; + KALDI_ASSERT(num_frames > 0); + const int32 bucket_count = num_toks_/2 + 3; + unordered_map tok_map(bucket_count); + // First create all states. + std::vector token_list; + for (int32 f = 0; f <= num_frames; f++) { + if (active_toks_[f].toks == NULL) { + KALDI_WARN << "GetRawLattice: no tokens active on frame " << f + << ": not producing lattice.\n"; + return false; + } + TopSortTokens(active_toks_[f].toks, &token_list); + for (size_t i = 0; i < token_list.size(); i++) + if (token_list[i] != NULL) + tok_map[token_list[i]] = ofst->AddState(); + } + // The next statement sets the start state of the output FST. Because we + // topologically sorted the tokens, state zero must be the start-state. + ofst->SetStart(0); + + KALDI_VLOG(4) << "init:" << num_toks_/2 + 3 << " buckets:" + << tok_map.bucket_count() << " load:" << tok_map.load_factor() + << " max:" << tok_map.max_load_factor(); + // Now create all arcs. + for (int32 f = 0; f <= num_frames; f++) { + for (Token *tok = active_toks_[f].toks; tok != NULL; tok = tok->next) { + StateId cur_state = tok_map[tok]; + for (ForwardLinkT *l = tok->links; + l != NULL; + l = l->next) { + typename unordered_map::const_iterator + iter = tok_map.find(l->next_tok); + StateId nextstate = iter->second; + KALDI_ASSERT(iter != tok_map.end()); + BaseFloat cost_offset = 0.0; + if (l->ilabel != 0) { // emitting.. + KALDI_ASSERT(f >= 0 && f < cost_offsets_.size()); + cost_offset = cost_offsets_[f]; + } + Arc arc(l->ilabel, l->olabel, + Weight(l->graph_cost, l->acoustic_cost - cost_offset), + nextstate); + ofst->AddArc(cur_state, arc); + } + if (f == num_frames) { + if (use_final_probs && !final_costs.empty()) { + typename unordered_map::const_iterator + iter = final_costs.find(tok); + if (iter != final_costs.end()) + ofst->SetFinal(cur_state, LatticeWeight(iter->second, 0)); + } else { + ofst->SetFinal(cur_state, LatticeWeight::One()); + } + } + } + } + return (ofst->NumStates() > 0); +} + + +// This function is now deprecated, since now we do determinization from outside +// the LatticeFasterDecoder class. Outputs an FST corresponding to the +// lattice-determinized lattice (one path per word sequence). +template +bool LatticeFasterDecoderTpl::GetLattice(CompactLattice *ofst, + bool use_final_probs) const { + Lattice raw_fst; + GetRawLattice(&raw_fst, use_final_probs); + Invert(&raw_fst); // make it so word labels are on the input. + // (in phase where we get backward-costs). + fst::ILabelCompare ilabel_comp; + ArcSort(&raw_fst, ilabel_comp); // sort on ilabel; makes + // lattice-determinization more efficient. + + fst::DeterminizeLatticePrunedOptions lat_opts; + lat_opts.max_mem = config_.det_opts.max_mem; + + DeterminizeLatticePruned(raw_fst, config_.lattice_beam, ofst, lat_opts); + raw_fst.DeleteStates(); // Free memory-- raw_fst no longer needed. + Connect(ofst); // Remove unreachable states... there might be + // a small number of these, in some cases. + // Note: if something went wrong and the raw lattice was empty, + // we should still get to this point in the code without warnings or failures. + return (ofst->NumStates() != 0); +} + +template +void LatticeFasterDecoderTpl::PossiblyResizeHash(size_t num_toks) { + size_t new_sz = static_cast(static_cast(num_toks) + * config_.hash_ratio); + if (new_sz > toks_.Size()) { + toks_.SetSize(new_sz); + } +} + +/* + A note on the definition of extra_cost. + + extra_cost is used in pruning tokens, to save memory. + + extra_cost can be thought of as a beta (backward) cost assuming + we had set the betas on currently-active tokens to all be the negative + of the alphas for those tokens. (So all currently active tokens would + be on (tied) best paths). + + We can use the extra_cost to accurately prune away tokens that we know will + never appear in the lattice. If the extra_cost is greater than the desired + lattice beam, the token would provably never appear in the lattice, so we can + prune away the token. + + (Note: we don't update all the extra_costs every time we update a frame; we + only do it every 'config_.prune_interval' frames). + */ + +// FindOrAddToken either locates a token in hash of toks_, +// or if necessary inserts a new, empty token (i.e. with no forward links) +// for the current frame. [note: it's inserted if necessary into hash toks_ +// and also into the singly linked list of tokens active on this frame +// (whose head is at active_toks_[frame]). +template +inline typename LatticeFasterDecoderTpl::Elem* +LatticeFasterDecoderTpl::FindOrAddToken( + StateId state, int32 frame_plus_one, BaseFloat tot_cost, + Token *backpointer, bool *changed, StateId bias_lm_state) { + // Returns the Token pointer. Sets "changed" (if non-NULL) to true + // if the token was newly created or the cost changed. + KALDI_ASSERT(frame_plus_one < active_toks_.size()); + Token *&toks = active_toks_[frame_plus_one].toks; + Elem *e_found = toks_.Insert(state, NULL); + if (e_found->val == NULL) { // no such token presently. + const BaseFloat extra_cost = 0.0; + // tokens on the currently final frame have zero extra_cost + // as any of them could end up + // on the winning path. + Token *new_tok = new (token_pool_.Allocate()) + Token(tot_cost, extra_cost, NULL, toks, backpointer); + // NULL: no forward links yet + new_tok->bias_lm_state = bias_lm_state; + toks = new_tok; + num_toks_++; + e_found->val = new_tok; + if (changed) *changed = true; + return e_found; + } else { + Token *tok = e_found->val; // There is an existing Token for this state. + if (tok->tot_cost > tot_cost) { // replace old token + tok->bias_lm_state = bias_lm_state; + tok->tot_cost = tot_cost; + // SetBackpointer() just does tok->backpointer = backpointer in + // the case where Token == BackpointerToken, else nothing. + tok->SetBackpointer(backpointer); + // we don't allocate a new token, the old stays linked in active_toks_ + // we only replace the tot_cost + // in the current frame, there are no forward links (and no extra_cost) + // only in ProcessNonemitting we have to delete forward links + // in case we visit a state for the second time + // those forward links, that lead to this replaced token before: + // they remain and will hopefully be pruned later (PruneForwardLinks...) + if (changed) *changed = true; + } else { + if (changed) *changed = false; + } + return e_found; + } +} + +// prunes outgoing links for all tokens in active_toks_[frame] +// it's called by PruneActiveTokens +// all links, that have link_extra_cost > lattice_beam are pruned +template +void LatticeFasterDecoderTpl::PruneForwardLinks( + int32 frame_plus_one, bool *extra_costs_changed, + bool *links_pruned, BaseFloat delta) { + // delta is the amount by which the extra_costs must change + // If delta is larger, we'll tend to go back less far + // toward the beginning of the file. + // extra_costs_changed is set to true if extra_cost was changed for any token + // links_pruned is set to true if any link in any token was pruned + + *extra_costs_changed = false; + *links_pruned = false; + KALDI_ASSERT(frame_plus_one >= 0 && frame_plus_one < active_toks_.size()); + if (active_toks_[frame_plus_one].toks == NULL) { // empty list; should not happen. + if (!warned_) { + KALDI_WARN << "No tokens alive [doing pruning].. warning first " + "time only for each utterance\n"; + warned_ = true; + } + } + + // We have to iterate until there is no more change, because the links + // are not guaranteed to be in topological order. + bool changed = true; // difference new minus old extra cost >= delta ? + while (changed) { + changed = false; + for (Token *tok = active_toks_[frame_plus_one].toks; + tok != NULL; tok = tok->next) { + ForwardLinkT *link, *prev_link = NULL; + // will recompute tok_extra_cost for tok. + BaseFloat tok_extra_cost = std::numeric_limits::infinity(); + // tok_extra_cost is the best (min) of link_extra_cost of outgoing links + for (link = tok->links; link != NULL; ) { + // See if we need to excise this link... + Token *next_tok = link->next_tok; + BaseFloat link_extra_cost = next_tok->extra_cost + + ((tok->tot_cost + link->acoustic_cost + link->graph_cost) + - next_tok->tot_cost); // difference in brackets is >= 0 + // link_exta_cost is the difference in score between the best paths + // through link source state and through link destination state + KALDI_ASSERT(link_extra_cost == link_extra_cost); // check for NaN + if (link_extra_cost > config_.lattice_beam) { // excise link + ForwardLinkT *next_link = link->next; + if (prev_link != NULL) prev_link->next = next_link; + else tok->links = next_link; + forward_link_pool_.Free(link); + link = next_link; // advance link but leave prev_link the same. + *links_pruned = true; + } else { // keep the link and update the tok_extra_cost if needed. + if (link_extra_cost < 0.0) { // this is just a precaution. + //if (link_extra_cost < -0.01) + // KALDI_WARN << "Negative extra_cost: " << link_extra_cost; + link_extra_cost = 0.0; + } + if (link_extra_cost < tok_extra_cost) + tok_extra_cost = link_extra_cost; + prev_link = link; // move to next link + link = link->next; + } + } // for all outgoing links + if (fabs(tok_extra_cost - tok->extra_cost) > delta) + changed = true; // difference new minus old is bigger than delta + tok->extra_cost = tok_extra_cost; + // will be +infinity or <= lattice_beam_. + // infinity indicates, that no forward link survived pruning + } // for all Token on active_toks_[frame] + if (changed) *extra_costs_changed = true; + + // Note: it's theoretically possible that aggressive compiler + // optimizations could cause an infinite loop here for small delta and + // high-dynamic-range scores. + } // while changed +} + +// PruneForwardLinksFinal is a version of PruneForwardLinks that we call +// on the final frame. If there are final tokens active, it uses +// the final-probs for pruning, otherwise it treats all tokens as final. +template +void LatticeFasterDecoderTpl::PruneForwardLinksFinal() { + KALDI_ASSERT(!active_toks_.empty()); + int32 frame_plus_one = active_toks_.size() - 1; + + if (active_toks_[frame_plus_one].toks == NULL) // empty list; should not happen. + KALDI_WARN << "No tokens alive at end of file"; + + typedef typename unordered_map::const_iterator IterType; + ComputeFinalCosts(&final_costs_, &final_relative_cost_, &final_best_cost_); + decoding_finalized_ = true; + // We call DeleteElems() as a nicety, not because it's really necessary; + // otherwise there would be a time, after calling PruneTokensForFrame() on the + // final frame, when toks_.GetList() or toks_.Clear() would contain pointers + // to nonexistent tokens. + DeleteElems(toks_.Clear()); + + // Now go through tokens on this frame, pruning forward links... may have to + // iterate a few times until there is no more change, because the list is not + // in topological order. This is a modified version of the code in + // PruneForwardLinks, but here we also take account of the final-probs. + bool changed = true; + BaseFloat delta = 1.0e-05; + while (changed) { + changed = false; + for (Token *tok = active_toks_[frame_plus_one].toks; + tok != NULL; tok = tok->next) { + ForwardLinkT *link, *prev_link = NULL; + // will recompute tok_extra_cost. It has a term in it that corresponds + // to the "final-prob", so instead of initializing tok_extra_cost to infinity + // below we set it to the difference between the (score+final_prob) of this token, + // and the best such (score+final_prob). + BaseFloat final_cost; + if (final_costs_.empty()) { + final_cost = 0.0; + } else { + IterType iter = final_costs_.find(tok); + if (iter != final_costs_.end()) + final_cost = iter->second; + else + final_cost = std::numeric_limits::infinity(); + } + BaseFloat tok_extra_cost = tok->tot_cost + final_cost - final_best_cost_; + // tok_extra_cost will be a "min" over either directly being final, or + // being indirectly final through other links, and the loop below may + // decrease its value: + for (link = tok->links; link != NULL; ) { + // See if we need to excise this link... + Token *next_tok = link->next_tok; + BaseFloat link_extra_cost = next_tok->extra_cost + + ((tok->tot_cost + link->acoustic_cost + link->graph_cost) + - next_tok->tot_cost); + if (link_extra_cost > config_.lattice_beam) { // excise link + ForwardLinkT *next_link = link->next; + if (prev_link != NULL) prev_link->next = next_link; + else tok->links = next_link; + forward_link_pool_.Free(link); + link = next_link; // advance link but leave prev_link the same. + } else { // keep the link and update the tok_extra_cost if needed. + if (link_extra_cost < 0.0) { // this is just a precaution. + //if (link_extra_cost < -0.01) + // KALDI_WARN << "Negative extra_cost: " << link_extra_cost; + link_extra_cost = 0.0; + } + if (link_extra_cost < tok_extra_cost) + tok_extra_cost = link_extra_cost; + prev_link = link; + link = link->next; + } + } + // prune away tokens worse than lattice_beam above best path. This step + // was not necessary in the non-final case because then, this case + // showed up as having no forward links. Here, the tok_extra_cost has + // an extra component relating to the final-prob. + if (tok_extra_cost > config_.lattice_beam) + tok_extra_cost = std::numeric_limits::infinity(); + // to be pruned in PruneTokensForFrame + + if (!ApproxEqual(tok->extra_cost, tok_extra_cost, delta)) + changed = true; + tok->extra_cost = tok_extra_cost; // will be +infinity or <= lattice_beam_. + } + } // while changed +} + +template +BaseFloat LatticeFasterDecoderTpl::FinalRelativeCost() const { + if (!decoding_finalized_) { + BaseFloat relative_cost; + ComputeFinalCosts(NULL, &relative_cost, NULL); + return relative_cost; + } else { + // we're not allowed to call that function if FinalizeDecoding() has + // been called; return a cached value. + return final_relative_cost_; + } +} + + +// Prune away any tokens on this frame that have no forward links. +// [we don't do this in PruneForwardLinks because it would give us +// a problem with dangling pointers]. +// It's called by PruneActiveTokens if any forward links have been pruned +template +void LatticeFasterDecoderTpl::PruneTokensForFrame(int32 frame_plus_one) { + KALDI_ASSERT(frame_plus_one >= 0 && frame_plus_one < active_toks_.size()); + Token *&toks = active_toks_[frame_plus_one].toks; + if (toks == NULL) + KALDI_WARN << "No tokens alive [doing pruning]"; + Token *tok, *next_tok, *prev_tok = NULL; + for (tok = toks; tok != NULL; tok = next_tok) { + next_tok = tok->next; + if (tok->extra_cost == std::numeric_limits::infinity()) { + // token is unreachable from end of graph; (no forward links survived) + // excise tok from list and delete tok. + if (prev_tok != NULL) prev_tok->next = tok->next; + else toks = tok->next; + token_pool_.Free(tok); + num_toks_--; + } else { // fetch next Token + prev_tok = tok; + } + } +} + +// Go backwards through still-alive tokens, pruning them, starting not from +// the current frame (where we want to keep all tokens) but from the frame before +// that. We go backwards through the frames and stop when we reach a point +// where the delta-costs are not changing (and the delta controls when we consider +// a cost to have "not changed"). +template +void LatticeFasterDecoderTpl::PruneActiveTokens(BaseFloat delta) { + int32 cur_frame_plus_one = NumFramesDecoded(); + int32 num_toks_begin = num_toks_; + // The index "f" below represents a "frame plus one", i.e. you'd have to subtract + // one to get the corresponding index for the decodable object. + for (int32 f = cur_frame_plus_one - 1; f >= 0; f--) { + // Reason why we need to prune forward links in this situation: + // (1) we have never pruned them (new TokenList) + // (2) we have not yet pruned the forward links to the next f, + // after any of those tokens have changed their extra_cost. + if (active_toks_[f].must_prune_forward_links) { + bool extra_costs_changed = false, links_pruned = false; + PruneForwardLinks(f, &extra_costs_changed, &links_pruned, delta); + if (extra_costs_changed && f > 0) // any token has changed extra_cost + active_toks_[f-1].must_prune_forward_links = true; + if (links_pruned) // any link was pruned + active_toks_[f].must_prune_tokens = true; + active_toks_[f].must_prune_forward_links = false; // job done + } + if (f+1 < cur_frame_plus_one && // except for last f (no forward links) + active_toks_[f+1].must_prune_tokens) { + PruneTokensForFrame(f+1); + active_toks_[f+1].must_prune_tokens = false; + } + } + KALDI_VLOG(4) << "PruneActiveTokens: pruned tokens from " << num_toks_begin + << " to " << num_toks_; +} + +template +void LatticeFasterDecoderTpl::ComputeFinalCosts( + unordered_map *final_costs, + BaseFloat *final_relative_cost, + BaseFloat *final_best_cost) const { + KALDI_ASSERT(!decoding_finalized_); + if (final_costs != NULL) + final_costs->clear(); + const Elem *final_toks = toks_.GetList(); + BaseFloat infinity = std::numeric_limits::infinity(); + BaseFloat best_cost = infinity, + best_cost_with_final = infinity; + + while (final_toks != NULL) { + StateId state = final_toks->key; + Token *tok = final_toks->val; + const Elem *next = final_toks->tail; + BaseFloat final_cost = fst_->Final(state).Value(); + BaseFloat cost = tok->tot_cost, + cost_with_final = cost + final_cost; + best_cost = std::min(cost, best_cost); + best_cost_with_final = std::min(cost_with_final, best_cost_with_final); + if (final_costs != NULL && final_cost != infinity) + (*final_costs)[tok] = final_cost; + final_toks = next; + } + if (final_relative_cost != NULL) { + if (best_cost == infinity && best_cost_with_final == infinity) { + // Likely this will only happen if there are no tokens surviving. + // This seems the least bad way to handle it. + *final_relative_cost = infinity; + } else { + *final_relative_cost = best_cost_with_final - best_cost; + } + } + if (final_best_cost != NULL) { + if (best_cost_with_final != infinity) { // final-state exists. + *final_best_cost = best_cost_with_final; + } else { // no final-state exists. + *final_best_cost = best_cost; + } + } +} + +template +void LatticeFasterDecoderTpl::AdvanceDecoding(DecodableInterface *decodable, + int32 max_num_frames) { + if (std::is_same >::value) { + // if the type 'FST' is the FST base-class, then see if the FST type of fst_ + // is actually VectorFst or ConstFst. If so, call the AdvanceDecoding() + // function after casting *this to the more specific type. + if (fst_->Type() == "const") { + LatticeFasterDecoderTpl, Token> *this_cast = + reinterpret_cast, Token>* >(this); + this_cast->AdvanceDecoding(decodable, max_num_frames); + return; + } else if (fst_->Type() == "vector") { + LatticeFasterDecoderTpl, Token> *this_cast = + reinterpret_cast, Token>* >(this); + this_cast->AdvanceDecoding(decodable, max_num_frames); + return; + } + } + + + KALDI_ASSERT(!active_toks_.empty() && !decoding_finalized_ && + "You must call InitDecoding() before AdvanceDecoding"); + int32 num_frames_ready = decodable->NumFramesReady(); + // num_frames_ready must be >= num_frames_decoded, or else + // the number of frames ready must have decreased (which doesn't + // make sense) or the decodable object changed between calls + // (which isn't allowed). + KALDI_ASSERT(num_frames_ready >= NumFramesDecoded()); + int32 target_frames_decoded = num_frames_ready; + if (max_num_frames >= 0) + target_frames_decoded = std::min(target_frames_decoded, + NumFramesDecoded() + max_num_frames); + while (NumFramesDecoded() < target_frames_decoded) { + if (NumFramesDecoded() % config_.prune_interval == 0) { + PruneActiveTokens(config_.lattice_beam * config_.prune_scale); + } + BaseFloat cost_cutoff = ProcessEmitting(decodable); + ProcessNonemitting(cost_cutoff); + } +} + +// FinalizeDecoding() is a version of PruneActiveTokens that we call +// (optionally) on the final frame. Takes into account the final-prob of +// tokens. This function used to be called PruneActiveTokensFinal(). +template +void LatticeFasterDecoderTpl::FinalizeDecoding() { + int32 final_frame_plus_one = NumFramesDecoded(); + int32 num_toks_begin = num_toks_; + // PruneForwardLinksFinal() prunes final frame (with final-probs), and + // sets decoding_finalized_. + PruneForwardLinksFinal(); + for (int32 f = final_frame_plus_one - 1; f >= 0; f--) { + bool b1, b2; // values not used. + BaseFloat dontcare = 0.0; // delta of zero means we must always update + PruneForwardLinks(f, &b1, &b2, dontcare); + PruneTokensForFrame(f + 1); + } + PruneTokensForFrame(0); + KALDI_VLOG(4) << "pruned tokens from " << num_toks_begin + << " to " << num_toks_; +} + +/// Gets the weight cutoff. Also counts the active tokens. +template +BaseFloat LatticeFasterDecoderTpl::GetCutoff(Elem *list_head, size_t *tok_count, + BaseFloat *adaptive_beam, Elem **best_elem) { + BaseFloat best_weight = std::numeric_limits::infinity(); + // positive == high cost == bad. + size_t count = 0; + if (config_.max_active == std::numeric_limits::max() && + config_.min_active == 0) { + for (Elem *e = list_head; e != NULL; e = e->tail, count++) { + BaseFloat w = static_cast(e->val->tot_cost); + if (w < best_weight) { + best_weight = w; + if (best_elem) *best_elem = e; + } + } + if (tok_count != NULL) *tok_count = count; + if (adaptive_beam != NULL) *adaptive_beam = config_.beam; + return best_weight + config_.beam; + } else { + tmp_array_.clear(); + for (Elem *e = list_head; e != NULL; e = e->tail, count++) { + BaseFloat w = e->val->tot_cost; + tmp_array_.push_back(w); + if (w < best_weight) { + best_weight = w; + if (best_elem) *best_elem = e; + } + } + if (tok_count != NULL) *tok_count = count; + + BaseFloat beam_cutoff = best_weight + config_.beam, + min_active_cutoff = std::numeric_limits::infinity(), + max_active_cutoff = std::numeric_limits::infinity(); + + KALDI_VLOG(6) << "Number of tokens active on frame " << NumFramesDecoded() + << " is " << tmp_array_.size(); + + if (tmp_array_.size() > static_cast(config_.max_active)) { + std::nth_element(tmp_array_.begin(), + tmp_array_.begin() + config_.max_active, + tmp_array_.end()); + max_active_cutoff = tmp_array_[config_.max_active]; + } + if (max_active_cutoff < beam_cutoff) { // max_active is tighter than beam. + if (adaptive_beam) + *adaptive_beam = max_active_cutoff - best_weight + config_.beam_delta; + return max_active_cutoff; + } + if (tmp_array_.size() > static_cast(config_.min_active)) { + if (config_.min_active == 0) min_active_cutoff = best_weight; + else { + std::nth_element(tmp_array_.begin(), + tmp_array_.begin() + config_.min_active, + tmp_array_.size() > static_cast(config_.max_active) ? + tmp_array_.begin() + config_.max_active : + tmp_array_.end()); + min_active_cutoff = tmp_array_[config_.min_active]; + } + } + if (min_active_cutoff > beam_cutoff) { // min_active is looser than beam. + if (adaptive_beam) + *adaptive_beam = min_active_cutoff - best_weight + config_.beam_delta; + return min_active_cutoff; + } else { + *adaptive_beam = config_.beam; + return beam_cutoff; + } + } +} + +template +BaseFloat LatticeFasterDecoderTpl::ProcessEmitting( + DecodableInterface *decodable) { + KALDI_ASSERT(active_toks_.size() > 0); + int32 frame = active_toks_.size() - 1; // frame is the frame-index + // (zero-based) used to get likelihoods + // from the decodable object. + active_toks_.resize(active_toks_.size() + 1); + + Elem *final_toks = toks_.Clear(); // analogous to swapping prev_toks_ / cur_toks_ + // in simple-decoder.h. Removes the Elems from + // being indexed in the hash in toks_. + Elem *best_elem = NULL; + BaseFloat adaptive_beam; + size_t tok_cnt; + BaseFloat cur_cutoff = GetCutoff(final_toks, &tok_cnt, &adaptive_beam, &best_elem); + KALDI_VLOG(6) << "Adaptive beam on frame " << NumFramesDecoded() << " is " + << adaptive_beam; + + PossiblyResizeHash(tok_cnt); // This makes sure the hash is always big enough. + + BaseFloat next_cutoff = std::numeric_limits::infinity(); + // pruning "online" before having seen all tokens + + BaseFloat cost_offset = 0.0; // Used to keep probabilities in a good + // dynamic range. + + + // First process the best token to get a hopefully + // reasonably tight bound on the next cutoff. The only + // products of the next block are "next_cutoff" and "cost_offset". + if (best_elem) { + StateId state = best_elem->key; + Token *tok = best_elem->val; + cost_offset = - tok->tot_cost; + for (fst::ArcIterator aiter(*fst_, state); + !aiter.Done(); + aiter.Next()) { + const Arc &arc = aiter.Value(); + if (arc.ilabel != 0) { // propagate.. + if (arc.nextstate == state) continue; + BaseFloat new_weight = arc.weight.Value() + cost_offset - + decodable->LogLikelihood(frame, arc.ilabel) + tok->tot_cost; + + if (new_weight + adaptive_beam < next_cutoff) + next_cutoff = new_weight + adaptive_beam; + } + } + } + + // Store the offset on the acoustic likelihoods that we're applying. + // Could just do cost_offsets_.push_back(cost_offset), but we + // do it this way as it's more robust to future code changes. + cost_offsets_.resize(frame + 1, 0.0); + cost_offsets_[frame] = cost_offset; + + // the tokens are now owned here, in final_toks, and the hash is empty. + // 'owned' is a complex thing here; the point is we need to call DeleteElem + // on each elem 'e' to let toks_ know we're done with them. + for (Elem *e = final_toks, *e_tail; e != NULL; e = e_tail) { + // loop this way because we delete "e" as we go. + StateId state = e->key; + Token *tok = e->val; + + if (tok->tot_cost <= cur_cutoff) { + for (fst::ArcIterator aiter(*fst_, state); + !aiter.Done(); + aiter.Next()) { + const Arc &arc = aiter.Value(); + if (arc.ilabel != 0) { // propagate.. + if (arc.nextstate == state) continue; + StateId new_bias_state = 0; + BaseFloat ac_cost = cost_offset - + decodable->LogLikelihood(frame, arc.ilabel), + graph_cost = arc.weight.Value(), + cur_cost = tok->tot_cost, + tot_cost = cur_cost + ac_cost + graph_cost; + + + if (bias_lm_) { + float bias_lm_score = 0.0f; + int frm = frame; + int stat = state; + int ilab = arc.ilabel-1; + if (arc.ilabel - 1 == 0) { + new_bias_state = tok->bias_lm_state; + } else { + bias_lm_score = bias_lm_->BiasLmScore(tok->bias_lm_state, + arc.ilabel - 1, new_bias_state); + } + graph_cost -= bias_lm_score; + tot_cost -= bias_lm_score; + } + + if (tot_cost >= next_cutoff) { + continue; + } else if (tot_cost + adaptive_beam < next_cutoff) { + next_cutoff = tot_cost + adaptive_beam; // prune by best current token + } + // Note: the frame indexes into active_toks_ are one-based, + // hence the + 1. + Elem *e_next = FindOrAddToken(arc.nextstate, + frame + 1, tot_cost, tok, NULL, new_bias_state); + // NULL: no change indicator needed + + // Add ForwardLink from tok to next_tok (put on head of list tok->links) + tok->links = new (forward_link_pool_.Allocate()) + ForwardLinkT(e_next->val, arc.ilabel, arc.olabel, graph_cost, + ac_cost, tok->links); + } + } // for all arcs + } + e_tail = e->tail; + toks_.Delete(e); // delete Elem + } + return next_cutoff; +} + +template +std::string LatticeFasterDecoderTpl::GetTokResult(Token *tok) { + if (!tok) { return ""; } + std::string res; + ForwardLinkT* link; + std::vector phn_id; + tok->GetLabelSeq(tok, phn_id); + for (int i = 0; i < phn_id.size(); i++) { + res = bias_lm_->GetPhoneLabel(phn_id[i]) + res; + } + return res; +} + +// static inline +template +void LatticeFasterDecoderTpl::DeleteForwardLinks(Token *tok) { + ForwardLinkT *l = tok->links, *m; + while (l != NULL) { + m = l->next; + forward_link_pool_.Free(l); + l = m; + } + tok->links = NULL; +} + +template +void LatticeFasterDecoderTpl::ProcessNonemitting(BaseFloat cutoff) { + KALDI_ASSERT(!active_toks_.empty()); + int32 frame = static_cast(active_toks_.size()) - 2; + // Note: "frame" is the time-index we just processed, or -1 if + // we are processing the nonemitting transitions before the + // first frame (called from InitDecoding()). + + // Processes nonemitting arcs for one frame. Propagates within toks_. + // Note-- this queue structure is not very optimal as + // it may cause us to process states unnecessarily (e.g. more than once), + // but in the baseline code, turning this vector into a set to fix this + // problem did not improve overall speed. + + KALDI_ASSERT(queue_.empty()); + + if (toks_.GetList() == NULL) { + if (!warned_) { + KALDI_WARN << "Error, no surviving tokens: frame is " << frame; + warned_ = true; + } + } + + for (const Elem *e = toks_.GetList(); e != NULL; e = e->tail) { + StateId state = e->key; + if (fst_->NumInputEpsilons(state) != 0) + queue_.push_back(e); + } + + while (!queue_.empty()) { + const Elem *e = queue_.back(); + queue_.pop_back(); + + StateId state = e->key; + Token *tok = e->val; // would segfault if e is a NULL pointer but this can't happen. + BaseFloat cur_cost = tok->tot_cost; + if (cur_cost >= cutoff) // Don't bother processing successors. + continue; + // If "tok" has any existing forward links, delete them, + // because we're about to regenerate them. This is a kind + // of non-optimality (remember, this is the simple decoder), + // but since most states are emitting it's not a huge issue. + DeleteForwardLinks(tok); // necessary when re-visiting + tok->links = NULL; + for (fst::ArcIterator aiter(*fst_, state); + !aiter.Done(); + aiter.Next()) { + const Arc &arc = aiter.Value(); + if (arc.ilabel == 0) { // propagate nonemitting only... + BaseFloat graph_cost = arc.weight.Value(), + tot_cost = cur_cost + graph_cost; + if (tot_cost < cutoff) { + bool changed; + + Elem *e_new = FindOrAddToken(arc.nextstate, frame + 1, tot_cost, + tok, &changed); + if (bias_lm_ && changed) { + e_new->val->bias_lm_state = tok->bias_lm_state; + } + + tok->links = new (forward_link_pool_.Allocate()) ForwardLinkT( + e_new->val, 0, arc.olabel, graph_cost, 0, tok->links); + + // "changed" tells us whether the new token has a different + // cost from before, or is new [if so, add into queue]. + if (changed && fst_->NumInputEpsilons(arc.nextstate) != 0) + queue_.push_back(e_new); + } + } + } // for all arcs + } // while queue not empty +} + + +template +void LatticeFasterDecoderTpl::DeleteElems(Elem *list) { + for (Elem *e = list, *e_tail; e != NULL; e = e_tail) { + e_tail = e->tail; + toks_.Delete(e); + } +} + +template +void LatticeFasterDecoderTpl::ClearActiveTokens() { // a cleanup routine, at utt end/begin + for (size_t i = 0; i < active_toks_.size(); i++) { + // Delete all tokens alive on this frame, and any forward + // links they may have. + for (Token *tok = active_toks_[i].toks; tok != NULL; ) { + DeleteForwardLinks(tok); + Token *next_tok = tok->next; + token_pool_.Free(tok); + num_toks_--; + tok = next_tok; + } + } + active_toks_.clear(); + KALDI_ASSERT(num_toks_ == 0); +} + +// static +template +void LatticeFasterDecoderTpl::TopSortTokens( + Token *tok_list, std::vector *topsorted_list) { + unordered_map token2pos; + typedef typename unordered_map::iterator IterType; + int32 num_toks = 0; + for (Token *tok = tok_list; tok != NULL; tok = tok->next) + num_toks++; + int32 cur_pos = 0; + // We assign the tokens numbers num_toks - 1, ... , 2, 1, 0. + // This is likely to be in closer to topological order than + // if we had given them ascending order, because of the way + // new tokens are put at the front of the list. + for (Token *tok = tok_list; tok != NULL; tok = tok->next) + token2pos[tok] = num_toks - ++cur_pos; + + unordered_set reprocess; + + for (IterType iter = token2pos.begin(); iter != token2pos.end(); ++iter) { + Token *tok = iter->first; + int32 pos = iter->second; + for (ForwardLinkT *link = tok->links; link != NULL; link = link->next) { + if (link->ilabel == 0) { + // We only need to consider epsilon links, since non-epsilon links + // transition between frames and this function only needs to sort a list + // of tokens from a single frame. + IterType following_iter = token2pos.find(link->next_tok); + if (following_iter != token2pos.end()) { // another token on this frame, + // so must consider it. + int32 next_pos = following_iter->second; + if (next_pos < pos) { // reassign the position of the next Token. + following_iter->second = cur_pos++; + reprocess.insert(link->next_tok); + } + } + } + } + // In case we had previously assigned this token to be reprocessed, we can + // erase it from that set because it's "happy now" (we just processed it). + reprocess.erase(tok); + } + + size_t max_loop = 1000000, loop_count; // max_loop is to detect epsilon cycles. + for (loop_count = 0; + !reprocess.empty() && loop_count < max_loop; ++loop_count) { + std::vector reprocess_vec; + for (typename unordered_set::iterator iter = reprocess.begin(); + iter != reprocess.end(); ++iter) + reprocess_vec.push_back(*iter); + reprocess.clear(); + for (typename std::vector::iterator iter = reprocess_vec.begin(); + iter != reprocess_vec.end(); ++iter) { + Token *tok = *iter; + int32 pos = token2pos[tok]; + // Repeat the processing we did above (for comments, see above). + for (ForwardLinkT *link = tok->links; link != NULL; link = link->next) { + if (link->ilabel == 0) { + IterType following_iter = token2pos.find(link->next_tok); + if (following_iter != token2pos.end()) { + int32 next_pos = following_iter->second; + if (next_pos < pos) { + following_iter->second = cur_pos++; + reprocess.insert(link->next_tok); + } + } + } + } + } + } + KALDI_ASSERT(loop_count < max_loop && "Epsilon loops exist in your decoding " + "graph (this is not allowed!)"); + + topsorted_list->clear(); + topsorted_list->resize(cur_pos, NULL); // create a list with NULLs in between. + for (IterType iter = token2pos.begin(); iter != token2pos.end(); ++iter) + (*topsorted_list)[iter->second] = iter->first; +} + +// Instantiate the template for the combination of token types and FST types +// that we'll need. +template class LatticeFasterDecoderTpl, decoder::StdToken>; +template class LatticeFasterDecoderTpl, decoder::StdToken >; +template class LatticeFasterDecoderTpl, decoder::StdToken >; + +//template class LatticeFasterDecoderTpl; +//template class LatticeFasterDecoderTpl; + +template class LatticeFasterDecoderTpl , decoder::BackpointerToken>; +template class LatticeFasterDecoderTpl, decoder::BackpointerToken >; +template class LatticeFasterDecoderTpl, decoder::BackpointerToken >; +//template class LatticeFasterDecoderTpl; +//template class LatticeFasterDecoderTpl; + + +} // end namespace kaldi. diff --git a/runtime/onnxruntime/third_party/kaldi/decoder/lattice-faster-decoder.h b/runtime/onnxruntime/third_party/kaldi/decoder/lattice-faster-decoder.h new file mode 100644 index 000000000..b752e1783 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/decoder/lattice-faster-decoder.h @@ -0,0 +1,597 @@ +// decoder/lattice-faster-decoder.h + +// Copyright 2009-2013 Microsoft Corporation; Mirko Hannemann; +// 2013-2014 Johns Hopkins University (Author: Daniel Povey) +// 2014 Guoguo Chen +// 2018 Zhehuai Chen + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#ifndef KALDI_DECODER_LATTICE_FASTER_DECODER_H_ +#define KALDI_DECODER_LATTICE_FASTER_DECODER_H_ + +//#include "decoder/grammar-fst.h" +#include "fst/fstlib.h" +#include "fst/memory.h" +#include "fstext/fstext-lib.h" +#include "itf/decodable-itf.h" +#include "lat/determinize-lattice-pruned.h" +#include "lat/kaldi-lattice.h" +#include "util/hash-list.h" +#include "util/stl-utils.h" +#include "bias-lm.h" + +namespace kaldi { + +struct LatticeFasterDecoderConfig { + BaseFloat beam; + int32 max_active; + int32 min_active; + BaseFloat lattice_beam; + int32 prune_interval; + bool determinize_lattice; // not inspected by this class... used in + // command-line program. + BaseFloat beam_delta; + BaseFloat hash_ratio; + // Note: we don't make prune_scale configurable on the command line, it's not + // a very important parameter. It affects the algorithm that prunes the + // tokens as we go. + BaseFloat prune_scale; + + // Number of elements in the block for Token and ForwardLink memory + // pool allocation. + int32 memory_pool_tokens_block_size; + int32 memory_pool_links_block_size; + + // Most of the options inside det_opts are not actually queried by the + // LatticeFasterDecoder class itself, but by the code that calls it, for + // example in the function DecodeUtteranceLatticeFaster. + fst::DeterminizeLatticePhonePrunedOptions det_opts; + + LatticeFasterDecoderConfig(float glob_beam, float lat_beam) + : beam(glob_beam), + max_active(std::numeric_limits::max()), + min_active(200), + lattice_beam(lat_beam), + prune_interval(25), + determinize_lattice(true), + beam_delta(0.5), + hash_ratio(2.0), + prune_scale(0.1), + memory_pool_tokens_block_size(1 << 8), + memory_pool_links_block_size(1 << 8) {} + + LatticeFasterDecoderConfig() + : beam(3.0), + max_active(std::numeric_limits::max()), + min_active(200), + lattice_beam(3.0), + prune_interval(25), + determinize_lattice(true), + beam_delta(0.5), + hash_ratio(2.0), + prune_scale(0.1), + memory_pool_tokens_block_size(1 << 8), + memory_pool_links_block_size(1 << 8) {} + void Register(OptionsItf *opts) { + det_opts.Register(opts); + opts->Register("beam", &beam, "Decoding beam. Larger->slower, more accurate."); + opts->Register("max-active", &max_active, "Decoder max active states. Larger->slower; " + "more accurate"); + opts->Register("min-active", &min_active, "Decoder minimum #active states."); + opts->Register("lattice-beam", &lattice_beam, "Lattice generation beam. Larger->slower, " + "and deeper lattices"); + opts->Register("prune-interval", &prune_interval, "Interval (in frames) at " + "which to prune tokens"); + opts->Register("determinize-lattice", &determinize_lattice, "If true, " + "determinize the lattice (lattice-determinization, keeping only " + "best pdf-sequence for each word-sequence)."); + opts->Register("beam-delta", &beam_delta, "Increment used in decoding-- this " + "parameter is obscure and relates to a speedup in the way the " + "max-active constraint is applied. Larger is more accurate."); + opts->Register("hash-ratio", &hash_ratio, "Setting used in decoder to " + "control hash behavior"); + opts->Register("memory-pool-tokens-block-size", &memory_pool_tokens_block_size, + "Memory pool block size suggestion for storing tokens (in elements). " + "Smaller uses less memory but increases cache misses."); + opts->Register("memory-pool-links-block-size", &memory_pool_links_block_size, + "Memory pool block size suggestion for storing links (in elements). " + "Smaller uses less memory but increases cache misses."); + } + void Check() const { + KALDI_ASSERT(beam > 0.0 && max_active > 1 && lattice_beam > 0.0 + && min_active <= max_active + && prune_interval > 0 && beam_delta > 0.0 && hash_ratio >= 1.0 + && prune_scale > 0.0 && prune_scale < 1.0); + } +}; + +namespace decoder { +// We will template the decoder on the token type as well as the FST type; this +// is a mechanism so that we can use the same underlying decoder code for +// versions of the decoder that support quickly getting the best path +// (LatticeFasterOnlineDecoder, see lattice-faster-online-decoder.h) and also +// those that do not (LatticeFasterDecoder). + + +// ForwardLinks are the links from a token to a token on the next frame. +// or sometimes on the current frame (for input-epsilon links). +template +struct ForwardLink { + using Label = fst::StdArc::Label; + + Token *next_tok; // the next token [or NULL if represents final-state] + Label ilabel; // ilabel on arc + Label olabel; // olabel on arc + BaseFloat graph_cost; // graph cost of traversing arc (contains LM, etc.) + BaseFloat acoustic_cost; // acoustic cost (pre-scaled) of traversing arc + ForwardLink *next; // next in singly-linked list of forward arcs (arcs + // in the state-level lattice) from a token. + inline ForwardLink(Token *next_tok, Label ilabel, Label olabel, + BaseFloat graph_cost, BaseFloat acoustic_cost, + ForwardLink *next): + next_tok(next_tok), ilabel(ilabel), olabel(olabel), + graph_cost(graph_cost), acoustic_cost(acoustic_cost), + next(next) { } +}; + + +struct StdToken { + using ForwardLinkT = ForwardLink; + using Token = StdToken; + + // Standard token type for LatticeFasterDecoder. Each active HCLG + // (decoding-graph) state on each frame has one token. + + // tot_cost is the total (LM + acoustic) cost from the beginning of the + // utterance up to this point. (but see cost_offset_, which is subtracted + // to keep it in a good numerical range). + BaseFloat tot_cost; + + // exta_cost is >= 0. After calling PruneForwardLinks, this equals the + // minimum difference between the cost of the best path that this link is a + // part of, and the cost of the absolute best path, under the assumption that + // any of the currently active states at the decoding front may eventually + // succeed (e.g. if you were to take the currently active states one by one + // and compute this difference, and then take the minimum). + BaseFloat extra_cost; + // 'links' is the head of singly-linked list of ForwardLinks, which is what we + // use for lattice generation. + ForwardLinkT *links; + + //'next' is the next in the singly-linked list of tokens for this frame. + Token *next; + + // bias_lm_state is used to record the state of tokens in the bias lm network + LatticeArc::StateId bias_lm_state; + + // This function does nothing and should be optimized out; it's needed + // so we can share the regular LatticeFasterDecoderTpl code and the code + // for LatticeFasterOnlineDecoder that supports fast traceback. + inline void SetBackpointer (Token *backpointer) { } + + // This constructor just ignores the 'backpointer' argument. That argument is + // needed so that we can use the same decoder code for LatticeFasterDecoderTpl + // and LatticeFasterOnlineDecoderTpl (which needs backpointers to support a + // fast way to obtain the best path). + inline StdToken(BaseFloat tot_cost, BaseFloat extra_cost, ForwardLinkT *links, + Token *next, Token *backpointer): + tot_cost(tot_cost), extra_cost(extra_cost), links(links), next(next), bias_lm_state(0) { } + + inline void GetLabelSeq(Token *tok, std::vector &phn_id) {} +}; + +struct BackpointerToken { + using ForwardLinkT = ForwardLink; + using Token = BackpointerToken; + + // BackpointerToken is like Token but also + // Standard token type for LatticeFasterDecoder. Each active HCLG + // (decoding-graph) state on each frame has one token. + + // tot_cost is the total (LM + acoustic) cost from the beginning of the + // utterance up to this point. (but see cost_offset_, which is subtracted + // to keep it in a good numerical range). + BaseFloat tot_cost; + + // exta_cost is >= 0. After calling PruneForwardLinks, this equals + // the minimum difference between the cost of the best path, and the cost of + // this is on, and the cost of the absolute best path, under the assumption + // that any of the currently active states at the decoding front may + // eventually succeed (e.g. if you were to take the currently active states + // one by one and compute this difference, and then take the minimum). + BaseFloat extra_cost; + + // 'links' is the head of singly-linked list of ForwardLinks, which is what we + // use for lattice generation. + ForwardLinkT *links; + + //'next' is the next in the singly-linked list of tokens for this frame. + BackpointerToken *next; + + // Best preceding BackpointerToken (could be a on this frame, connected to + // this via an epsilon transition, or on a previous frame). This is only + // required for an efficient GetBestPath function in + // LatticeFasterOnlineDecoderTpl; it plays no part in the lattice generation + // (the "links" list is what stores the forward links, for that). + Token *backpointer; + + // bias_lm_state is used to record the state of tokens in the bias lm network + LatticeArc::StateId bias_lm_state; + + inline void SetBackpointer (Token *backpointer) { + this->backpointer = backpointer; + } + + inline BackpointerToken(BaseFloat tot_cost, BaseFloat extra_cost, ForwardLinkT *links, + Token *next, Token *backpointer): + tot_cost(tot_cost), extra_cost(extra_cost), links(links), next(next), + backpointer(backpointer), bias_lm_state(0) { } + + inline void GetLabelSeq(Token *token, std::vector &phn_id) { + ForwardLinkT* link; + Token *tok = token; + while (tok && tok->backpointer) { + for (link = tok->backpointer->links; link != NULL; link = link->next) { + if (link->next_tok == tok) { + phn_id.push_back(link->ilabel - 1); + break; + } + } + tok = tok->backpointer; + } + } +}; + +} // namespace decoder + + +/** This is the "normal" lattice-generating decoder. + See \ref lattices_generation \ref decoders_faster and \ref decoders_simple + for more information. + + The decoder is templated on the FST type and the token type. The token type + will normally be StdToken, but also may be BackpointerToken which is to support + quick lookup of the current best path (see lattice-faster-online-decoder.h) + + The FST you invoke this decoder which is expected to equal + Fst::Fst, a.k.a. StdFst, or GrammarFst. If you invoke it with + FST == StdFst and it notices that the actual FST type is + fst::VectorFst or fst::ConstFst, the decoder object + will internally cast itself to one that is templated on those more specific + types; this is an optimization for speed. + */ +template +class LatticeFasterDecoderTpl { + public: + using Arc = typename FST::Arc; + using Label = typename Arc::Label; + using StateId = typename Arc::StateId; + using Weight = typename Arc::Weight; + using ForwardLinkT = decoder::ForwardLink; + + // Instantiate this class once for each thing you have to decode. + // This version of the constructor does not take ownership of + // 'fst'. + LatticeFasterDecoderTpl(const FST &fst, + const LatticeFasterDecoderConfig &config); + + // This version of the constructor takes ownership of the fst, and will delete + // it when this object is destroyed. + LatticeFasterDecoderTpl(const LatticeFasterDecoderConfig &config, + FST *fst); + //LatticeFasterDecoderTpl() { } + + void SetOptions(const LatticeFasterDecoderConfig &config) { + config_ = config; + } + + const LatticeFasterDecoderConfig &GetOptions() const { + return config_; + } + + ~LatticeFasterDecoderTpl(); + + /// Decodes until there are no more frames left in the "decodable" object.. + /// note, this may block waiting for input if the "decodable" object blocks. + /// Returns true if any kind of traceback is available (not necessarily from a + /// final state). + bool Decode(DecodableInterface *decodable); + + + /// says whether a final-state was active on the last frame. If it was not, the + /// lattice (or traceback) will end with states that are not final-states. + bool ReachedFinal() const { + return FinalRelativeCost() != std::numeric_limits::infinity(); + } + + /// Outputs an FST corresponding to the single best path through the lattice. + /// Returns true if result is nonempty (using the return status is deprecated, + /// it will become void). If "use_final_probs" is true AND we reached the + /// final-state of the graph then it will include those as final-probs, else + /// it will treat all final-probs as one. Note: this just calls GetRawLattice() + /// and figures out the shortest path. + bool GetBestPath(Lattice *ofst, + bool use_final_probs = true) const; + + /// Outputs an FST corresponding to the raw, state-level + /// tracebacks. Returns true if result is nonempty. + /// If "use_final_probs" is true AND we reached the final-state + /// of the graph then it will include those as final-probs, else + /// it will treat all final-probs as one. + /// The raw lattice will be topologically sorted. + /// + /// See also GetRawLatticePruned in lattice-faster-online-decoder.h, + /// which also supports a pruning beam, in case for some reason + /// you want it pruned tighter than the regular lattice beam. + /// We could put that here in future needed. + bool GetRawLattice(Lattice *ofst, bool use_final_probs = true) const; + + + + /// [Deprecated, users should now use GetRawLattice and determinize it + /// themselves, e.g. using DeterminizeLatticePhonePrunedWrapper]. + /// Outputs an FST corresponding to the lattice-determinized + /// lattice (one path per word sequence). Returns true if result is nonempty. + /// If "use_final_probs" is true AND we reached the final-state of the graph + /// then it will include those as final-probs, else it will treat all + /// final-probs as one. + bool GetLattice(CompactLattice *ofst, + bool use_final_probs = true) const; + + /// InitDecoding initializes the decoding, and should only be used if you + /// intend to call AdvanceDecoding(). If you call Decode(), you don't need to + /// call this. You can also call InitDecoding if you have already decoded an + /// utterance and want to start with a new utterance. + void InitDecoding(); + + /// This will decode until there are no more frames ready in the decodable + /// object. You can keep calling it each time more frames become available. + /// If max_num_frames is specified, it specifies the maximum number of frames + /// the function will decode before returning. + void AdvanceDecoding(DecodableInterface *decodable, + int32 max_num_frames = -1); + + /// This function may be optionally called after AdvanceDecoding(), when you + /// do not plan to decode any further. It does an extra pruning step that + /// will help to prune the lattices output by GetLattice and (particularly) + /// GetRawLattice more completely, particularly toward the end of the + /// utterance. If you call this, you cannot call AdvanceDecoding again (it + /// will fail), and you cannot call GetLattice() and related functions with + /// use_final_probs = false. Used to be called PruneActiveTokensFinal(). + void FinalizeDecoding(); + + /// FinalRelativeCost() serves the same purpose as ReachedFinal(), but gives + /// more information. It returns the difference between the best (final-cost + /// plus cost) of any token on the final frame, and the best cost of any token + /// on the final frame. If it is infinity it means no final-states were + /// present on the final frame. It will usually be nonnegative. If it not + /// too positive (e.g. < 5 is my first guess, but this is not tested) you can + /// take it as a good indication that we reached the final-state with + /// reasonable likelihood. + BaseFloat FinalRelativeCost() const; + + + // Returns the number of frames decoded so far. The value returned changes + // whenever we call ProcessEmitting(). + inline int32 NumFramesDecoded() const { return active_toks_.size() - 1; } + + std::string GetTokResult(Token *tok); + + void SetBiasLm(std::shared_ptr &bias_lm) { + bias_lm_ = bias_lm; + } + + void ClearBiasLm() { + bias_lm_.reset(); + } + + protected: + // we make things protected instead of private, as code in + // LatticeFasterOnlineDecoderTpl, which inherits from this, also uses the + // internals. + + // Deletes the elements of the singly linked list tok->links. + void DeleteForwardLinks(Token *tok); + + // head of per-frame list of Tokens (list is in topological order), + // and something saying whether we ever pruned it using PruneForwardLinks. + struct TokenList { + Token *toks; + bool must_prune_forward_links; + bool must_prune_tokens; + TokenList(): toks(NULL), must_prune_forward_links(true), + must_prune_tokens(true) { } + }; + + using Elem = typename HashList::Elem; + // Equivalent to: + // struct Elem { + // StateId key; + // Token *val; + // Elem *tail; + // }; + + void PossiblyResizeHash(size_t num_toks); + + // FindOrAddToken either locates a token in hash of toks_, or if necessary + // inserts a new, empty token (i.e. with no forward links) for the current + // frame. [note: it's inserted if necessary into hash toks_ and also into the + // singly linked list of tokens active on this frame (whose head is at + // active_toks_[frame]). The frame_plus_one argument is the acoustic frame + // index plus one, which is used to index into the active_toks_ array. + // Returns the Token pointer. Sets "changed" (if non-NULL) to true if the + // token was newly created or the cost changed. + // If Token == StdToken, the 'backpointer' argument has no purpose (and will + // hopefully be optimized out). + inline Elem *FindOrAddToken(StateId state, int32 frame_plus_one, + BaseFloat tot_cost, Token *backpointer, + bool *changed, StateId bias_lm_state = 0); + + // prunes outgoing links for all tokens in active_toks_[frame] + // it's called by PruneActiveTokens + // all links, that have link_extra_cost > lattice_beam are pruned + // delta is the amount by which the extra_costs must change + // before we set *extra_costs_changed = true. + // If delta is larger, we'll tend to go back less far + // toward the beginning of the file. + // extra_costs_changed is set to true if extra_cost was changed for any token + // links_pruned is set to true if any link in any token was pruned + void PruneForwardLinks(int32 frame_plus_one, bool *extra_costs_changed, + bool *links_pruned, + BaseFloat delta); + + // This function computes the final-costs for tokens active on the final + // frame. It outputs to final-costs, if non-NULL, a map from the Token* + // pointer to the final-prob of the corresponding state, for all Tokens + // that correspond to states that have final-probs. This map will be + // empty if there were no final-probs. It outputs to + // final_relative_cost, if non-NULL, the difference between the best + // forward-cost including the final-prob cost, and the best forward-cost + // without including the final-prob cost (this will usually be positive), or + // infinity if there were no final-probs. [c.f. FinalRelativeCost(), which + // outputs this quanitity]. It outputs to final_best_cost, if + // non-NULL, the lowest for any token t active on the final frame, of + // forward-cost[t] + final-cost[t], where final-cost[t] is the final-cost in + // the graph of the state corresponding to token t, or the best of + // forward-cost[t] if there were no final-probs active on the final frame. + // You cannot call this after FinalizeDecoding() has been called; in that + // case you should get the answer from class-member variables. + void ComputeFinalCosts(unordered_map *final_costs, + BaseFloat *final_relative_cost, + BaseFloat *final_best_cost) const; + + // PruneForwardLinksFinal is a version of PruneForwardLinks that we call + // on the final frame. If there are final tokens active, it uses + // the final-probs for pruning, otherwise it treats all tokens as final. + void PruneForwardLinksFinal(); + + // Prune away any tokens on this frame that have no forward links. + // [we don't do this in PruneForwardLinks because it would give us + // a problem with dangling pointers]. + // It's called by PruneActiveTokens if any forward links have been pruned + void PruneTokensForFrame(int32 frame_plus_one); + + + // Go backwards through still-alive tokens, pruning them if the + // forward+backward cost is more than lat_beam away from the best path. It's + // possible to prove that this is "correct" in the sense that we won't lose + // anything outside of lat_beam, regardless of what happens in the future. + // delta controls when it considers a cost to have changed enough to continue + // going backward and propagating the change. larger delta -> will recurse + // less far. + void PruneActiveTokens(BaseFloat delta); + + /// Gets the weight cutoff. Also counts the active tokens. + BaseFloat GetCutoff(Elem *list_head, size_t *tok_count, + BaseFloat *adaptive_beam, Elem **best_elem); + + /// Processes emitting arcs for one frame. Propagates from prev_toks_ to + /// cur_toks_. Returns the cost cutoff for subsequent ProcessNonemitting() to + /// use. + BaseFloat ProcessEmitting(DecodableInterface *decodable); + + /// Processes nonemitting (epsilon) arcs for one frame. Called after + /// ProcessEmitting() on each frame. The cost cutoff is computed by the + /// preceding ProcessEmitting(). + void ProcessNonemitting(BaseFloat cost_cutoff); + + // HashList defined in ../util/hash-list.h. It actually allows us to maintain + // more than one list (e.g. for current and previous frames), but only one of + // them at a time can be indexed by StateId. It is indexed by frame-index + // plus one, where the frame-index is zero-based, as used in decodable object. + // That is, the emitting probs of frame t are accounted for in tokens at + // toks_[t+1]. The zeroth frame is for nonemitting transition at the start of + // the graph. + HashList toks_; + + std::vector active_toks_; // Lists of tokens, indexed by + // frame (members of TokenList are toks, must_prune_forward_links, + // must_prune_tokens). + std::vector queue_; // temp variable used in ProcessNonemitting, + std::vector tmp_array_; // used in GetCutoff. + + // fst_ is a pointer to the FST we are decoding from. + const FST *fst_; + // delete_fst_ is true if the pointer fst_ needs to be deleted when this + // object is destroyed. + bool delete_fst_; + + std::vector cost_offsets_; // This contains, for each + // frame, an offset that was added to the acoustic log-likelihoods on that + // frame in order to keep everything in a nice dynamic range i.e. close to + // zero, to reduce roundoff errors. + LatticeFasterDecoderConfig config_; + int32 num_toks_; // current total #toks allocated... + bool warned_; + + /// decoding_finalized_ is true if someone called FinalizeDecoding(). [note, + /// calling this is optional]. If true, it's forbidden to decode more. Also, + /// if this is set, then the output of ComputeFinalCosts() is in the next + /// three variables. The reason we need to do this is that after + /// FinalizeDecoding() calls PruneTokensForFrame() for the final frame, some + /// of the tokens on the last frame are freed, so we free the list from toks_ + /// to avoid having dangling pointers hanging around. + bool decoding_finalized_; + /// For the meaning of the next 3 variables, see the comment for + /// decoding_finalized_ above., and ComputeFinalCosts(). + unordered_map final_costs_; + BaseFloat final_relative_cost_; + BaseFloat final_best_cost_; + + // Memory pools for storing tokens and forward links. + // We use it to decrease the work put on allocator and to move some of data + // together. Too small block sizes will result in more work to allocator but + // bigger ones increase the memory usage. + fst::MemoryPool token_pool_; + fst::MemoryPool forward_link_pool_; + + // There are various cleanup tasks... the toks_ structure contains + // singly linked lists of Token pointers, where Elem is the list type. + // It also indexes them in a hash, indexed by state (this hash is only + // maintained for the most recent frame). toks_.Clear() + // deletes them from the hash and returns the list of Elems. The + // function DeleteElems calls toks_.Delete(elem) for each elem in + // the list, which returns ownership of the Elem to the toks_ structure + // for reuse, but does not delete the Token pointer. The Token pointers + // are reference-counted and are ultimately deleted in PruneTokensForFrame, + // but are also linked together on each frame by their own linked-list, + // using the "next" pointer. We delete them manually. + void DeleteElems(Elem *list); + + // This function takes a singly linked list of tokens for a single frame, and + // outputs a list of them in topological order (it will crash if no such order + // can be found, which will typically be due to decoding graphs with epsilon + // cycles, which are not allowed). Note: the output list may contain NULLs, + // which the caller should pass over; it just happens to be more efficient for + // the algorithm to output a list that contains NULLs. + static void TopSortTokens(Token *tok_list, + std::vector *topsorted_list); + + void ClearActiveTokens(); + + KALDI_DISALLOW_COPY_AND_ASSIGN(LatticeFasterDecoderTpl); + + std::shared_ptr bias_lm_ = nullptr; +}; + +typedef LatticeFasterDecoderTpl LatticeFasterDecoder; + + + +} // end namespace kaldi. + +#endif diff --git a/runtime/onnxruntime/third_party/kaldi/decoder/lattice-faster-online-decoder.cc b/runtime/onnxruntime/third_party/kaldi/decoder/lattice-faster-online-decoder.cc new file mode 100644 index 000000000..b5261503c --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/decoder/lattice-faster-online-decoder.cc @@ -0,0 +1,285 @@ +// decoder/lattice-faster-online-decoder.cc + +// Copyright 2009-2012 Microsoft Corporation Mirko Hannemann +// 2013-2014 Johns Hopkins University (Author: Daniel Povey) +// 2014 Guoguo Chen +// 2014 IMSL, PKU-HKUST (author: Wei Shi) +// 2018 Zhehuai Chen + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +// see note at the top of lattice-faster-decoder.cc, about how to maintain this +// file in sync with lattice-faster-decoder.cc + +#include "decoder/lattice-faster-online-decoder.h" +#include "lat/lattice-functions.h" + +namespace kaldi { + +template +bool LatticeFasterOnlineDecoderTpl::TestGetBestPath( + bool use_final_probs) const { + Lattice lat1; + { + Lattice raw_lat; + this->GetRawLattice(&raw_lat, use_final_probs); + ShortestPath(raw_lat, &lat1); + } + Lattice lat2; + GetBestPath(&lat2, use_final_probs); + BaseFloat delta = 0.1; + int32 num_paths = 1; + if (!fst::RandEquivalent(lat1, lat2, num_paths, delta, rand())) { + KALDI_WARN << "Best-path test failed"; + return false; + } else { + return true; + } +} + + +// Outputs an FST corresponding to the single best path through the lattice. +template +bool LatticeFasterOnlineDecoderTpl::GetBestPath(Lattice *olat, + bool use_final_probs) const { + olat->DeleteStates(); + BaseFloat final_graph_cost; + BestPathIterator iter = BestPathEnd(use_final_probs, &final_graph_cost); + if (iter.Done()) + return false; // would have printed warning. + StateId state = olat->AddState(); + olat->SetFinal(state, LatticeWeight(final_graph_cost, 0.0)); + while (!iter.Done()) { + LatticeArc arc; + iter = TraceBackBestPath(iter, &arc); + arc.nextstate = state; + StateId new_state = olat->AddState(); + olat->AddArc(new_state, arc); + state = new_state; + } + olat->SetStart(state); + return true; +} + +template +typename LatticeFasterOnlineDecoderTpl::BestPathIterator LatticeFasterOnlineDecoderTpl::BestPathEnd( + bool use_final_probs, + BaseFloat *final_cost_out) const { + if (this->decoding_finalized_ && !use_final_probs) + KALDI_ERR << "You cannot call FinalizeDecoding() and then call " + << "BestPathEnd() with use_final_probs == false"; + KALDI_ASSERT(this->NumFramesDecoded() > 0 && + "You cannot call BestPathEnd if no frames were decoded."); + + unordered_map final_costs_local; + + const unordered_map &final_costs = + (this->decoding_finalized_ ? this->final_costs_ :final_costs_local); + if (!this->decoding_finalized_ && use_final_probs) + this->ComputeFinalCosts(&final_costs_local, NULL, NULL); + + // Singly linked list of tokens on last frame (access list through "next" + // pointer). + BaseFloat best_cost = std::numeric_limits::infinity(); + BaseFloat best_final_cost = 0; + Token *best_tok = NULL; + for (Token *tok = this->active_toks_.back().toks; + tok != NULL; tok = tok->next) { + BaseFloat cost = tok->tot_cost, final_cost = 0.0; + if (use_final_probs && !final_costs.empty()) { + // if we are instructed to use final-probs, and any final tokens were + // active on final frame, include the final-prob in the cost of the token. + typename unordered_map::const_iterator + iter = final_costs.find(tok); + if (iter != final_costs.end()) { + final_cost = iter->second; + cost += final_cost; + } else { + cost = std::numeric_limits::infinity(); + } + } + if (cost < best_cost) { + best_cost = cost; + best_tok = tok; + best_final_cost = final_cost; + } + } + if (best_tok == NULL) { // this should not happen, and is likely a code error or + // caused by infinities in likelihoods, but I'm not making + // it a fatal error for now. + KALDI_WARN << "No final token found."; + } + if (final_cost_out) + *final_cost_out = best_final_cost; + return BestPathIterator(best_tok, this->NumFramesDecoded() - 1); +} + + +template +typename LatticeFasterOnlineDecoderTpl::BestPathIterator LatticeFasterOnlineDecoderTpl::TraceBackBestPath( + BestPathIterator iter, LatticeArc *oarc) const { + KALDI_ASSERT(!iter.Done() && oarc != NULL); + Token *tok = static_cast(iter.tok); + int32 cur_t = iter.frame, step_t = 0; + if (tok->backpointer != NULL) { + // retrieve the correct forward link(with the best link cost) + BaseFloat best_cost = std::numeric_limits::infinity(); + ForwardLinkT *link; + for (link = tok->backpointer->links; + link != NULL; link = link->next) { + if (link->next_tok == tok) { // this is a link to "tok" + BaseFloat graph_cost = link->graph_cost, + acoustic_cost = link->acoustic_cost; + BaseFloat cost = graph_cost + acoustic_cost; + if (cost < best_cost) { + oarc->ilabel = link->ilabel; + oarc->olabel = link->olabel; + if (link->ilabel != 0) { + KALDI_ASSERT(static_cast(cur_t) < this->cost_offsets_.size()); + acoustic_cost -= this->cost_offsets_[cur_t]; + step_t = -1; + } else { + step_t = 0; + } + oarc->weight = LatticeWeight(graph_cost, acoustic_cost); + best_cost = cost; + } + } + } + if (link == NULL && + best_cost == std::numeric_limits::infinity()) { // Did not find correct link. + KALDI_ERR << "Error tracing best-path back (likely " + << "bug in token-pruning algorithm)"; + } + } else { + oarc->ilabel = 0; + oarc->olabel = 0; + oarc->weight = LatticeWeight::One(); // zero costs. + } + return BestPathIterator(tok->backpointer, cur_t + step_t); +} + +template +bool LatticeFasterOnlineDecoderTpl::GetRawLatticePruned( + Lattice *ofst, + bool use_final_probs, + BaseFloat beam) const { + typedef LatticeArc Arc; + typedef Arc::StateId StateId; + typedef Arc::Weight Weight; + typedef Arc::Label Label; + + // Note: you can't use the old interface (Decode()) if you want to + // get the lattice with use_final_probs = false. You'd have to do + // InitDecoding() and then AdvanceDecoding(). + if (this->decoding_finalized_ && !use_final_probs) + KALDI_ERR << "You cannot call FinalizeDecoding() and then call " + << "GetRawLattice() with use_final_probs == false"; + + unordered_map final_costs_local; + + const unordered_map &final_costs = + (this->decoding_finalized_ ? this->final_costs_ : final_costs_local); + if (!this->decoding_finalized_ && use_final_probs) + this->ComputeFinalCosts(&final_costs_local, NULL, NULL); + + ofst->DeleteStates(); + // num-frames plus one (since frames are one-based, and we have + // an extra frame for the start-state). + int32 num_frames = this->active_toks_.size() - 1; + KALDI_ASSERT(num_frames > 0); + for (int32 f = 0; f <= num_frames; f++) { + if (this->active_toks_[f].toks == NULL) { + KALDI_WARN << "No tokens active on frame " << f + << ": not producing lattice.\n"; + return false; + } + } + unordered_map tok_map; + std::queue > tok_queue; + // First initialize the queue and states. Put the initial state on the queue; + // this is the last token in the list active_toks_[0].toks. + for (Token *tok = this->active_toks_[0].toks; + tok != NULL; tok = tok->next) { + if (tok->next == NULL) { + tok_map[tok] = ofst->AddState(); + ofst->SetStart(tok_map[tok]); + std::pair tok_pair(tok, 0); // #frame = 0 + tok_queue.push(tok_pair); + } + } + + // Next create states for "good" tokens + while (!tok_queue.empty()) { + std::pair cur_tok_pair = tok_queue.front(); + tok_queue.pop(); + Token *cur_tok = cur_tok_pair.first; + int32 cur_frame = cur_tok_pair.second; + KALDI_ASSERT(cur_frame >= 0 && + cur_frame <= this->cost_offsets_.size()); + + typename unordered_map::const_iterator iter = + tok_map.find(cur_tok); + KALDI_ASSERT(iter != tok_map.end()); + StateId cur_state = iter->second; + + for (ForwardLinkT *l = cur_tok->links; + l != NULL; + l = l->next) { + Token *next_tok = l->next_tok; + if (next_tok->extra_cost < beam) { + // so both the current and the next token are good; create the arc + int32 next_frame = l->ilabel == 0 ? cur_frame : cur_frame + 1; + StateId nextstate; + if (tok_map.find(next_tok) == tok_map.end()) { + nextstate = tok_map[next_tok] = ofst->AddState(); + tok_queue.push(std::pair(next_tok, next_frame)); + } else { + nextstate = tok_map[next_tok]; + } + BaseFloat cost_offset = (l->ilabel != 0 ? + this->cost_offsets_[cur_frame] : 0); + Arc arc(l->ilabel, l->olabel, + Weight(l->graph_cost, l->acoustic_cost - cost_offset), + nextstate); + ofst->AddArc(cur_state, arc); + } + } + if (cur_frame == num_frames) { + if (use_final_probs && !final_costs.empty()) { + typename unordered_map::const_iterator iter = + final_costs.find(cur_tok); + if (iter != final_costs.end()) + ofst->SetFinal(cur_state, LatticeWeight(iter->second, 0)); + } else { + ofst->SetFinal(cur_state, LatticeWeight::One()); + } + } + } + return (ofst->NumStates() != 0); +} + + + +// Instantiate the template for the FST types that we'll need. +template class LatticeFasterOnlineDecoderTpl >; +template class LatticeFasterOnlineDecoderTpl >; +template class LatticeFasterOnlineDecoderTpl >; +//template class LatticeFasterOnlineDecoderTpl; +//template class LatticeFasterOnlineDecoderTpl; + + +} // end namespace kaldi. diff --git a/runtime/onnxruntime/third_party/kaldi/decoder/lattice-faster-online-decoder.h b/runtime/onnxruntime/third_party/kaldi/decoder/lattice-faster-online-decoder.h new file mode 100644 index 000000000..4040d8f2e --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/decoder/lattice-faster-online-decoder.h @@ -0,0 +1,146 @@ +// decoder/lattice-faster-online-decoder.h + +// Copyright 2009-2013 Microsoft Corporation; Mirko Hannemann; +// 2013-2014 Johns Hopkins University (Author: Daniel Povey) +// 2014 Guoguo Chen +// 2018 Zhehuai Chen + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +// see note at the top of lattice-faster-decoder.h, about how to maintain this +// file in sync with lattice-faster-decoder.h + + +#ifndef KALDI_DECODER_LATTICE_FASTER_ONLINE_DECODER_H_ +#define KALDI_DECODER_LATTICE_FASTER_ONLINE_DECODER_H_ + +#include "util/stl-utils.h" +#include "util/hash-list.h" +#include "fst/fstlib.h" +#include "itf/decodable-itf.h" +#include "fstext/fstext-lib.h" +#include "lat/determinize-lattice-pruned.h" +#include "lat/kaldi-lattice.h" +#include "decoder/lattice-faster-decoder.h" + +namespace kaldi { + + + +/** LatticeFasterOnlineDecoderTpl is as LatticeFasterDecoderTpl but also + supports an efficient way to get the best path (see the function + BestPathEnd()), which is useful in endpointing and in situations where you + might want to frequently access the best path. + + This is only templated on the FST type, since the Token type is required to + be BackpointerToken. Actually it only makes sense to instantiate + LatticeFasterDecoderTpl with Token == BackpointerToken if you do so indirectly via + this child class. + */ +template +class LatticeFasterOnlineDecoderTpl: + public LatticeFasterDecoderTpl { + public: + using Arc = typename FST::Arc; + using Label = typename Arc::Label; + using StateId = typename Arc::StateId; + using Weight = typename Arc::Weight; + using Token = decoder::BackpointerToken; + using ForwardLinkT = decoder::ForwardLink; + + // Instantiate this class once for each thing you have to decode. + // This version of the constructor does not take ownership of + // 'fst'. + LatticeFasterOnlineDecoderTpl(const FST &fst, + const LatticeFasterDecoderConfig &config): + LatticeFasterDecoderTpl(fst, config) { } + + // This version of the initializer takes ownership of 'fst', and will delete + // it when this object is destroyed. + LatticeFasterOnlineDecoderTpl(const LatticeFasterDecoderConfig &config, FST *fst): + LatticeFasterDecoderTpl(config, fst) { } + //LatticeFasterOnlineDecoderTpl() {} + + struct BestPathIterator { + void *tok; + int32 frame; + // note, "frame" is the frame-index of the frame you'll get the + // transition-id for next time, if you call TraceBackBestPath on this + // iterator (assuming it's not an epsilon transition). Note that this + // is one less than you might reasonably expect, e.g. it's -1 for + // the nonemitting transitions before the first frame. + BestPathIterator(void *t, int32 f): tok(t), frame(f) { } + bool Done() const { return tok == NULL; } + }; + + + /// Outputs an FST corresponding to the single best path through the lattice. + /// This is quite efficient because it doesn't get the entire raw lattice and find + /// the best path through it; instead, it uses the BestPathEnd and BestPathIterator + /// so it basically traces it back through the lattice. + /// Returns true if result is nonempty (using the return status is deprecated, + /// it will become void). If "use_final_probs" is true AND we reached the + /// final-state of the graph then it will include those as final-probs, else + /// it will treat all final-probs as one. + bool GetBestPath(Lattice *ofst, + bool use_final_probs = true) const; + + + /// This function does a self-test of GetBestPath(). Returns true on + /// success; returns false and prints a warning on failure. + bool TestGetBestPath(bool use_final_probs = true) const; + + + /// This function returns an iterator that can be used to trace back + /// the best path. If use_final_probs == true and at least one final state + /// survived till the end, it will use the final-probs in working out the best + /// final Token, and will output the final cost to *final_cost (if non-NULL), + /// else it will use only the forward likelihood, and will put zero in + /// *final_cost (if non-NULL). + /// Requires that NumFramesDecoded() > 0. + BestPathIterator BestPathEnd(bool use_final_probs, + BaseFloat *final_cost = NULL) const; + + + /// This function can be used in conjunction with BestPathEnd() to trace back + /// the best path one link at a time (e.g. this can be useful in endpoint + /// detection). By "link" we mean a link in the graph; not all links cross + /// frame boundaries, but each time you see a nonzero ilabel you can interpret + /// that as a frame. The return value is the updated iterator. It outputs + /// the ilabel and olabel, and the (graph and acoustic) weight to the "arc" pointer, + /// while leaving its "nextstate" variable unchanged. + BestPathIterator TraceBackBestPath( + BestPathIterator iter, LatticeArc *arc) const; + + + /// Behaves the same as GetRawLattice but only processes tokens whose + /// extra_cost is smaller than the best-cost plus the specified beam. + /// It is only worthwhile to call this function if beam is less than + /// the lattice_beam specified in the config; otherwise, it would + /// return essentially the same thing as GetRawLattice, but more slowly. + bool GetRawLatticePruned(Lattice *ofst, + bool use_final_probs, + BaseFloat beam) const; + + KALDI_DISALLOW_COPY_AND_ASSIGN(LatticeFasterOnlineDecoderTpl); +}; + +typedef LatticeFasterOnlineDecoderTpl LatticeFasterOnlineDecoder; + + +} // end namespace kaldi. + +#endif diff --git a/runtime/onnxruntime/third_party/kaldi/decoder/lattice-incremental-decoder.cc b/runtime/onnxruntime/third_party/kaldi/decoder/lattice-incremental-decoder.cc new file mode 100644 index 000000000..b02247474 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/decoder/lattice-incremental-decoder.cc @@ -0,0 +1,1730 @@ +// decoder/lattice-incremental-decoder.cc + +// Copyright 2019 Zhehuai Chen, Daniel Povey + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#include "decoder/lattice-incremental-decoder.h" +#include "lat/lattice-functions.h" +#include "base/timer.h" + +namespace kaldi { + +// instantiate this class once for each thing you have to decode. +template +LatticeIncrementalDecoderTpl::LatticeIncrementalDecoderTpl( + const FST &fst, const TransitionInformation &trans_model, + const LatticeIncrementalDecoderConfig &config) + : fst_(&fst), + delete_fst_(false), + num_toks_(0), + config_(config), + determinizer_(trans_model, config) { + config.Check(); + toks_.SetSize(1000); // just so on the first frame we do something reasonable. +} + +template +LatticeIncrementalDecoderTpl::LatticeIncrementalDecoderTpl( + const LatticeIncrementalDecoderConfig &config, FST *fst, + const TransitionInformation &trans_model) + : fst_(fst), + delete_fst_(true), + num_toks_(0), + config_(config), + determinizer_(trans_model, config) { + config.Check(); + toks_.SetSize(1000); // just so on the first frame we do something reasonable. +} + +template +LatticeIncrementalDecoderTpl::~LatticeIncrementalDecoderTpl() { + DeleteElems(toks_.Clear()); + ClearActiveTokens(); + if (delete_fst_) delete fst_; +} + +template +void LatticeIncrementalDecoderTpl::InitDecoding() { + // clean up from last time: + DeleteElems(toks_.Clear()); + cost_offsets_.clear(); + ClearActiveTokens(); + warned_ = false; + num_toks_ = 0; + decoding_finalized_ = false; + final_costs_.clear(); + StateId start_state = fst_->Start(); + KALDI_ASSERT(start_state != fst::kNoStateId); + active_toks_.resize(1); + Token *start_tok = new Token(0.0, 0.0, NULL, NULL, NULL); + active_toks_[0].toks = start_tok; + toks_.Insert(start_state, start_tok); + num_toks_++; + + determinizer_.Init(); + num_frames_in_lattice_ = 0; + token2label_map_.clear(); + next_token_label_ = LatticeIncrementalDeterminizer::kTokenLabelOffset; + ProcessNonemitting(config_.beam); +} + +template +void LatticeIncrementalDecoderTpl::UpdateLatticeDeterminization() { + if (NumFramesDecoded() - num_frames_in_lattice_ < + config_.determinize_max_delay) + return; + + + /* Make sure the token-pruning is active. Note: PruneActiveTokens() has + internal logic that prevents it from doing unnecessary work if you + call it and then immediately call it again. */ + PruneActiveTokens(config_.lattice_beam * config_.prune_scale); + + int32 first = num_frames_in_lattice_ + config_.determinize_min_chunk_size, + last = NumFramesDecoded(), + fewest_tokens = std::numeric_limits::max(), + best_frame = -1; + for (int32 t = last; t >= first; t--) { + /* Make sure PruneActiveTokens() has computed num_toks for all these + frames... */ + KALDI_ASSERT(active_toks_[t].num_toks != -1); + if (active_toks_[t].num_toks < fewest_tokens) { + // <= because we want the latest one in case of ties. + fewest_tokens = active_toks_[t].num_toks; + best_frame = t; + } + } + /* Skip this update if we have too many tokens, determinization will take too long, + postpone it to the next update */ + if (fewest_tokens > config_.determinize_max_active) + return; + + /* OK, determinize the chunk that spans from num_frames_in_lattice_ to + best_frame. */ + bool use_final_probs = false; + GetLattice(best_frame, use_final_probs); + return; +} +// Returns true if any kind of traceback is available (not necessarily from +// a final state). It should only very rarely return false; this indicates +// an unusual search error. +template +bool LatticeIncrementalDecoderTpl::Decode(DecodableInterface *decodable) { + InitDecoding(); + + // We use 1-based indexing for frames in this decoder (if you view it in + // terms of features), but note that the decodable object uses zero-based + // numbering, which we have to correct for when we call it. + + while (!decodable->IsLastFrame(NumFramesDecoded() - 1)) { + if (NumFramesDecoded() % config_.prune_interval == 0) { + PruneActiveTokens(config_.lattice_beam * config_.prune_scale); + } + UpdateLatticeDeterminization(); + + BaseFloat cost_cutoff = ProcessEmitting(decodable); + ProcessNonemitting(cost_cutoff); + } + Timer timer; + FinalizeDecoding(); + bool use_final_probs = true; + GetLattice(NumFramesDecoded(), use_final_probs); + KALDI_VLOG(2) << "Delay time during and after FinalizeDecoding()" + << "(secs): " << timer.Elapsed(); + + // Returns true if we have any kind of traceback available (not necessarily + // to the end state; query ReachedFinal() for that). + return !active_toks_.empty() && active_toks_.back().toks != NULL; +} + + +template +void LatticeIncrementalDecoderTpl::PossiblyResizeHash(size_t num_toks) { + size_t new_sz = + static_cast(static_cast(num_toks) * config_.hash_ratio); + if (new_sz > toks_.Size()) { + toks_.SetSize(new_sz); + } +} + +/* + A note on the definition of extra_cost. + + extra_cost is used in pruning tokens, to save memory. + + extra_cost can be thought of as a beta (backward) cost assuming + we had set the betas on currently-active tokens to all be the negative + of the alphas for those tokens. (So all currently active tokens would + be on (tied) best paths). + + + Define the 'forward cost' of a token as zero for any token on the frame + we're currently decoding; and for other frames, as the shortest-path cost + between that token and a token on the frame we're currently decoding. + (by "currently decoding" I mean the most recently processed frame). + + Then define the extra_cost of a token (always >= 0) as the forward-cost of + the token minus the smallest forward-cost of any token on the same frame. + + We can use the extra_cost to accurately prune away tokens that we know will + never appear in the lattice. If the extra_cost is greater than the desired + lattice beam, the token would provably never appear in the lattice, so we can + prune away the token. + + The advantage of storing the extra_cost rather than the forward-cost, is that + it is less costly to keep the extra_cost up-to-date when we process new frames. + When we process a new frame, *all* the previous frames' forward-costs would change; + but in general the extra_cost will change only for a finite number of frames. + (Actually we don't update all the extra_costs every time we update a frame; we + only do it every 'config_.prune_interval' frames). + */ + +// FindOrAddToken either locates a token in hash of toks_, +// or if necessary inserts a new, empty token (i.e. with no forward links) +// for the current frame. [note: it's inserted if necessary into hash toks_ +// and also into the singly linked list of tokens active on this frame +// (whose head is at active_toks_[frame]). +template +inline Token *LatticeIncrementalDecoderTpl::FindOrAddToken( + StateId state, int32 frame_plus_one, BaseFloat tot_cost, Token *backpointer, + bool *changed) { + // Returns the Token pointer. Sets "changed" (if non-NULL) to true + // if the token was newly created or the cost changed. + KALDI_ASSERT(frame_plus_one < active_toks_.size()); + Token *&toks = active_toks_[frame_plus_one].toks; + Elem *e_found = toks_.Find(state); + if (e_found == NULL) { // no such token presently. + const BaseFloat extra_cost = 0.0; + // tokens on the currently final frame have zero extra_cost + // as any of them could end up + // on the winning path. + Token *new_tok = new Token(tot_cost, extra_cost, NULL, toks, backpointer); + // NULL: no forward links yet + toks = new_tok; + num_toks_++; + toks_.Insert(state, new_tok); + if (changed) *changed = true; + return new_tok; + } else { + Token *tok = e_found->val; // There is an existing Token for this state. + if (tok->tot_cost > tot_cost) { // replace old token + tok->tot_cost = tot_cost; + // SetBackpointer() just does tok->backpointer = backpointer in + // the case where Token == BackpointerToken, else nothing. + tok->SetBackpointer(backpointer); + // we don't allocate a new token, the old stays linked in active_toks_ + // we only replace the tot_cost + // in the current frame, there are no forward links (and no extra_cost) + // only in ProcessNonemitting we have to delete forward links + // in case we visit a state for the second time + // those forward links, that lead to this replaced token before: + // they remain and will hopefully be pruned later (PruneForwardLinks...) + if (changed) *changed = true; + } else { + if (changed) *changed = false; + } + return tok; + } +} + +// prunes outgoing links for all tokens in active_toks_[frame] +// it's called by PruneActiveTokens +// all links, that have link_extra_cost > lattice_beam are pruned +template +void LatticeIncrementalDecoderTpl::PruneForwardLinks( + int32 frame_plus_one, bool *extra_costs_changed, bool *links_pruned, + BaseFloat delta) { + // delta is the amount by which the extra_costs must change + // If delta is larger, we'll tend to go back less far + // toward the beginning of the file. + // extra_costs_changed is set to true if extra_cost was changed for any token + // links_pruned is set to true if any link in any token was pruned + + *extra_costs_changed = false; + *links_pruned = false; + KALDI_ASSERT(frame_plus_one >= 0 && frame_plus_one < active_toks_.size()); + if (active_toks_[frame_plus_one].toks == NULL) { // empty list; should not happen. + if (!warned_) { + KALDI_WARN << "No tokens alive [doing pruning].. warning first " + "time only for each utterance\n"; + warned_ = true; + } + } + + // We have to iterate until there is no more change, because the links + // are not guaranteed to be in topological order. + bool changed = true; // difference new minus old extra cost >= delta ? + while (changed) { + changed = false; + for (Token *tok = active_toks_[frame_plus_one].toks; tok != NULL; + tok = tok->next) { + ForwardLinkT *link, *prev_link = NULL; + // will recompute tok_extra_cost for tok. + BaseFloat tok_extra_cost = std::numeric_limits::infinity(); + // tok_extra_cost is the best (min) of link_extra_cost of outgoing links + for (link = tok->links; link != NULL;) { + // See if we need to excise this link... + Token *next_tok = link->next_tok; + BaseFloat link_extra_cost = + next_tok->extra_cost + + ((tok->tot_cost + link->acoustic_cost + link->graph_cost) - + next_tok->tot_cost); // difference in brackets is >= 0 + // link_exta_cost is the difference in score between the best paths + // through link source state and through link destination state + KALDI_ASSERT(link_extra_cost == link_extra_cost); // check for NaN + if (link_extra_cost > config_.lattice_beam) { // excise link + ForwardLinkT *next_link = link->next; + if (prev_link != NULL) + prev_link->next = next_link; + else + tok->links = next_link; + delete link; + link = next_link; // advance link but leave prev_link the same. + *links_pruned = true; + } else { // keep the link and update the tok_extra_cost if needed. + if (link_extra_cost < 0.0) { // this is just a precaution. + if (link_extra_cost < -0.01) + KALDI_WARN << "Negative extra_cost: " << link_extra_cost; + link_extra_cost = 0.0; + } + if (link_extra_cost < tok_extra_cost) tok_extra_cost = link_extra_cost; + prev_link = link; // move to next link + link = link->next; + } + } // for all outgoing links + if (fabs(tok_extra_cost - tok->extra_cost) > delta) + changed = true; // difference new minus old is bigger than delta + tok->extra_cost = tok_extra_cost; + // will be +infinity or <= lattice_beam_. + // infinity indicates, that no forward link survived pruning + } // for all Token on active_toks_[frame] + if (changed) *extra_costs_changed = true; + + // Note: it's theoretically possible that aggressive compiler + // optimizations could cause an infinite loop here for small delta and + // high-dynamic-range scores. + } // while changed +} + +// PruneForwardLinksFinal is a version of PruneForwardLinks that we call +// on the final frame. If there are final tokens active, it uses +// the final-probs for pruning, otherwise it treats all tokens as final. +template +void LatticeIncrementalDecoderTpl::PruneForwardLinksFinal() { + KALDI_ASSERT(!active_toks_.empty()); + int32 frame_plus_one = active_toks_.size() - 1; + + if (active_toks_[frame_plus_one].toks == NULL) // empty list; should not happen. + KALDI_WARN << "No tokens alive at end of file"; + + typedef typename unordered_map::const_iterator IterType; + ComputeFinalCosts(&final_costs_, &final_relative_cost_, &final_best_cost_); + decoding_finalized_ = true; + // We call DeleteElems() as a nicety, not because it's really necessary; + // otherwise there would be a time, after calling PruneTokensForFrame() on the + // final frame, when toks_.GetList() or toks_.Clear() would contain pointers + // to nonexistent tokens. + DeleteElems(toks_.Clear()); + + // Now go through tokens on this frame, pruning forward links... may have to + // iterate a few times until there is no more change, because the list is not + // in topological order. This is a modified version of the code in + // PruneForwardLinks, but here we also take account of the final-probs. + bool changed = true; + BaseFloat delta = 1.0e-05; + while (changed) { + changed = false; + for (Token *tok = active_toks_[frame_plus_one].toks; tok != NULL; + tok = tok->next) { + ForwardLinkT *link, *prev_link = NULL; + // will recompute tok_extra_cost. It has a term in it that corresponds + // to the "final-prob", so instead of initializing tok_extra_cost to infinity + // below we set it to the difference between the (score+final_prob) of this + // token, + // and the best such (score+final_prob). + BaseFloat final_cost; + if (final_costs_.empty()) { + final_cost = 0.0; + } else { + IterType iter = final_costs_.find(tok); + if (iter != final_costs_.end()) + final_cost = iter->second; + else + final_cost = std::numeric_limits::infinity(); + } + BaseFloat tok_extra_cost = tok->tot_cost + final_cost - final_best_cost_; + // tok_extra_cost will be a "min" over either directly being final, or + // being indirectly final through other links, and the loop below may + // decrease its value: + for (link = tok->links; link != NULL;) { + // See if we need to excise this link... + Token *next_tok = link->next_tok; + BaseFloat link_extra_cost = + next_tok->extra_cost + + ((tok->tot_cost + link->acoustic_cost + link->graph_cost) - + next_tok->tot_cost); + if (link_extra_cost > config_.lattice_beam) { // excise link + ForwardLinkT *next_link = link->next; + if (prev_link != NULL) + prev_link->next = next_link; + else + tok->links = next_link; + delete link; + link = next_link; // advance link but leave prev_link the same. + } else { // keep the link and update the tok_extra_cost if needed. + if (link_extra_cost < 0.0) { // this is just a precaution. + if (link_extra_cost < -0.01) + KALDI_WARN << "Negative extra_cost: " << link_extra_cost; + link_extra_cost = 0.0; + } + if (link_extra_cost < tok_extra_cost) tok_extra_cost = link_extra_cost; + prev_link = link; + link = link->next; + } + } + // prune away tokens worse than lattice_beam above best path. This step + // was not necessary in the non-final case because then, this case + // showed up as having no forward links. Here, the tok_extra_cost has + // an extra component relating to the final-prob. + if (tok_extra_cost > config_.lattice_beam) + tok_extra_cost = std::numeric_limits::infinity(); + // to be pruned in PruneTokensForFrame + + if (!ApproxEqual(tok->extra_cost, tok_extra_cost, delta)) changed = true; + tok->extra_cost = tok_extra_cost; // will be +infinity or <= lattice_beam_. + } + } // while changed +} + +template +BaseFloat LatticeIncrementalDecoderTpl::FinalRelativeCost() const { + BaseFloat relative_cost; + ComputeFinalCosts(NULL, &relative_cost, NULL); + return relative_cost; +} + +// Prune away any tokens on this frame that have no forward links. +// [we don't do this in PruneForwardLinks because it would give us +// a problem with dangling pointers]. +// It's called by PruneActiveTokens if any forward links have been pruned +template +void LatticeIncrementalDecoderTpl::PruneTokensForFrame( + int32 frame_plus_one) { + KALDI_ASSERT(frame_plus_one >= 0 && frame_plus_one < active_toks_.size()); + Token *&toks = active_toks_[frame_plus_one].toks; + if (toks == NULL) KALDI_WARN << "No tokens alive [doing pruning]"; + Token *tok, *next_tok, *prev_tok = NULL; + int32 num_toks = 0; + for (tok = toks; tok != NULL; tok = next_tok, num_toks++) { + next_tok = tok->next; + if (tok->extra_cost == std::numeric_limits::infinity()) { + // token is unreachable from end of graph; (no forward links survived) + // excise tok from list and delete tok. + if (prev_tok != NULL) + prev_tok->next = tok->next; + else + toks = tok->next; + delete tok; + num_toks_--; + } else { // fetch next Token + prev_tok = tok; + } + } + active_toks_[frame_plus_one].num_toks = num_toks; +} + +// Go backwards through still-alive tokens, pruning them, starting not from +// the current frame (where we want to keep all tokens) but from the frame before +// that. We go backwards through the frames and stop when we reach a point +// where the delta-costs are not changing (and the delta controls when we consider +// a cost to have "not changed"). +template +void LatticeIncrementalDecoderTpl::PruneActiveTokens(BaseFloat delta) { + int32 cur_frame_plus_one = NumFramesDecoded(); + int32 num_toks_begin = num_toks_; + + if (active_toks_[cur_frame_plus_one].num_toks == -1){ + // The current frame's tokens don't get pruned so they don't get counted + // (the count is needed by the incremental determinization code). + // Fix this. + int this_frame_num_toks = 0; + for (Token *t = active_toks_[cur_frame_plus_one].toks; t != NULL; t = t->next) + this_frame_num_toks++; + active_toks_[cur_frame_plus_one].num_toks = this_frame_num_toks; + } + + // The index "f" below represents a "frame plus one", i.e. you'd have to subtract + // one to get the corresponding index for the decodable object. + for (int32 f = cur_frame_plus_one - 1; f >= 0; f--) { + // Reason why we need to prune forward links in this situation: + // (1) we have never pruned them (new TokenList) + // (2) we have not yet pruned the forward links to the next f, + // after any of those tokens have changed their extra_cost. + if (active_toks_[f].must_prune_forward_links) { + bool extra_costs_changed = false, links_pruned = false; + PruneForwardLinks(f, &extra_costs_changed, &links_pruned, delta); + if (extra_costs_changed && f > 0) // any token has changed extra_cost + active_toks_[f - 1].must_prune_forward_links = true; + if (links_pruned) // any link was pruned + active_toks_[f].must_prune_tokens = true; + active_toks_[f].must_prune_forward_links = false; // job done + } + if (f + 1 < cur_frame_plus_one && // except for last f (no forward links) + active_toks_[f + 1].must_prune_tokens) { + PruneTokensForFrame(f + 1); + active_toks_[f + 1].must_prune_tokens = false; + } + } + KALDI_VLOG(4) << "pruned tokens from " << num_toks_begin + << " to " << num_toks_; +} + +template +void LatticeIncrementalDecoderTpl::ComputeFinalCosts( + unordered_map *final_costs, BaseFloat *final_relative_cost, + BaseFloat *final_best_cost) const { + if (decoding_finalized_) { + // If we finalized decoding, the list toks_ will no longer exist, so return + // something we already computed. + if (final_costs) *final_costs = final_costs_; + if (final_relative_cost) *final_relative_cost = final_relative_cost_; + if (final_best_cost) *final_best_cost = final_best_cost_; + return; + } + if (final_costs != NULL) final_costs->clear(); + const Elem *final_toks = toks_.GetList(); + BaseFloat infinity = std::numeric_limits::infinity(); + BaseFloat best_cost = infinity, best_cost_with_final = infinity; + + while (final_toks != NULL) { + StateId state = final_toks->key; + Token *tok = final_toks->val; + const Elem *next = final_toks->tail; + BaseFloat final_cost = fst_->Final(state).Value(); + BaseFloat cost = tok->tot_cost, cost_with_final = cost + final_cost; + best_cost = std::min(cost, best_cost); + best_cost_with_final = std::min(cost_with_final, best_cost_with_final); + if (final_costs != NULL && final_cost != infinity) + (*final_costs)[tok] = final_cost; + final_toks = next; + } + if (final_relative_cost != NULL) { + if (best_cost == infinity && best_cost_with_final == infinity) { + // Likely this will only happen if there are no tokens surviving. + // This seems the least bad way to handle it. + *final_relative_cost = infinity; + } else { + *final_relative_cost = best_cost_with_final - best_cost; + } + } + if (final_best_cost != NULL) { + if (best_cost_with_final != infinity) { // final-state exists. + *final_best_cost = best_cost_with_final; + } else { // no final-state exists. + *final_best_cost = best_cost; + } + } +} + +template +void LatticeIncrementalDecoderTpl::AdvanceDecoding( + DecodableInterface *decodable, int32 max_num_frames) { + if (std::is_same >::value) { + // if the type 'FST' is the FST base-class, then see if the FST type of fst_ + // is actually VectorFst or ConstFst. If so, call the AdvanceDecoding() + // function after casting *this to the more specific type. + if (fst_->Type() == "const") { + LatticeIncrementalDecoderTpl, Token> *this_cast = + reinterpret_cast< + LatticeIncrementalDecoderTpl, Token> *>( + this); + this_cast->AdvanceDecoding(decodable, max_num_frames); + return; + } else if (fst_->Type() == "vector") { + LatticeIncrementalDecoderTpl, Token> *this_cast = + reinterpret_cast< + LatticeIncrementalDecoderTpl, Token> *>( + this); + this_cast->AdvanceDecoding(decodable, max_num_frames); + return; + } + } + + KALDI_ASSERT(!active_toks_.empty() && !decoding_finalized_ && + "You must call InitDecoding() before AdvanceDecoding"); + int32 num_frames_ready = decodable->NumFramesReady(); + // num_frames_ready must be >= num_frames_decoded, or else + // the number of frames ready must have decreased (which doesn't + // make sense) or the decodable object changed between calls + // (which isn't allowed). + KALDI_ASSERT(num_frames_ready >= NumFramesDecoded()); + int32 target_frames_decoded = num_frames_ready; + if (max_num_frames >= 0) + target_frames_decoded = + std::min(target_frames_decoded, NumFramesDecoded() + max_num_frames); + while (NumFramesDecoded() < target_frames_decoded) { + if (NumFramesDecoded() % config_.prune_interval == 0) { + PruneActiveTokens(config_.lattice_beam * config_.prune_scale); + } + BaseFloat cost_cutoff = ProcessEmitting(decodable); + ProcessNonemitting(cost_cutoff); + } + UpdateLatticeDeterminization(); +} + +// FinalizeDecoding() is a version of PruneActiveTokens that we call +// (optionally) on the final frame. Takes into account the final-prob of +// tokens. This function used to be called PruneActiveTokensFinal(). +template +void LatticeIncrementalDecoderTpl::FinalizeDecoding() { + int32 final_frame_plus_one = NumFramesDecoded(); + int32 num_toks_begin = num_toks_; + // PruneForwardLinksFinal() prunes the final frame (with final-probs), and + // sets decoding_finalized_. + PruneForwardLinksFinal(); + for (int32 f = final_frame_plus_one - 1; f >= 0; f--) { + bool b1, b2; // values not used. + BaseFloat dontcare = 0.0; // delta of zero means we must always update + PruneForwardLinks(f, &b1, &b2, dontcare); + PruneTokensForFrame(f + 1); + } + PruneTokensForFrame(0); + KALDI_VLOG(4) << "pruned tokens from " << num_toks_begin << " to " << num_toks_; +} + +/// Gets the weight cutoff. Also counts the active tokens. +template +BaseFloat LatticeIncrementalDecoderTpl::GetCutoff( + Elem *list_head, size_t *tok_count, BaseFloat *adaptive_beam, Elem **best_elem) { + BaseFloat best_weight = std::numeric_limits::infinity(); + // positive == high cost == bad. + size_t count = 0; + if (config_.max_active == std::numeric_limits::max() && + config_.min_active == 0) { + for (Elem *e = list_head; e != NULL; e = e->tail, count++) { + BaseFloat w = static_cast(e->val->tot_cost); + if (w < best_weight) { + best_weight = w; + if (best_elem) *best_elem = e; + } + } + if (tok_count != NULL) *tok_count = count; + if (adaptive_beam != NULL) *adaptive_beam = config_.beam; + return best_weight + config_.beam; + } else { + tmp_array_.clear(); + for (Elem *e = list_head; e != NULL; e = e->tail, count++) { + BaseFloat w = e->val->tot_cost; + tmp_array_.push_back(w); + if (w < best_weight) { + best_weight = w; + if (best_elem) *best_elem = e; + } + } + if (tok_count != NULL) *tok_count = count; + + BaseFloat beam_cutoff = best_weight + config_.beam, + min_active_cutoff = std::numeric_limits::infinity(), + max_active_cutoff = std::numeric_limits::infinity(); + + KALDI_VLOG(6) << "Number of tokens active on frame " << NumFramesDecoded() + << " is " << tmp_array_.size(); + + if (tmp_array_.size() > static_cast(config_.max_active)) { + std::nth_element(tmp_array_.begin(), tmp_array_.begin() + config_.max_active, + tmp_array_.end()); + max_active_cutoff = tmp_array_[config_.max_active]; + } + if (max_active_cutoff < beam_cutoff) { // max_active is tighter than beam. + if (adaptive_beam) + *adaptive_beam = max_active_cutoff - best_weight + config_.beam_delta; + return max_active_cutoff; + } + if (tmp_array_.size() > static_cast(config_.min_active)) { + if (config_.min_active == 0) + min_active_cutoff = best_weight; + else { + std::nth_element(tmp_array_.begin(), tmp_array_.begin() + config_.min_active, + tmp_array_.size() > static_cast(config_.max_active) + ? tmp_array_.begin() + config_.max_active + : tmp_array_.end()); + min_active_cutoff = tmp_array_[config_.min_active]; + } + } + if (min_active_cutoff > beam_cutoff) { // min_active is looser than beam. + if (adaptive_beam) + *adaptive_beam = min_active_cutoff - best_weight + config_.beam_delta; + return min_active_cutoff; + } else { + *adaptive_beam = config_.beam; + return beam_cutoff; + } + } +} + +template +BaseFloat LatticeIncrementalDecoderTpl::ProcessEmitting( + DecodableInterface *decodable) { + KALDI_ASSERT(active_toks_.size() > 0); + int32 frame = active_toks_.size() - 1; // frame is the frame-index + // (zero-based) used to get likelihoods + // from the decodable object. + active_toks_.resize(active_toks_.size() + 1); + + Elem *final_toks = toks_.Clear(); // analogous to swapping prev_toks_ / cur_toks_ + // in simple-decoder.h. Removes the Elems from + // being indexed in the hash in toks_. + Elem *best_elem = NULL; + BaseFloat adaptive_beam; + size_t tok_cnt; + BaseFloat cur_cutoff = GetCutoff(final_toks, &tok_cnt, &adaptive_beam, &best_elem); + KALDI_VLOG(6) << "Adaptive beam on frame " << NumFramesDecoded() << " is " + << adaptive_beam; + + PossiblyResizeHash(tok_cnt); // This makes sure the hash is always big enough. + + BaseFloat next_cutoff = std::numeric_limits::infinity(); + // pruning "online" before having seen all tokens + + BaseFloat cost_offset = 0.0; // Used to keep probabilities in a good + // dynamic range. + + // First process the best token to get a hopefully + // reasonably tight bound on the next cutoff. The only + // products of the next block are "next_cutoff" and "cost_offset". + if (best_elem) { + StateId state = best_elem->key; + Token *tok = best_elem->val; + cost_offset = -tok->tot_cost; + for (fst::ArcIterator aiter(*fst_, state); !aiter.Done(); aiter.Next()) { + const Arc &arc = aiter.Value(); + if (arc.ilabel != 0) { // propagate.. + BaseFloat new_weight = arc.weight.Value() + cost_offset - + decodable->LogLikelihood(frame, arc.ilabel) + + tok->tot_cost; + if (new_weight + adaptive_beam < next_cutoff) + next_cutoff = new_weight + adaptive_beam; + } + } + } + + // Store the offset on the acoustic likelihoods that we're applying. + // Could just do cost_offsets_.push_back(cost_offset), but we + // do it this way as it's more robust to future code changes. + cost_offsets_.resize(frame + 1, 0.0); + cost_offsets_[frame] = cost_offset; + + // the tokens are now owned here, in final_toks, and the hash is empty. + // 'owned' is a complex thing here; the point is we need to call DeleteElem + // on each elem 'e' to let toks_ know we're done with them. + for (Elem *e = final_toks, *e_tail; e != NULL; e = e_tail) { + // loop this way because we delete "e" as we go. + StateId state = e->key; + Token *tok = e->val; + if (tok->tot_cost <= cur_cutoff) { + for (fst::ArcIterator aiter(*fst_, state); !aiter.Done(); aiter.Next()) { + const Arc &arc = aiter.Value(); + if (arc.ilabel != 0) { // propagate.. + BaseFloat ac_cost = + cost_offset - decodable->LogLikelihood(frame, arc.ilabel), + graph_cost = arc.weight.Value(), cur_cost = tok->tot_cost, + tot_cost = cur_cost + ac_cost + graph_cost; + if (tot_cost >= next_cutoff) + continue; + else if (tot_cost + adaptive_beam < next_cutoff) + next_cutoff = tot_cost + adaptive_beam; // prune by best current token + // Note: the frame indexes into active_toks_ are one-based, + // hence the + 1. + Token *next_tok = + FindOrAddToken(arc.nextstate, frame + 1, tot_cost, tok, NULL); + // NULL: no change indicator needed + + // Add ForwardLink from tok to next_tok (put on head of list tok->links) + tok->links = new ForwardLinkT(next_tok, arc.ilabel, arc.olabel, graph_cost, + ac_cost, tok->links); + } + } // for all arcs + } + e_tail = e->tail; + toks_.Delete(e); // delete Elem + } + return next_cutoff; +} + +// static inline +template +void LatticeIncrementalDecoderTpl::DeleteForwardLinks(Token *tok) { + ForwardLinkT *l = tok->links, *m; + while (l != NULL) { + m = l->next; + delete l; + l = m; + } + tok->links = NULL; +} + +template +void LatticeIncrementalDecoderTpl::ProcessNonemitting(BaseFloat cutoff) { + KALDI_ASSERT(!active_toks_.empty()); + int32 frame = static_cast(active_toks_.size()) - 2; + // Note: "frame" is the time-index we just processed, or -1 if + // we are processing the nonemitting transitions before the + // first frame (called from InitDecoding()). + + // Processes nonemitting arcs for one frame. Propagates within toks_. + // Note-- this queue structure is is not very optimal as + // it may cause us to process states unnecessarily (e.g. more than once), + // but in the baseline code, turning this vector into a set to fix this + // problem did not improve overall speed. + + KALDI_ASSERT(queue_.empty()); + + if (toks_.GetList() == NULL) { + if (!warned_) { + KALDI_WARN << "Error, no surviving tokens: frame is " << frame; + warned_ = true; + } + } + + for (const Elem *e = toks_.GetList(); e != NULL; e = e->tail) { + StateId state = e->key; + if (fst_->NumInputEpsilons(state) != 0) queue_.push_back(state); + } + + while (!queue_.empty()) { + StateId state = queue_.back(); + queue_.pop_back(); + + Token *tok = + toks_.Find(state) + ->val; // would segfault if state not in toks_ but this can't happen. + BaseFloat cur_cost = tok->tot_cost; + if (cur_cost >= cutoff) // Don't bother processing successors. + continue; + // If "tok" has any existing forward links, delete them, + // because we're about to regenerate them. This is a kind + // of non-optimality (remember, this is the simple decoder), + // but since most states are emitting it's not a huge issue. + DeleteForwardLinks(tok); // necessary when re-visiting + tok->links = NULL; + for (fst::ArcIterator aiter(*fst_, state); !aiter.Done(); aiter.Next()) { + const Arc &arc = aiter.Value(); + if (arc.ilabel == 0) { // propagate nonemitting only... + BaseFloat graph_cost = arc.weight.Value(), tot_cost = cur_cost + graph_cost; + if (tot_cost < cutoff) { + bool changed; + + Token *new_tok = + FindOrAddToken(arc.nextstate, frame + 1, tot_cost, tok, &changed); + + tok->links = + new ForwardLinkT(new_tok, 0, arc.olabel, graph_cost, 0, tok->links); + + // "changed" tells us whether the new token has a different + // cost from before, or is new [if so, add into queue]. + if (changed && fst_->NumInputEpsilons(arc.nextstate) != 0) + queue_.push_back(arc.nextstate); + } + } + } // for all arcs + } // while queue not empty +} + +template +void LatticeIncrementalDecoderTpl::DeleteElems(Elem *list) { + for (Elem *e = list, *e_tail; e != NULL; e = e_tail) { + e_tail = e->tail; + toks_.Delete(e); + } +} + +template +void LatticeIncrementalDecoderTpl< + FST, Token>::ClearActiveTokens() { // a cleanup routine, at utt end/begin + for (size_t i = 0; i < active_toks_.size(); i++) { + // Delete all tokens alive on this frame, and any forward + // links they may have. + for (Token *tok = active_toks_[i].toks; tok != NULL;) { + DeleteForwardLinks(tok); + Token *next_tok = tok->next; + delete tok; + num_toks_--; + tok = next_tok; + } + } + active_toks_.clear(); + KALDI_ASSERT(num_toks_ == 0); +} + + +template +const CompactLattice& LatticeIncrementalDecoderTpl::GetLattice( + int32 num_frames_to_include, + bool use_final_probs) { + KALDI_ASSERT(num_frames_to_include >= num_frames_in_lattice_ && + num_frames_to_include <= NumFramesDecoded()); + + + if (num_frames_in_lattice_ > 0 && + determinizer_.GetLattice().NumStates() == 0) { + /* Something went wrong, lattice is empty and will continue to be empty. + User-level code should detect and deal with this. + */ + num_frames_in_lattice_ = num_frames_to_include; + return determinizer_.GetLattice(); + } + + if (decoding_finalized_ && !use_final_probs) { + // This is not supported + KALDI_ERR << "You cannot get the lattice without final-probs after " + "calling FinalizeDecoding()."; + } + if (use_final_probs && num_frames_to_include != NumFramesDecoded()) { + /* This is because we only remember the relation between HCLG states and + Tokens for the current frame; the Token does not have a `state` field. */ + KALDI_ERR << "use-final-probs may no be true if you are not " + "getting a lattice for all frames decoded so far."; + } + + + if (num_frames_to_include > num_frames_in_lattice_) { + /* Make sure the token-pruning is up to date. If we just pruned the tokens, + this will do very little work. */ + PruneActiveTokens(config_.lattice_beam * config_.prune_scale); + + if (determinizer_.GetLattice().NumStates() == 0 || + determinizer_.GetLattice().Final(0) != CompactLatticeWeight::Zero()) { + num_frames_in_lattice_ = 0; + determinizer_.Init(); + } + + Lattice chunk_lat; + + unordered_map token_label2state; + if (num_frames_in_lattice_ != 0) { + determinizer_.InitializeRawLatticeChunk(&chunk_lat, + &token_label2state); + } + + // tok_map will map from Token* to state-id in chunk_lat. + // The cur and prev versions alternate on different frames. + unordered_map &tok2state_map(temp_token_map_); + tok2state_map.clear(); + + unordered_map &next_token2label_map(token2label_map_temp_); + next_token2label_map.clear(); + + { // Deal with the last frame in the chunk, the one numbered `num_frames_to_include`. + // (Yes, this is backwards). We allocate token labels, and set tokens as + // final, but don't add any transitions. This may leave some states + // disconnected (e.g. due to chains of nonemitting arcs), but it's OK; we'll + // fix it when we generate the next chunk of lattice. + int32 frame = num_frames_to_include; + // Allocate state-ids for all tokens on this frame. + + for (Token *tok = active_toks_[frame].toks; tok != NULL; tok = tok->next) { + /* If we included the final-costs at this stage, they will cause + non-final states to be pruned out from the end of the lattice. */ + BaseFloat final_cost; + { // This block computes final_cost + if (decoding_finalized_) { + if (final_costs_.empty()) { + final_cost = 0.0; /* No final-state survived, so treat all as final + * with probability One(). */ + } else { + auto iter = final_costs_.find(tok); + if (iter == final_costs_.end()) + final_cost = std::numeric_limits::infinity(); + else + final_cost = iter->second; + } + } else { + /* this is a `fake` final-cost used to guide pruning. It's as if we + set the betas (backward-probs) on the final frame to the + negatives of the corresponding alphas, so all tokens on the last + frae will be on a best path.. the extra_cost for each token + always corresponds to its alpha+beta on this assumption. We want + the final_cost here to correspond to the beta (backward-prob), so + we get that by final_cost = extra_cost - tot_cost. + [The tot_cost is the forward/alpha cost.] + */ + final_cost = tok->extra_cost - tok->tot_cost; + } + } + + StateId state = chunk_lat.AddState(); + tok2state_map[tok] = state; + if (final_cost < std::numeric_limits::infinity()) { + next_token2label_map[tok] = AllocateNewTokenLabel(); + StateId token_final_state = chunk_lat.AddState(); + LatticeArc::Label ilabel = 0, + olabel = (next_token2label_map[tok] = AllocateNewTokenLabel()); + chunk_lat.AddArc(state, + LatticeArc(ilabel, olabel, + LatticeWeight::One(), + token_final_state)); + chunk_lat.SetFinal(token_final_state, LatticeWeight(final_cost, 0.0)); + } + } + } + + // Go in reverse order over the remaining frames so we can create arcs as we + // go, and their destination-states will already be in the map. + for (int32 frame = num_frames_to_include; + frame >= num_frames_in_lattice_; frame--) { + // The conditional below is needed for the last frame of the utterance. + BaseFloat cost_offset = (frame < cost_offsets_.size() ? + cost_offsets_[frame] : 0.0); + + // For the first frame of the chunk, we need to make sure the states are + // the ones created by InitializeRawLatticeChunk() (where not pruned away). + if (frame == num_frames_in_lattice_ && num_frames_in_lattice_ != 0) { + for (Token *tok = active_toks_[frame].toks; tok != NULL; tok = tok->next) { + auto iter = token2label_map_.find(tok); + KALDI_ASSERT(iter != token2label_map_.end()); + Label token_label = iter->second; + auto iter2 = token_label2state.find(token_label); + if (iter2 != token_label2state.end()) { + StateId state = iter2->second; + tok2state_map[tok] = state; + } else { + // Some states may have been pruned out, but we should still allocate + // them. They might have been part of chains of nonemitting arcs + // where the state became disconnected because the last chunk didn't + // include arcs starting at this frame. + StateId state = chunk_lat.AddState(); + tok2state_map[tok] = state; + } + } + } else if (frame != num_frames_to_include) { // We already created states + // for the last frame. + for (Token *tok = active_toks_[frame].toks; tok != NULL; tok = tok->next) { + StateId state = chunk_lat.AddState(); + tok2state_map[tok] = state; + } + } + for (Token *tok = active_toks_[frame].toks; tok != NULL; tok = tok->next) { + auto iter = tok2state_map.find(tok); + KALDI_ASSERT(iter != tok2state_map.end()); + StateId cur_state = iter->second; + for (ForwardLinkT *l = tok->links; l != NULL; l = l->next) { + auto next_iter = tok2state_map.find(l->next_tok); + if (next_iter == tok2state_map.end()) { + // Emitting arcs from the last frame we're including -- ignore + // these. + KALDI_ASSERT(frame == num_frames_to_include); + continue; + } + StateId next_state = next_iter->second; + BaseFloat this_offset = (l->ilabel != 0 ? cost_offset : 0); + LatticeArc arc(l->ilabel, l->olabel, + LatticeWeight(l->graph_cost, l->acoustic_cost - this_offset), + next_state); + // Note: the epsilons get redundantly included at the end and beginning + // of successive chunks. These will get removed in the determinization. + chunk_lat.AddArc(cur_state, arc); + } + } + } + if (num_frames_in_lattice_ == 0) { + // This block locates the start token. NOTE: we use the fact that in the + // linked list of tokens, things are added at the head, so the start state + // must be at the tail. If this data structure is changed in future, we + // might need to explicitly store the start token as a class member. + Token *tok = active_toks_[0].toks; + if (tok == NULL) { + KALDI_WARN << "No tokens exist on start frame"; + return determinizer_.GetLattice(); // will be empty. + } + while (tok->next != NULL) + tok = tok->next; + Token *start_token = tok; + auto iter = tok2state_map.find(start_token); + KALDI_ASSERT(iter != tok2state_map.end()); + StateId start_state = iter->second; + chunk_lat.SetStart(start_state); + } + token2label_map_.swap(next_token2label_map); + + // bool finished_before_beam = + determinizer_.AcceptRawLatticeChunk(&chunk_lat); + // We are ignoring the return status, which say whether it finished before the beam. + + num_frames_in_lattice_ = num_frames_to_include; + + if (determinizer_.GetLattice().NumStates() == 0) + return determinizer_.GetLattice(); // Something went wrong, lattice is empty. + } + + unordered_map token2final_cost; + unordered_map token_label2final_cost; + if (use_final_probs) { + ComputeFinalCosts(&token2final_cost, NULL, NULL); + for (const auto &p: token2final_cost) { + Token *tok = p.first; + BaseFloat cost = p.second; + auto iter = token2label_map_.find(tok); + if (iter != token2label_map_.end()) { + /* Some tokens may not have survived the pruned determinization. */ + Label token_label = iter->second; + bool ret = token_label2final_cost.insert({token_label, cost}).second; + KALDI_ASSERT(ret); /* Make sure it was inserted. */ + } + } + } + /* Note: these final-probs won't affect the next chunk, only the lattice + returned from GetLattice(). They are kind of temporaries. */ + determinizer_.SetFinalCosts(token_label2final_cost.empty() ? NULL : + &token_label2final_cost); + + return determinizer_.GetLattice(); +} + + +template +int32 LatticeIncrementalDecoderTpl::GetNumToksForFrame(int32 frame) { + int32 r = 0; + for (Token *tok = active_toks_[frame].toks; tok; tok = tok->next) r++; + return r; +} + + + +/* This utility function adds an arc to a Lattice, but where the source is a + CompactLatticeArc. If the CompactLatticeArc has a string with length greater + than 1, this will require adding extra states to `lat`. + */ +static void AddCompactLatticeArcToLattice( + const CompactLatticeArc &clat_arc, + LatticeArc::StateId src_state, + Lattice *lat) { + const std::vector &string = clat_arc.weight.String(); + size_t N = string.size(); + if (N == 0) { + LatticeArc arc; + arc.ilabel = 0; + arc.olabel = clat_arc.ilabel; + arc.nextstate = clat_arc.nextstate; + arc.weight = clat_arc.weight.Weight(); + lat->AddArc(src_state, arc); + } else { + LatticeArc::StateId cur_state = src_state; + for (size_t i = 0; i < N; i++) { + LatticeArc arc; + arc.ilabel = string[i]; + arc.olabel = (i == 0 ? clat_arc.ilabel : 0); + arc.nextstate = (i + 1 == N ? clat_arc.nextstate : lat->AddState()); + arc.weight = (i == 0 ? clat_arc.weight.Weight() : LatticeWeight::One()); + lat->AddArc(cur_state, arc); + cur_state = arc.nextstate; + } + } +} + + +void LatticeIncrementalDeterminizer::Init() { + non_final_redet_states_.clear(); + clat_.DeleteStates(); + final_arcs_.clear(); + forward_costs_.clear(); + arcs_in_.clear(); +} + +CompactLattice::StateId LatticeIncrementalDeterminizer::AddStateToClat() { + CompactLattice::StateId ans = clat_.AddState(); + forward_costs_.push_back(std::numeric_limits::infinity()); + KALDI_ASSERT(forward_costs_.size() == ans + 1); + arcs_in_.resize(ans + 1); + return ans; +} + +void LatticeIncrementalDeterminizer::AddArcToClat( + CompactLattice::StateId state, + const CompactLatticeArc &arc) { + BaseFloat forward_cost = forward_costs_[state] + + ConvertToCost(arc.weight); + if (forward_cost == std::numeric_limits::infinity()) + return; + int32 arc_idx = clat_.NumArcs(state); + clat_.AddArc(state, arc); + arcs_in_[arc.nextstate].push_back({state, arc_idx}); + if (forward_cost < forward_costs_[arc.nextstate]) + forward_costs_[arc.nextstate] = forward_cost; +} + +// See documentation in header +void LatticeIncrementalDeterminizer::IdentifyTokenFinalStates( + const CompactLattice &chunk_clat, + std::unordered_map *token_map) const { + token_map->clear(); + using StateId = CompactLattice::StateId; + using Label = CompactLatticeArc::Label; + + StateId num_states = chunk_clat.NumStates(); + for (StateId state = 0; state < num_states; state++) { + for (fst::ArcIterator aiter(chunk_clat, state); + !aiter.Done(); aiter.Next()) { + const CompactLatticeArc &arc = aiter.Value(); + if (arc.olabel >= kTokenLabelOffset && arc.olabel < kMaxTokenLabel) { + StateId nextstate = arc.nextstate; + auto r = token_map->insert({nextstate, arc.olabel}); + // Check consistency of labels on incoming arcs + KALDI_ASSERT(r.first->second == arc.olabel); + } + } + } +} + + + + +void LatticeIncrementalDeterminizer::GetNonFinalRedetStates() { + using StateId = CompactLattice::StateId; + non_final_redet_states_.clear(); + non_final_redet_states_.reserve(final_arcs_.size()); + + std::vector state_queue; + for (const CompactLatticeArc &arc: final_arcs_) { + // Note: we abuse the .nextstate field to store the state which is really + // the source of that arc. + StateId redet_state = arc.nextstate; + if (forward_costs_[redet_state] != std::numeric_limits::infinity()) { + // if it is accessible.. + if (non_final_redet_states_.insert(redet_state).second) { + // it was not already there + state_queue.push_back(redet_state); + } + } + } + // Add any states that are reachable from the states above. + while (!state_queue.empty()) { + StateId s = state_queue.back(); + state_queue.pop_back(); + for (fst::ArcIterator aiter(clat_, s); !aiter.Done(); + aiter.Next()) { + const CompactLatticeArc &arc = aiter.Value(); + StateId nextstate = arc.nextstate; + if (non_final_redet_states_.insert(nextstate).second) + state_queue.push_back(nextstate); // it was not already there + } + } +} + + +void LatticeIncrementalDeterminizer::InitializeRawLatticeChunk( + Lattice *olat, + unordered_map *token_label2state) { + using namespace fst; + + olat->DeleteStates(); + LatticeArc::StateId start_state = olat->AddState(); + olat->SetStart(start_state); + token_label2state->clear(); + + // redet_state_map maps from state-ids in clat_ to state-ids in olat. This + // will be the set of states from which the arcs to final-states in the + // canonical appended lattice leave (physically, these are in the .nextstate + // elements of arcs_, since we use that field for the source state), plus any + // states reachable from those states. + unordered_map redet_state_map; + + for (CompactLattice::StateId redet_state: non_final_redet_states_) + redet_state_map[redet_state] = olat->AddState(); + + // First, process any arcs leaving the non-final redeterminized states that + // are not to final-states. (What we mean by "not to final states" is, not to + // stats that are final in the `canonical appended lattice`.. they may + // actually be physically final in clat_, because we make clat_ what we want + // to return to the user. + for (CompactLattice::StateId redet_state: non_final_redet_states_) { + LatticeArc::StateId lat_state = redet_state_map[redet_state]; + + for (ArcIterator aiter(clat_, redet_state); + !aiter.Done(); aiter.Next()) { + const CompactLatticeArc &arc = aiter.Value(); + CompactLattice::StateId nextstate = arc.nextstate; + LatticeArc::StateId lat_nextstate = olat->NumStates(); + auto r = redet_state_map.insert({nextstate, lat_nextstate}); + if (r.second) { // Was inserted. + LatticeArc::StateId s = olat->AddState(); + KALDI_ASSERT(s == lat_nextstate); + } else { + // was not inserted -> was already there. + lat_nextstate = r.first->second; + } + CompactLatticeArc clat_arc(arc); + clat_arc.nextstate = lat_nextstate; + AddCompactLatticeArcToLattice(clat_arc, lat_state, olat); + } + clat_.DeleteArcs(redet_state); + clat_.SetFinal(redet_state, CompactLatticeWeight::Zero()); + } + + for (const CompactLatticeArc &arc: final_arcs_) { + // We abuse the `nextstate` field to store the source state. + CompactLattice::StateId src_state = arc.nextstate; + auto iter = redet_state_map.find(src_state); + if (forward_costs_[src_state] == std::numeric_limits::infinity()) + continue; /* Unreachable state */ + KALDI_ASSERT(iter != redet_state_map.end()); + LatticeArc::StateId src_lat_state = iter->second; + Label token_label = arc.ilabel; // will be == arc.olabel. + KALDI_ASSERT(token_label >= kTokenLabelOffset && + token_label < kMaxTokenLabel); + auto r = token_label2state->insert({token_label, + olat->NumStates()}); + LatticeArc::StateId dest_lat_state = r.first->second; + if (r.second) { // was inserted + LatticeArc::StateId new_state = olat->AddState(); + KALDI_ASSERT(new_state == dest_lat_state); + } + CompactLatticeArc new_arc; + new_arc.nextstate = dest_lat_state; + /* We convert the token-label to epsilon; it's not needed anymore. */ + new_arc.ilabel = new_arc.olabel = 0; + new_arc.weight = arc.weight; + AddCompactLatticeArcToLattice(new_arc, src_lat_state, olat); + } + + // Now deal with the initial-probs. Arcs from initial-states to + // redeterminized-states in the raw lattice have an olabel that identifies the + // id of that redeterminized-state in clat_, and a cost that is derived from + // its entry in forward_costs_. These forward-probs are used to get the + // pruned lattice determinization to behave correctly, and will be canceled + // out later on. + // + // In the paper this is the second-from-last bullet in Sec. 5.2. NOTE: in the + // paper we state that we only include such arcs for "each redeterminized + // state that is either initial in det(A) or that has an arc entering it from + // a state that is not a redeterminized state." In fact, we include these + // arcs for all redeterminized states. I realized that it won't make a + // difference to the outcome, and it's easier to do it this way. + for (CompactLattice::StateId state_id: non_final_redet_states_) { + BaseFloat forward_cost = forward_costs_[state_id]; + LatticeArc arc; + arc.ilabel = 0; + // The olabel (which appears where the word-id would) is what + // we call a 'state-label'. It identifies a state in clat_. + arc.olabel = state_id + kStateLabelOffset; + // It doesn't matter what field we put forward_cost in (or whether we + // divide it among them both; the effect on pruning is the same, and + // we will cancel it out later anyway. + arc.weight = LatticeWeight(forward_cost, 0); + auto iter = redet_state_map.find(state_id); + KALDI_ASSERT(iter != redet_state_map.end()); + arc.nextstate = iter->second; + olat->AddArc(start_state, arc); + } +} + +void LatticeIncrementalDeterminizer::GetRawLatticeFinalCosts( + const Lattice &raw_fst, + std::unordered_map *old_final_costs) { + LatticeArc::StateId raw_fst_num_states = raw_fst.NumStates(); + for (LatticeArc::StateId s = 0; s < raw_fst_num_states; s++) { + for (fst::ArcIterator aiter(raw_fst, s); !aiter.Done(); + aiter.Next()) { + const LatticeArc &value = aiter.Value(); + if (value.olabel >= (Label)kTokenLabelOffset && + value.olabel < (Label)kMaxTokenLabel) { + LatticeWeight final_weight = raw_fst.Final(value.nextstate); + if (final_weight != LatticeWeight::Zero() && + final_weight.Value2() != 0) { + KALDI_ERR << "Label " << value.olabel << " from state " << s + << " looks like a token-label but its next-state " + << value.nextstate << + " has unexpected final-weight " << final_weight.Value1() << ',' + << final_weight.Value2(); + } + auto r = old_final_costs->insert({value.olabel, + final_weight.Value1()}); + if (!r.second && r.first->second != final_weight.Value1()) { + // For any given token-label, all arcs in raw_fst with that + // olabel should go to the same state, so this should be + // impossible. + KALDI_ERR << "Unexpected mismatch in final-costs for tokens, " + << r.first->second << " vs " << final_weight.Value1(); + } + } + } + } +} + + +bool LatticeIncrementalDeterminizer::ProcessArcsFromChunkStartState( + const CompactLattice &chunk_clat, + std::unordered_map *state_map) { + using StateId = CompactLattice::StateId; + StateId clat_num_states = clat_.NumStates(); + + // Process arcs leaving the start state of chunk_clat. These arcs will have + // state-labels on them (unless this is the first chunk). + // For destination-states of those arcs, work out which states in + // clat_ they correspond to and update their forward_costs. + for (fst::ArcIterator aiter(chunk_clat, chunk_clat.Start()); + !aiter.Done(); aiter.Next()) { + const CompactLatticeArc &arc = aiter.Value(); + Label label = arc.ilabel; // ilabel == olabel; would be the olabel + // in a Lattice. + if (!(label >= kStateLabelOffset && + label - kStateLabelOffset < clat_num_states)) { + // The label was not a state-label. This should only be possible on the + // first chunk. + KALDI_ASSERT(state_map->empty()); + return true; // this is the first chunk. + } + StateId clat_state = label - kStateLabelOffset; + StateId chunk_state = arc.nextstate; + auto p = state_map->insert({chunk_state, clat_state}); + StateId dest_clat_state = p.first->second; + // We deleted all its arcs in InitializeRawLatticeChunk + KALDI_ASSERT(clat_.NumArcs(clat_state) == 0); + /* + In almost all cases, dest_clat_state and clat_state will be the same state; + but there may be situations where two arcs with different state-labels + left the start state and entered the same next-state in chunk_clat; and in + these cases, they will be different. + + We didn't address this issue in the paper (or actually realize it could be + a problem). What we do is pick one of the clat_states as the "canonical" + one, and redirect all incoming transitions of the others to enter the + "canonical" one. (Search below for new_in_arc.nextstate = + dest_clat_state). + */ + if (clat_state != dest_clat_state) { + // Check that the start state isn't getting merged with any other state. + // If this were possible, we'd need to deal with it specially, but it + // can't be, because to be merged, 2 states must have identical arcs + // leaving them with identical weights, so we'd need to have another state + // on frame 0 identical to the start state, which is not possible if the + // lattice is deterministic and epsilon-free. + KALDI_ASSERT(clat_state != 0 && dest_clat_state != 0); + } + + // in_weight is an extra weight that we'll include on arcs entering this + // state from the previous chunk. We need to cancel out + // `forward_costs[clat_state]`, which was included in the corresponding arc + // in the raw lattice for pruning purposes; and we need to include the + // weight on the arc from the start-state of `chunk_clat` to this state. + CompactLatticeWeight extra_weight_in = arc.weight; + extra_weight_in.SetWeight( + fst::Times(extra_weight_in.Weight(), + LatticeWeight(-forward_costs_[clat_state], 0.0))); + + // We don't allow state 0 to be a redeterminized-state; calling code assures + // this. Search for `determinizer_.GetLattice().Final(0) != + // CompactLatticeWeight::Zero())` to find that calling code. + KALDI_ASSERT(clat_state != 0); + + // Note: 0 is the start state of clat_. This was checked. + forward_costs_[clat_state] = (clat_state == 0 ? 0 : + std::numeric_limits::infinity()); + std::vector > arcs_in; + arcs_in.swap(arcs_in_[clat_state]); + for (auto p: arcs_in) { + // Note: we'll be doing `continue` below if this input arc came from + // another redeterminized-state, because we did DeleteArcs() for them in + // InitializeRawLatticeChunk(). Those arcs will be transferred + // from chunk_clat later on. + CompactLattice::StateId src_state = p.first; + int32 arc_pos = p.second; + + if (arc_pos >= (int32)clat_.NumArcs(src_state)) + continue; + fst::MutableArcIterator aiter(&clat_, src_state); + aiter.Seek(arc_pos); + if (aiter.Value().nextstate != clat_state) + continue; // This arc record has become invalidated. + CompactLatticeArc new_in_arc(aiter.Value()); + // In most cases we will have dest_clat_state == clat_state, so the next + // line won't change the value of .nextstate + new_in_arc.nextstate = dest_clat_state; + new_in_arc.weight = fst::Times(new_in_arc.weight, extra_weight_in); + aiter.SetValue(new_in_arc); + + BaseFloat new_forward_cost = forward_costs_[src_state] + + ConvertToCost(new_in_arc.weight); + if (new_forward_cost < forward_costs_[dest_clat_state]) + forward_costs_[dest_clat_state] = new_forward_cost; + arcs_in_[dest_clat_state].push_back(p); + } + } + return false; // this is not the first chunk. +} + +void LatticeIncrementalDeterminizer::TransferArcsToClat( + const CompactLattice &chunk_clat, + bool is_first_chunk, + const std::unordered_map &state_map, + const std::unordered_map &chunk_state_to_token, + const std::unordered_map &old_final_costs) { + using StateId = CompactLattice::StateId; + StateId chunk_num_states = chunk_clat.NumStates(); + + // Now transfer arcs from chunk_clat to clat_. + for (StateId chunk_state = (is_first_chunk ? 0 : 1); + chunk_state < chunk_num_states; chunk_state++) { + auto iter = state_map.find(chunk_state); + if (iter == state_map.end()) { + KALDI_ASSERT(chunk_state_to_token.count(chunk_state) != 0); + // Don't process token-final states. Anyway they have no arcs leaving + // them. + continue; + } + StateId clat_state = iter->second; + + // We know that this point that `clat_state` is not a token-final state + // (see glossary for definition) as if it were, we would have done + // `continue` above. + // + // Only in the last chunk of the lattice would be there be a final-prob on + // states that are not `token-final states`; these final-probs would + // normally all be Zero() at this point. So in almost all cases the following + // call will do nothing. + clat_.SetFinal(clat_state, chunk_clat.Final(chunk_state)); + + // Process arcs leaving this state. + for (fst::ArcIterator aiter(chunk_clat, chunk_state); + !aiter.Done(); aiter.Next()) { + CompactLatticeArc arc(aiter.Value()); + + auto next_iter = state_map.find(arc.nextstate); + if (next_iter != state_map.end()) { + // The normal case (when the .nextstate has a corresponding + // state in clat_) is very simple. Just copy the arc over. + arc.nextstate = next_iter->second; + KALDI_ASSERT(arc.ilabel < kTokenLabelOffset || + arc.ilabel > kMaxTokenLabel); + AddArcToClat(clat_state, arc); + } else { + // This is the case when the arc is to a `token-final` state (see + // glossary.) + + // TODO: remove the following slightly excessive assertion? + KALDI_ASSERT(chunk_clat.Final(arc.nextstate) != CompactLatticeWeight::Zero() && + arc.olabel >= (Label)kTokenLabelOffset && + arc.olabel < (Label)kMaxTokenLabel && + chunk_state_to_token.count(arc.nextstate) != 0 && + old_final_costs.count(arc.olabel) != 0); + + // Include the final-cost of the next state (which should be final) + // in arc.weight. + arc.weight = fst::Times(arc.weight, + chunk_clat.Final(arc.nextstate)); + + auto cost_iter = old_final_costs.find(arc.olabel); + KALDI_ASSERT(cost_iter != old_final_costs.end()); + BaseFloat old_final_cost = cost_iter->second; + + // `arc` is going to become an element of final_arcs_. These + // contain information about transitions from states in clat_ to + // `token-final` states (i.e. states that have a token-label on the arc + // to them and that are final in the canonical compact lattice). + // We subtract the old_final_cost as it was just a temporary cost + // introduced for pruning purposes. + arc.weight.SetWeight(fst::Times(arc.weight.Weight(), + LatticeWeight{-old_final_cost, 0.0})); + // In a slight abuse of the Arc data structure, the nextstate is set to + // the source state. The label (ilabel == olabel) indicates the + // token it is associated with. + arc.nextstate = clat_state; + final_arcs_.push_back(arc); + } + } + } + +} + +bool LatticeIncrementalDeterminizer::AcceptRawLatticeChunk( + Lattice *raw_fst) { + using Label = CompactLatticeArc::Label; + using StateId = CompactLattice::StateId; + + // old_final_costs is a map from a `token-label` (see glossary) to the + // associated final-prob in a final-state of `raw_fst`, that is associated + // with that Token. These are Tokens that were active at the end of the + // chunk. The final-probs may arise from beta (backward) costs, introduced + // for pruning purposes, and/or from final-probs in HCLG. Those costs will + // not be included in anything we store permamently in this class; they used + // only to guide pruned determinization, and we will use `old_final_costs` + // later to cancel them out. + std::unordered_map old_final_costs; + GetRawLatticeFinalCosts(*raw_fst, &old_final_costs); + + CompactLattice chunk_clat; + bool determinized_till_beam = DeterminizeLatticePhonePrunedWrapper( + trans_model_, raw_fst, config_.lattice_beam, &chunk_clat, + config_.det_opts); + + TopSortCompactLatticeIfNeeded(&chunk_clat); + + std::unordered_map chunk_state_to_token; + IdentifyTokenFinalStates(chunk_clat, + &chunk_state_to_token); + + StateId chunk_num_states = chunk_clat.NumStates(); + if (chunk_num_states == 0) { + // This will be an error but user-level calling code can detect it from the + // lattice being empty. + KALDI_WARN << "Empty lattice, something went wrong."; + clat_.DeleteStates(); + return false; + } + + StateId start_state = chunk_clat.Start(); // would be 0. + KALDI_ASSERT(start_state == 0); + + // Process arcs leaving the start state of chunk_clat. Unless this is the + // first chunk in the lattice, all arcs leaving the start state of chunk_clat + // will have `state labels` on them (identifying redeterminized-states in + // clat_), and will transition to a state in `chunk_clat` that we can identify + // with that redeterminized-state. + + // state_map maps from (non-initial, non-token-final state s in chunk_clat) to + // a state in clat_. + std::unordered_map state_map; + + + bool is_first_chunk = ProcessArcsFromChunkStartState(chunk_clat, &state_map); + + // Remove any existing arcs in clat_ that leave redeterminized-states, and + // make those states non-final. Below, we'll add arcs leaving those states + // (and possibly new final-probs.) + for (StateId clat_state: non_final_redet_states_) { + clat_.DeleteArcs(clat_state); + clat_.SetFinal(clat_state, CompactLatticeWeight::Zero()); + } + + // The previous final-arc info is no longer relevant; we'll recreate it below. + final_arcs_.clear(); + + // assume chunk_lat.Start() == 0; we asserted it above. Allocate state-ids + // for all remaining states in chunk_clat, except for token-final states. + for (StateId state = (is_first_chunk ? 0 : 1); + state < chunk_num_states; state++) { + if (chunk_state_to_token.count(state) != 0) + continue; // these `token-final` states don't get a state allocated. + + StateId new_clat_state = clat_.NumStates(); + if (state_map.insert({state, new_clat_state}).second) { + // If it was inserted then we need to actually allocate that state + StateId s = AddStateToClat(); + KALDI_ASSERT(s == new_clat_state); + } // else do nothing; it would have been a redeterminized-state and no + } // allocation is needed since they already exist in clat_. and + // in state_map. + + if (is_first_chunk) { + auto iter = state_map.find(start_state); + KALDI_ASSERT(iter != state_map.end()); + CompactLattice::StateId clat_start_state = iter->second; + KALDI_ASSERT(clat_start_state == 0); // topological order. + clat_.SetStart(clat_start_state); + forward_costs_[clat_start_state] = 0.0; + } + + TransferArcsToClat(chunk_clat, is_first_chunk, + state_map, chunk_state_to_token, old_final_costs); + + GetNonFinalRedetStates(); + + return determinized_till_beam; +} + + + +void LatticeIncrementalDeterminizer::SetFinalCosts( + const unordered_map *token_label2final_cost) { + if (final_arcs_.empty()) { + KALDI_WARN << "SetFinalCosts() called when final_arcs_.empty()... possibly " + "means you are calling this after Finalize()? Not allowed: could " + "indicate a code error. Or possibly decoding failed somehow."; + } + + /* + prefinal states a terminology that does not appear in the paper. What it + means is: the set of states that have an arc with a Token-label as the label + leaving them in the canonical appended lattice. + */ + std::unordered_set &prefinal_states(temp_); + prefinal_states.clear(); + for (const auto &arc: final_arcs_) { + /* Caution: `state` is actually the state the arc would + leave from in the canonical appended lattice; we just store + that in the .nextstate field. */ + CompactLattice::StateId state = arc.nextstate; + prefinal_states.insert(state); + } + + for (int32 state: prefinal_states) + clat_.SetFinal(state, CompactLatticeWeight::Zero()); + + + for (const CompactLatticeArc &arc: final_arcs_) { + Label token_label = arc.ilabel; + /* Note: we store the source state in the .nextstate field. */ + CompactLattice::StateId src_state = arc.nextstate; + BaseFloat graph_final_cost; + if (token_label2final_cost == NULL) { + graph_final_cost = 0.0; + } else { + auto iter = token_label2final_cost->find(token_label); + if (iter == token_label2final_cost->end()) + continue; + else + graph_final_cost = iter->second; + } + /* It might seem odd to set a final-prob on the src-state of the arc.. + the point is that the symbol on the arc is a token-label, which should not + appear in the lattice the user sees, so after that token-label is removed + the arc would just become a final-prob. + */ + clat_.SetFinal(src_state, + fst::Plus(clat_.Final(src_state), + fst::Times(arc.weight, + CompactLatticeWeight( + LatticeWeight(graph_final_cost, 0), {})))); + } +} + + + + +// Instantiate the template for the combination of token types and FST types +// that we'll need. +template class LatticeIncrementalDecoderTpl, decoder::StdToken>; +template class LatticeIncrementalDecoderTpl, + decoder::StdToken>; +template class LatticeIncrementalDecoderTpl, + decoder::StdToken>; +template class LatticeIncrementalDecoderTpl; +template class LatticeIncrementalDecoderTpl; + +template class LatticeIncrementalDecoderTpl, + decoder::BackpointerToken>; +template class LatticeIncrementalDecoderTpl, + decoder::BackpointerToken>; +template class LatticeIncrementalDecoderTpl, + decoder::BackpointerToken>; +template class LatticeIncrementalDecoderTpl; +template class LatticeIncrementalDecoderTpl; + +} // end namespace kaldi. diff --git a/runtime/onnxruntime/third_party/kaldi/decoder/lattice-incremental-decoder.h b/runtime/onnxruntime/third_party/kaldi/decoder/lattice-incremental-decoder.h new file mode 100644 index 000000000..575a926aa --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/decoder/lattice-incremental-decoder.h @@ -0,0 +1,734 @@ +// decoder/lattice-incremental-decoder.h + +// Copyright 2019 Zhehuai Chen, Daniel Povey + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#ifndef KALDI_DECODER_LATTICE_INCREMENTAL_DECODER_H_ +#define KALDI_DECODER_LATTICE_INCREMENTAL_DECODER_H_ + +#include "util/stl-utils.h" +#include "util/hash-list.h" +#include "fst/fstlib.h" +#include "itf/decodable-itf.h" +#include "fstext/fstext-lib.h" +#include "lat/determinize-lattice-pruned.h" +#include "lat/kaldi-lattice.h" +#include "decoder/grammar-fst.h" +#include "decoder/lattice-faster-decoder.h" + +namespace kaldi { +/** + The normal decoder, lattice-faster-decoder.h, sometimes has an issue when + doing real-time applications with long utterances, that each time you get the + lattice the lattice determinization can take a considerable amount of time; + this introduces latency. This version of the decoder spreads the work of + lattice determinization out throughout the decoding process. + + NOTE: + + Please see https://www.danielpovey.com/files/ *TBD* .pdf for a technical + explanation of what is going on here. + + GLOSSARY OF TERMS: + chunk: We do the determinization on chunks of frames; these + may coincide with the chunks on which the user calls + AdvanceDecoding(). The basic idea is to extract chunks + of the raw lattice and determinize them individually, but + it gets much more complicated than that. The chunks + should normally be at least as long as a word (let's say, + at least 20 frames), or the overhead of this algorithm + might become excessive and affect RTF. + + raw lattice chunk: A chunk of raw (i.e. undeterminized) lattice + that we will determinize. In the paper this corresponds + to the FST B that is described in Section 5.2. + + token_label, state_label / token-label, state-label: + + In the paper these are both referred to as `state labels` (these are + special, large integer id's that refer to states in the undeterminized + lattice and in the the determinized lattice); but we use two separate + terms here, for more clarity, when referring to the undeterminized + vs. determinized lattice. + + token_label conceptually refers to states in the + raw lattice, but we don't materialize the entire + raw lattice as a physical FST and and these tokens + are actually tokens (template type Token) held by + the decoder + + state_label when used in this code refers specifically + to labels that identify states in the determinized + lattice (i.e. state indexes in lat_). + + token-final state + A state in a raw lattice or in a determinized chunk that has an arc + entering it that has a `token-label` on it (as defined above). + These states will have nonzero final-probs. + + redeterminized-non-splice-state, aka ns_redet: + A redeterminized state which is not also a splice state; + refer to the paper for explanation. In the already-determinized + part this means a redeterminized state which is not final. + + canonical appended lattice: This is the appended compact lattice + that we conceptually have (i.e. what we described in the paper). + The difference from the "actual appended lattice" stored + in LatticeIncrementalDeterminizer::clat_ is that the + actual appended lattice has all its final-arcs replaced with + final-probs, and we keep the real final-arcs "on the side" in a + separate data structure. The final-probs in clat_ aren't + necessarily related to the costs on the final-arcs; instead + they can have arbitrary values passed in by the user (e.g. + if we want to include final-probs). This means that the + clat_ can be returned without modification to the user who wants + a partially determinized result. + + final-arc: An arc in the canonical appended CompactLattice which + goes to a final-state. These arcs will have `state-labels` as + their labels. + + */ +struct LatticeIncrementalDecoderConfig { + // All the configuration values until det_opts are the same as in + // LatticeFasterDecoder. For clarity we repeat them rather than inheriting. + BaseFloat beam; + int32 max_active; + int32 min_active; + BaseFloat lattice_beam; + int32 prune_interval; + BaseFloat beam_delta; // has nothing to do with beam_ratio + BaseFloat hash_ratio; + BaseFloat prune_scale; // Note: we don't make this configurable on the command line, + // it's not a very important parameter. It affects the + // algorithm that prunes the tokens as we go. + // Most of the options inside det_opts are not actually queried by the + // LatticeIncrementalDecoder class itself, but by the code that calls it, for + // example in the function DecodeUtteranceLatticeIncremental. + fst::DeterminizeLatticePhonePrunedOptions det_opts; + + // The configuration values from this point on are specific to the + // incremental determinization. See where they are registered for + // explanation. + // Caution: these are only inspected in UpdateLatticeDeterminization(). + // If you call + int32 determinize_max_delay; + int32 determinize_min_chunk_size; + int32 determinize_max_active; + + + LatticeIncrementalDecoderConfig() + : beam(16.0), + max_active(std::numeric_limits::max()), + min_active(200), + lattice_beam(10.0), + prune_interval(25), + beam_delta(0.5), + hash_ratio(2.0), + prune_scale(0.01), + determinize_max_delay(60), + determinize_min_chunk_size(20), + determinize_max_active(200) { + det_opts.minimize = false; + } + void Register(OptionsItf *opts) { + det_opts.Register(opts); + opts->Register("beam", &beam, "Decoding beam. Larger->slower, more accurate."); + opts->Register("max-active", &max_active, + "Decoder max active states. Larger->slower; " + "more accurate"); + opts->Register("min-active", &min_active, "Decoder minimum #active states."); + opts->Register("lattice-beam", &lattice_beam, + "Lattice generation beam. Larger->slower, " + "and deeper lattices"); + opts->Register("prune-interval", &prune_interval, + "Interval (in frames) at " + "which to prune tokens"); + opts->Register("beam-delta", &beam_delta, + "Increment used in decoding-- this " + "parameter is obscure and relates to a speedup in the way the " + "max-active constraint is applied. Larger is more accurate."); + opts->Register("hash-ratio", &hash_ratio, + "Setting used in decoder to " + "control hash behavior"); + opts->Register("determinize-max-delay", &determinize_max_delay, + "Maximum frames of delay between decoding a frame and " + "determinizing it"); + opts->Register("determinize-min-chunk-size", &determinize_min_chunk_size, + "Minimum chunk size used in determinization"); + opts->Register("determinize-max-active", &determinize_max_active, + "Maximum number of active tokens to update determinization"); + + } + void Check() const { + if (!(beam > 0.0 && max_active > 1 && lattice_beam > 0.0 && + min_active <= max_active && prune_interval > 0 && + beam_delta > 0.0 && hash_ratio >= 1.0 && + prune_scale > 0.0 && prune_scale < 1.0 && + determinize_max_delay > determinize_min_chunk_size && + determinize_min_chunk_size > 0 && + determinize_max_active >= 0)) + KALDI_ERR << "Invalid options given to decoder"; + /* Minimization of the chunks is not compatible withour algorithm (or at + least, would require additional complexity to implement.) */ + if (det_opts.minimize || !det_opts.word_determinize) + KALDI_ERR << "Invalid determinization options given to decoder."; + } +}; + + + +/** + This class is used inside LatticeIncrementalDecoderTpl; it handles + some of the details of incremental determinization. + https://www.danielpovey.com/files/ *TBD*.pdf for the paper. + +*/ +class LatticeIncrementalDeterminizer { + public: + using Label = typename LatticeArc::Label; /* Actualy the same labels appear + in both lattice and compact + lattice, so we don't use the + specific type all the time but + just say 'Label' */ + LatticeIncrementalDeterminizer( + const TransitionInformation &trans_model, + const LatticeIncrementalDecoderConfig &config): + trans_model_(trans_model), config_(config) { } + + // Resets the lattice determinization data for new utterance + void Init(); + + // Returns the current determinized lattice. + const CompactLattice &GetDeterminizedLattice() const { return clat_; } + + /** + Starts the process of creating a raw lattice chunk. (Search the glossary + for "raw lattice chunk"). This just sets up the initial states and + redeterminized-states in the chunk. Relates to sec. 5.2 in the paper, + specifically the initial-state i and the redeterminized-states. + + After calling this, the caller would add the remaining arcs and states + to `olat` and then call AcceptRawLatticeChunk() with the result. + + @param [out] olat The lattice to be (partially) created + + @param [out] token_label2state This function outputs to here + a map from `token-label` to the state we created for + it in *olat. See glossary for `token-label`. + The keys actually correspond to the .nextstate fields + in the arcs in final_arcs_; values are states in `olat`. + See the last bullet point before Sec. 5.3 in the paper. + */ + void InitializeRawLatticeChunk( + Lattice *olat, + unordered_map *token_label2state); + + /** + This function accepts the raw FST (state-level lattice) corresponding to a + single chunk of the lattice, determinizes it and appends it to this->clat_. + Unless this was the + + Note: final-probs in `raw_fst` are treated specially: they are used to + guide the pruned determinization, but when you call GetLattice() it will be + -- except for pruning effects-- as if all nonzero final-probs in `raw_fst` + were: One() if final_costs == NULL; else the value present in `final_costs`. + + @param [in] raw_fst (Consumed destructively). The input + raw (state-level) lattice. Would correspond to the + FST A in the paper if first_frame == 0, and B + otherwise. + + @return returns false if determinization finished earlier than the beam + or the determinized lattice was empty; true otherwise. + + NOTE: if this is not the final chunk, you will probably want to call + SetFinalCosts() directly after calling this. + */ + bool AcceptRawLatticeChunk(Lattice *raw_fst); + + /* + Sets final-probs in `clat_`. Must only be called if the final chunk + has not been processed. (The final chunk is whenever GetLattice() is + called with finalize == true). + + The reason this is a separate function from AcceptRawLatticeChunk() is that + there may be situations where a user wants to get the latice with + final-probs in it, after previously getting it without final-probs; or + vice versa. By final-probs, we mean the Final() probabilities in the + HCLG (decoding graph; this->fst_). + + @param [in] token_label2final_cost A map from the token-label + corresponding to Tokens active on the final frame of the + lattice in the object, to the final-cost we want to use for + those tokens. If NULL, it means all Tokens should be treated + as final with probability One(). If non-NULL, and a particular + token-label is not a key of this map, it means that Token + corresponded to a state that was not final in HCLG; and + such tokens will be treated as non-final. However, + if this would result in no states in the lattice being final, + we will treat all Tokens as final with probability One(), + a warning will be printed (this should not happen.) + */ + void SetFinalCosts(const unordered_map *token_label2final_cost = NULL); + + const CompactLattice &GetLattice() { return clat_; } + + // kStateLabelOffset is what we add to state-ids in clat_ to produce labels + // to identify them in the raw lattice chunk + // kTokenLabelOffset is where we start allocating labels corresponding to Tokens + // (these correspond with raw lattice states); + enum { kStateLabelOffset = (int)1e8, kTokenLabelOffset = (int)2e8, kMaxTokenLabel = (int)3e8 }; + + private: + + // [called from AcceptRawLatticeChunk()] + // Gets the final costs from token-final states in the raw lattice (see + // glossary for definition). These final costs will be subtracted after + // determinization; in the normal case they are `temporaries` used to guide + // pruning. NOTE: the index of the array is not the FST state that is final, + // but the label on arcs entering it (these will be `token-labels`). Each + // token-final state will have the same label on all arcs entering it. + // + // `old_final_costs` is assumed to be empty at entry. + void GetRawLatticeFinalCosts(const Lattice &raw_fst, + std::unordered_map *old_final_costs); + + // Sets up non_final_redet_states_. See documentation for that variable. + void GetNonFinalRedetStates(); + + /** [called from AcceptRawLatticeChunk()] Processes arcs that leave the + start-state of `chunk_clat` (if this is not the first chunk); does nothing + if this is the first chunk. This includes using the `state-labels` to + work out which states in clat_ these states correspond to, and writing + that mapping to `state_map`. + + Also modifies forward_costs_, because it has to do a kind of reweighting + of the clat states that are the values it puts in `state_map`, to take + account of the probabilities on the arcs from the start state of + chunk_clat to the states corresponding to those redeterminized-states + (i.e. the states in clat corresponding to the values it puts in + `*state_map`). It also modifies arcs_in_, mostly because there + are rare cases when we end up `merging` sets of those redeterminized-states, + because the determinization process mapped them to a single state, + and that means we need to reroute the arcs into members of that + set into one single member (which will appear as a value in + `*state_map`). + + @param [in] chunk_clat The determinized chunk of lattice we are + processing + @param [out] state_map Mapping from states in chunk_clat to + the state in clat_ they correspond to. + @return Returns true if this is the first chunk. + */ + bool ProcessArcsFromChunkStartState( + const CompactLattice &chunk_clat, + std::unordered_map *state_map); + + /** + This function, called from AcceptRawLatticeChunk(), transfers arcs from + `chunk_clat` to clat_. For those arcs that have `token-labels` on them, + they don't get written to clat_ but instead are stored in the arcs_ array. + + @param [in] chunk_clat The determinized lattice for the chunk + we are processing; this is the source of the arcs + we are moving. + @param [in] is_first_chunk True if this is the first chunk in the + utterance; it's needed because if it is, we + will also transfer arcs from the start state of + chunk_clat. + @param [in] state_map Map from state-ids in chunk_clat to state-ids + in clat_. + @param [in] chunk_state_to_token Map from `token-final states` + (see glossary) in chunk_clat, to the token-label + on arcs entering those states. + @param [in] old_final_costs Map from token-label to the + final-costs that were on the corresponding + token-final states in the undeterminized lattice; + these final-costs need to be removed when + we record the weights in final_arcs_, because + they were just temporary. + */ + void TransferArcsToClat( + const CompactLattice &chunk_clat, + bool is_first_chunk, + const std::unordered_map &state_map, + const std::unordered_map &chunk_state_to_token, + const std::unordered_map &old_final_costs); + + + + /** + Adds one arc to `clat_`. It's like clat_.AddArc(state, arc), except + it also modifies arcs_in_ and forward_costs_. + */ + void AddArcToClat(CompactLattice::StateId state, + const CompactLatticeArc &arc); + CompactLattice::StateId AddStateToClat(); + + + // Identifies token-final states in `chunk_clat`; see glossary above for + // definition of `token-final`. This function outputs a map from such states + // in chunk_clat, to the `token-label` on arcs entering them. (It is not + // possible that the same state would have multiple arcs entering it with + // different token-labels, or some arcs entering with one token-label and some + // another, or be both initial and have such arcs; this is true due to how we + // construct the raw lattice.) + void IdentifyTokenFinalStates( + const CompactLattice &chunk_clat, + std::unordered_map *token_map) const; + + // trans_model_ is needed by DeterminizeLatticePhonePrunedWrapper() which this + // class calls. + const TransitionInformation &trans_model_; + // config_ is needed by DeterminizeLatticePhonePrunedWrapper() which this + // class calls. + const LatticeIncrementalDecoderConfig &config_; + + + // Contains the set of redeterminized-states which are not final in the + // canonical appended lattice. Since the final ones don't physically appear + // in clat_, this means the set of redeterminized-states which are physically + // in clat_. In code terms, this means set of .first elements in final_arcs, + // plus whatever other states in clat_ are reachable from such states. + std::unordered_set non_final_redet_states_; + + + // clat_ is the appended lattice (containing all chunks processed so + // far), except its `final-arcs` (i.e. arcs which in the canonical + // lattice would go to final-states) are not present (they are stored + // separately in final_arcs_) and states which in the canonical lattice + // should have final-arcs leaving them will instead have a final-prob. + CompactLattice clat_; + + + // arcs_in_ is indexed by (state-id in clat_), and is a list of + // arcs that come into this state, in the form (prev-state, + // arc-index). CAUTION: not all these input-arc records will always + // be valid (some may be out-of-date, and may refer to an out-of-range + // arc or an arc that does not point to this state). But all + // input arcs will always be listed. + std::vector > > arcs_in_; + + // final_arcs_ contains arcs which would appear in the canonical appended + // lattice but for implementation reasons are not physically present in clat_. + // These are arcs to final states in the canonical appended lattice. The + // .first elements are the source states in clat_ (these will all be elements + // of non_final_redet_states_); the .nextstate elements of the arcs does not + // contain a physical state, but contain state-labels allocated by + // AllocateNewStateLabel(). + std::vector final_arcs_; + + // forward_costs_, indexed by the state-id in clat_, stores the alpha + // (forward) costs, i.e. the minimum cost from the start state to each state + // in clat_. This is relevant for pruned determinization. The BaseFloat can + // be thought of as the sum of a Value1() + Value2() in a LatticeWeight. + std::vector forward_costs_; + + // temporary used in a function, kept here to avoid excessive reallocation. + std::unordered_set temp_; + + KALDI_DISALLOW_COPY_AND_ASSIGN(LatticeIncrementalDeterminizer); +}; + + +/** This is an extention to the "normal" lattice-generating decoder. + See \ref lattices_generation \ref decoders_faster and \ref decoders_simple + for more information. + + The main difference is the incremental determinization which will be + discussed in the function GetLattice(). This means that the work of determinizatin + isn't done all at once at the end of the file, but incrementally while decoding. + See the comment at the top of this file for more explanation. + + The decoder is templated on the FST type and the token type. The token type + will normally be StdToken, but also may be BackpointerToken which is to support + quick lookup of the current best path (see lattice-faster-online-decoder.h) + + The FST you invoke this decoder with is expected to be of type + Fst::Fst, a.k.a. StdFst, or GrammarFst. If you invoke it with + FST == StdFst and it notices that the actual FST type is + fst::VectorFst or fst::ConstFst, the decoder object + will internally cast itself to one that is templated on those more specific + types; this is an optimization for speed. + */ +template +class LatticeIncrementalDecoderTpl { + public: + using Arc = typename FST::Arc; + using Label = typename Arc::Label; + using StateId = typename Arc::StateId; + using Weight = typename Arc::Weight; + using ForwardLinkT = decoder::ForwardLink; + + // Instantiate this class once for each thing you have to decode. + // This version of the constructor does not take ownership of + // 'fst'. + LatticeIncrementalDecoderTpl(const FST &fst, const TransitionInformation &trans_model, + const LatticeIncrementalDecoderConfig &config); + + // This version of the constructor takes ownership of the fst, and will delete + // it when this object is destroyed. + LatticeIncrementalDecoderTpl(const LatticeIncrementalDecoderConfig &config, + FST *fst, const TransitionInformation &trans_model); + + void SetOptions(const LatticeIncrementalDecoderConfig &config) { config_ = config; } + + const LatticeIncrementalDecoderConfig &GetOptions() const { return config_; } + + ~LatticeIncrementalDecoderTpl(); + + /** + CAUTION: it's unlikely that you will ever want to call this function. In a + scenario where you have the entire file and just want to decode it, there + is no point using this decoder. + + An example of how to do decoding together with incremental + determinization. It decodes until there are no more frames left in the + "decodable" object. + + In this example, config_.determinize_max_delay, config_.determinize_min_chunk_size + and config_.determinize_max_active are used to determine the time to + call GetLattice(). + + Users will probably want to use appropriate combinations of + AdvanceDecoding() and GetLattice() to build their application; this just + gives you some idea how. + + The function returns true if any kind of traceback is available (not + necessarily from a final state). + */ + bool Decode(DecodableInterface *decodable); + + /// says whether a final-state was active on the last frame. If it was not, + /// the lattice (or traceback) will end with states that are not final-states. + bool ReachedFinal() const { + return FinalRelativeCost() != std::numeric_limits::infinity(); + } + + /** + This decoder has no GetBestPath() function. + If you need that functionality you should probably use lattice-incremental-online-decoder.h, + which makes it very efficient to obtain the best path. */ + + /** + This GetLattice() function returns the lattice containing + `num_frames_to_decode` frames; this will be all frames decoded so + far, if you let num_frames_to_decode == NumFramesDecoded(), + but it will generally be better to make it a few frames less than + that to avoid the lattice having too many active states at + the end. + + @param [in] num_frames_to_include The number of frames that you want + to be included in the lattice. Must be >= + NumFramesInLattice() and <= NumFramesDecoded(). + + @param [in] use_final_probs True if you want the final-probs + of HCLG to be included in the output lattice. Must not + be set to true if num_frames_to_include != + NumFramesDecoded(). Must be set to true if you have + previously called FinalizeDecoding(). + + (If no state was final on frame `num_frames_to_include`, the + final-probs won't be included regardless of + `use_final_probs`; you can test whether this + was the case by calling ReachedFinal(). + + @return clat The CompactLattice representing what has been decoded + up until `num_frames_to_include` (e.g., LatticeStateTimes() + on this lattice would return `num_frames_to_include`). + + See also UpdateLatticeDeterminizaton(). Caution: this const ref + is only valid until the next time you call AdvanceDecoding() or + GetLattice(). + + CAUTION: the lattice may contain disconnnected states; you should + call Connect() on the output before writing it out. + */ + const CompactLattice &GetLattice(int32 num_frames_to_include, + bool use_final_probs = false); + + /* + Returns the number of frames in the currently-determinized part of the + lattice which will be a number in [0, NumFramesDecoded()]. It will + be the largest number that GetLattice() was called with, but note + that GetLattice() may be called from UpdateLatticeDeterminization(). + + Made available in case the user wants to give that same number to + GetLattice(). + */ + int NumFramesInLattice() const { return num_frames_in_lattice_; } + + /** + InitDecoding initializes the decoding, and should only be used if you + intend to call AdvanceDecoding(). If you call Decode(), you don't need to + call this. You can also call InitDecoding if you have already decoded an + utterance and want to start with a new utterance. + */ + void InitDecoding(); + + /** + This will decode until there are no more frames ready in the decodable + object. You can keep calling it each time more frames become available + (this is the normal pattern in a real-time/online decoding scenario). + If max_num_frames is specified, it specifies the maximum number of frames + the function will decode before returning. + */ + void AdvanceDecoding(DecodableInterface *decodable, int32 max_num_frames = -1); + + + /** FinalRelativeCost() serves the same purpose as ReachedFinal(), but gives + more information. It returns the difference between the best (final-cost + plus cost) of any token on the final frame, and the best cost of any token + on the final frame. If it is infinity it means no final-states were + present on the final frame. It will usually be nonnegative. If it not + too positive (e.g. < 5 is my first guess, but this is not tested) you can + take it as a good indication that we reached the final-state with + reasonable likelihood. */ + BaseFloat FinalRelativeCost() const; + + /** Returns the number of frames decoded so far. */ + inline int32 NumFramesDecoded() const { return active_toks_.size() - 1; } + + /** + Finalizes the decoding, doing an extra pruning step on the last frame + that uses the final-probs. May be called only once. + */ + void FinalizeDecoding(); + + protected: + /* Some protected things are needed in LatticeIncrementalOnlineDecoderTpl. */ + + /** NOTE: for parts the internal implementation that are shared with LatticeFasterDecoer, + we have removed the comments.*/ + inline static void DeleteForwardLinks(Token *tok); + struct TokenList { + Token *toks; + bool must_prune_forward_links; + bool must_prune_tokens; + int32 num_toks; /* Note: you can only trust `num_toks` if must_prune_tokens + * == false, because it is only set in + * PruneTokensForFrame(). */ + TokenList() + : toks(NULL), must_prune_forward_links(true), must_prune_tokens(true), + num_toks(-1) {} + }; + using Elem = typename HashList::Elem; + void PossiblyResizeHash(size_t num_toks); + inline Token *FindOrAddToken(StateId state, int32 frame_plus_one, + BaseFloat tot_cost, Token *backpointer, bool *changed); + void PruneForwardLinks(int32 frame_plus_one, bool *extra_costs_changed, + bool *links_pruned, BaseFloat delta); + void ComputeFinalCosts(unordered_map *final_costs, + BaseFloat *final_relative_cost, + BaseFloat *final_best_cost) const; + void PruneForwardLinksFinal(); + void PruneTokensForFrame(int32 frame_plus_one); + void PruneActiveTokens(BaseFloat delta); + BaseFloat GetCutoff(Elem *list_head, size_t *tok_count, BaseFloat *adaptive_beam, + Elem **best_elem); + BaseFloat ProcessEmitting(DecodableInterface *decodable); + void ProcessNonemitting(BaseFloat cost_cutoff); + + HashList toks_; + std::vector active_toks_; // indexed by frame. + std::vector queue_; // temp variable used in ProcessNonemitting, + std::vector tmp_array_; // used in GetCutoff. + const FST *fst_; + bool delete_fst_; + std::vector cost_offsets_; + int32 num_toks_; + bool warned_; + bool decoding_finalized_; + + unordered_map final_costs_; + BaseFloat final_relative_cost_; + BaseFloat final_best_cost_; + + /*********************** + Variables below this point relate to the incremental + determinization. + *********************/ + LatticeIncrementalDecoderConfig config_; + /** Much of the the incremental determinization algorithm is encapsulated in + the determinize_ object. */ + LatticeIncrementalDeterminizer determinizer_; + + + /* Just a temporary used in a function; stored here to avoid reallocation. */ + unordered_map temp_token_map_; + + /** num_frames_in_lattice_ is the highest `num_frames_to_include_` argument + for any prior call to GetLattice(). */ + int32 num_frames_in_lattice_; + + // A map from Token to its token_label. Will contain an entry for + // each Token in active_toks_[num_frames_in_lattice_]. + unordered_map token2label_map_; + + // A temporary used in a function, kept here to avoid reallocation. + unordered_map token2label_map_temp_; + + // we allocate a unique id for each Token + Label next_token_label_; + + inline Label AllocateNewTokenLabel() { return next_token_label_++; } + + + // There are various cleanup tasks... the the toks_ structure contains + // singly linked lists of Token pointers, where Elem is the list type. + // It also indexes them in a hash, indexed by state (this hash is only + // maintained for the most recent frame). toks_.Clear() + // deletes them from the hash and returns the list of Elems. The + // function DeleteElems calls toks_.Delete(elem) for each elem in + // the list, which returns ownership of the Elem to the toks_ structure + // for reuse, but does not delete the Token pointer. The Token pointers + // are reference-counted and are ultimately deleted in PruneTokensForFrame, + // but are also linked together on each frame by their own linked-list, + // using the "next" pointer. We delete them manually. + void DeleteElems(Elem *list); + + void ClearActiveTokens(); + + + // Returns the number of active tokens on frame `frame`. Can be used as part + // of a heuristic to decide which frame to determinize until, if you are not + // at the end of an utterance. + int32 GetNumToksForFrame(int32 frame); + + /** + UpdateLatticeDeterminization() ensures the work of determinization is kept + up to date so that when you do need the lattice you can get it fast. It + uses the configuration values `determinize_max_delay`, `determinize_min_chunk_size` + and `determinize_max_active`, to decide whether and when to call + GetLattice(). You can safely call this as often as you want (e.g. after + each time you call AdvanceDecoding(); it won't do subtantially more work if + it is called frequently. + */ + void UpdateLatticeDeterminization(); + + + KALDI_DISALLOW_COPY_AND_ASSIGN(LatticeIncrementalDecoderTpl); +}; + +typedef LatticeIncrementalDecoderTpl + LatticeIncrementalDecoder; + + +} // end namespace kaldi. + +#endif diff --git a/runtime/onnxruntime/third_party/kaldi/decoder/lattice-incremental-online-decoder.cc b/runtime/onnxruntime/third_party/kaldi/decoder/lattice-incremental-online-decoder.cc new file mode 100644 index 000000000..cacac2c08 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/decoder/lattice-incremental-online-decoder.cc @@ -0,0 +1,158 @@ +// decoder/lattice-incremental-online-decoder.cc + +// Copyright 2019 Zhehuai Chen + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +// see note at the top of lattice-faster-decoder.cc, about how to maintain this +// file in sync with lattice-faster-decoder.cc + +#include "decoder/lattice-incremental-decoder.h" +#include "decoder/lattice-incremental-online-decoder.h" +#include "lat/lattice-functions.h" +#include "base/timer.h" + +namespace kaldi { + +// Outputs an FST corresponding to the single best path through the lattice. +template +bool LatticeIncrementalOnlineDecoderTpl::GetBestPath(Lattice *olat, + bool use_final_probs) const { + olat->DeleteStates(); + BaseFloat final_graph_cost; + BestPathIterator iter = BestPathEnd(use_final_probs, &final_graph_cost); + if (iter.Done()) + return false; // would have printed warning. + StateId state = olat->AddState(); + olat->SetFinal(state, LatticeWeight(final_graph_cost, 0.0)); + while (!iter.Done()) { + LatticeArc arc; + iter = TraceBackBestPath(iter, &arc); + arc.nextstate = state; + StateId new_state = olat->AddState(); + olat->AddArc(new_state, arc); + state = new_state; + } + olat->SetStart(state); + return true; +} + +template +typename LatticeIncrementalOnlineDecoderTpl::BestPathIterator LatticeIncrementalOnlineDecoderTpl::BestPathEnd( + bool use_final_probs, + BaseFloat *final_cost_out) const { + if (this->decoding_finalized_ && !use_final_probs) + KALDI_ERR << "You cannot call FinalizeDecoding() and then call " + << "BestPathEnd() with use_final_probs == false"; + KALDI_ASSERT(this->NumFramesDecoded() > 0 && + "You cannot call BestPathEnd if no frames were decoded."); + + unordered_map final_costs_local; + + const unordered_map &final_costs = + (this->decoding_finalized_ ? this->final_costs_ :final_costs_local); + if (!this->decoding_finalized_ && use_final_probs) + this->ComputeFinalCosts(&final_costs_local, NULL, NULL); + + // Singly linked list of tokens on last frame (access list through "next" + // pointer). + BaseFloat best_cost = std::numeric_limits::infinity(); + BaseFloat best_final_cost = 0; + Token *best_tok = NULL; + for (Token *tok = this->active_toks_.back().toks; + tok != NULL; tok = tok->next) { + BaseFloat cost = tok->tot_cost, final_cost = 0.0; + if (use_final_probs && !final_costs.empty()) { + // if we are instructed to use final-probs, and any final tokens were + // active on final frame, include the final-prob in the cost of the token. + typename unordered_map::const_iterator + iter = final_costs.find(tok); + if (iter != final_costs.end()) { + final_cost = iter->second; + cost += final_cost; + } else { + cost = std::numeric_limits::infinity(); + } + } + if (cost < best_cost) { + best_cost = cost; + best_tok = tok; + best_final_cost = final_cost; + } + } + if (best_tok == NULL) { // this should not happen, and is likely a code error or + // caused by infinities in likelihoods, but I'm not making + // it a fatal error for now. + KALDI_WARN << "No final token found."; + } + if (final_cost_out != NULL) + *final_cost_out = best_final_cost; + return BestPathIterator(best_tok, this->NumFramesDecoded() - 1); +} + + +template +typename LatticeIncrementalOnlineDecoderTpl::BestPathIterator LatticeIncrementalOnlineDecoderTpl::TraceBackBestPath( + BestPathIterator iter, LatticeArc *oarc) const { + KALDI_ASSERT(!iter.Done() && oarc != NULL); + Token *tok = static_cast(iter.tok); + int32 cur_t = iter.frame, step_t = 0; + if (tok->backpointer != NULL) { + // retrieve the correct forward link(with the best link cost) + BaseFloat best_cost = std::numeric_limits::infinity(); + ForwardLinkT *link; + for (link = tok->backpointer->links; + link != NULL; link = link->next) { + if (link->next_tok == tok) { // this is the a to "tok" + BaseFloat graph_cost = link->graph_cost, + acoustic_cost = link->acoustic_cost; + BaseFloat cost = graph_cost + acoustic_cost; + if (cost < best_cost) { + oarc->ilabel = link->ilabel; + oarc->olabel = link->olabel; + if (link->ilabel != 0) { + KALDI_ASSERT(static_cast(cur_t) < this->cost_offsets_.size()); + acoustic_cost -= this->cost_offsets_[cur_t]; + step_t = -1; + } else { + step_t = 0; + } + oarc->weight = LatticeWeight(graph_cost, acoustic_cost); + best_cost = cost; + } + } + } + if (link == NULL && + best_cost == std::numeric_limits::infinity()) { // Did not find correct link. + KALDI_ERR << "Error tracing best-path back (likely " + << "bug in token-pruning algorithm)"; + } + } else { + oarc->ilabel = 0; + oarc->olabel = 0; + oarc->weight = LatticeWeight::One(); // zero costs. + } + return BestPathIterator(tok->backpointer, cur_t + step_t); +} + +// Instantiate the template for the FST types that we'll need. +template class LatticeIncrementalOnlineDecoderTpl >; +template class LatticeIncrementalOnlineDecoderTpl >; +template class LatticeIncrementalOnlineDecoderTpl >; +template class LatticeIncrementalOnlineDecoderTpl; +template class LatticeIncrementalOnlineDecoderTpl; + +} // end namespace kaldi. diff --git a/runtime/onnxruntime/third_party/kaldi/decoder/lattice-incremental-online-decoder.h b/runtime/onnxruntime/third_party/kaldi/decoder/lattice-incremental-online-decoder.h new file mode 100644 index 000000000..c82abed4e --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/decoder/lattice-incremental-online-decoder.h @@ -0,0 +1,132 @@ +// decoder/lattice-incremental-online-decoder.h + +// Copyright 2019 Zhehuai Chen +// +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +// see note at the top of lattice-faster-decoder.h, about how to maintain this +// file in sync with lattice-faster-decoder.h + + +#ifndef KALDI_DECODER_LATTICE_INCREMENTAL_ONLINE_DECODER_H_ +#define KALDI_DECODER_LATTICE_INCREMENTAL_ONLINE_DECODER_H_ + +#include "util/stl-utils.h" +#include "util/hash-list.h" +#include "fst/fstlib.h" +#include "itf/decodable-itf.h" +#include "fstext/fstext-lib.h" +#include "lat/determinize-lattice-pruned.h" +#include "lat/kaldi-lattice.h" +#include "decoder/lattice-incremental-decoder.h" + +namespace kaldi { + + + +/** LatticeIncrementalOnlineDecoderTpl is as LatticeIncrementalDecoderTpl but also + supports an efficient way to get the best path (see the function + BestPathEnd()), which is useful in endpointing and in situations where you + might want to frequently access the best path. + + This is only templated on the FST type, since the Token type is required to + be BackpointerToken. Actually it only makes sense to instantiate + LatticeIncrementalDecoderTpl with Token == BackpointerToken if you do so indirectly via + this child class. + */ +template +class LatticeIncrementalOnlineDecoderTpl: + public LatticeIncrementalDecoderTpl { + public: + using Arc = typename FST::Arc; + using Label = typename Arc::Label; + using StateId = typename Arc::StateId; + using Weight = typename Arc::Weight; + using Token = decoder::BackpointerToken; + using ForwardLinkT = decoder::ForwardLink; + + // Instantiate this class once for each thing you have to decode. + // This version of the constructor does not take ownership of + // 'fst'. + LatticeIncrementalOnlineDecoderTpl(const FST &fst, + const TransitionInformation &trans_model, + const LatticeIncrementalDecoderConfig &config): + LatticeIncrementalDecoderTpl(fst, trans_model, config) { } + + // This version of the initializer takes ownership of 'fst', and will delete + // it when this object is destroyed. + LatticeIncrementalOnlineDecoderTpl(const LatticeIncrementalDecoderConfig &config, + FST *fst, + const TransitionInformation &trans_model): + LatticeIncrementalDecoderTpl(config, fst, trans_model) { } + + + struct BestPathIterator { + void *tok; + int32 frame; + // note, "frame" is the frame-index of the frame you'll get the + // transition-id for next time, if you call TraceBackBestPath on this + // iterator (assuming it's not an epsilon transition). Note that this + // is one less than you might reasonably expect, e.g. it's -1 for + // the nonemitting transitions before the first frame. + BestPathIterator(void *t, int32 f): tok(t), frame(f) { } + bool Done() { return tok == NULL; } + }; + + + /// Outputs an FST corresponding to the single best path through the lattice. + /// This is quite efficient because it doesn't get the entire raw lattice and find + /// the best path through it; instead, it uses the BestPathEnd and BestPathIterator + /// so it basically traces it back through the lattice. + /// Returns true if result is nonempty (using the return status is deprecated, + /// it will become void). If "use_final_probs" is true AND we reached the + /// final-state of the graph then it will include those as final-probs, else + /// it will treat all final-probs as one. + bool GetBestPath(Lattice *ofst, + bool use_final_probs = true) const; + + + + /// This function returns an iterator that can be used to trace back + /// the best path. If use_final_probs == true and at least one final state + /// survived till the end, it will use the final-probs in working out the best + /// final Token, and will output the final cost to *final_cost (if non-NULL), + /// else it will use only the forward likelihood, and will put zero in + /// *final_cost (if non-NULL). + /// Requires that NumFramesDecoded() > 0. + BestPathIterator BestPathEnd(bool use_final_probs, + BaseFloat *final_cost = NULL) const; + + + /// This function can be used in conjunction with BestPathEnd() to trace back + /// the best path one link at a time (e.g. this can be useful in endpoint + /// detection). By "link" we mean a link in the graph; not all links cross + /// frame boundaries, but each time you see a nonzero ilabel you can interpret + /// that as a frame. The return value is the updated iterator. It outputs + /// the ilabel and olabel, and the (graph and acoustic) weight to the "arc" pointer, + /// while leaving its "nextstate" variable unchanged. + BestPathIterator TraceBackBestPath( + BestPathIterator iter, LatticeArc *arc) const; + + KALDI_DISALLOW_COPY_AND_ASSIGN(LatticeIncrementalOnlineDecoderTpl); +}; + +typedef LatticeIncrementalOnlineDecoderTpl LatticeIncrementalOnlineDecoder; + + +} // end namespace kaldi. + +#endif diff --git a/runtime/onnxruntime/third_party/kaldi/decoder/lattice-simple-decoder.cc b/runtime/onnxruntime/third_party/kaldi/decoder/lattice-simple-decoder.cc new file mode 100644 index 000000000..cc8712e85 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/decoder/lattice-simple-decoder.cc @@ -0,0 +1,666 @@ +// decoder/lattice-simple-decoder.cc + +// Copyright 2009-2012 Microsoft Corporation +// 2013-2014 Johns Hopkins University (Author: Daniel Povey) +// 2014 Guoguo Chen + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#include "decoder/lattice-simple-decoder.h" + +namespace kaldi { + + +void LatticeSimpleDecoder::InitDecoding() { + // clean up from last time: + cur_toks_.clear(); + prev_toks_.clear(); + ClearActiveTokens(); + warned_ = false; + decoding_finalized_ = false; + final_costs_.clear(); + num_toks_ = 0; + StateId start_state = fst_.Start(); + KALDI_ASSERT(start_state != fst::kNoStateId); + active_toks_.resize(1); + Token *start_tok = new Token(0.0, 0.0, NULL, NULL); + active_toks_[0].toks = start_tok; + cur_toks_[start_state] = start_tok; + num_toks_++; + ProcessNonemitting(); +} + +bool LatticeSimpleDecoder::Decode(DecodableInterface *decodable) { + InitDecoding(); + + while (!decodable->IsLastFrame(NumFramesDecoded() - 1)) { + if (NumFramesDecoded() % config_.prune_interval == 0) + PruneActiveTokens(config_.lattice_beam * config_.prune_scale); + ProcessEmitting(decodable); + // Important to call PruneCurrentTokens before ProcessNonemitting, or we + // would get dangling forward pointers. Anyway, ProcessNonemitting uses the + // beam. + PruneCurrentTokens(config_.beam, &cur_toks_); + ProcessNonemitting(); + } + FinalizeDecoding(); + + // Returns true if we have any kind of traceback available (not necessarily + // to the end state; query ReachedFinal() for that). + return !final_costs_.empty(); +} + +// Outputs an FST corresponding to the single best path +// through the lattice. +bool LatticeSimpleDecoder::GetBestPath(Lattice *ofst, + bool use_final_probs) const { + fst::VectorFst fst; + GetRawLattice(&fst, use_final_probs); + ShortestPath(fst, ofst); + return (ofst->NumStates() > 0); +} + +// Outputs an FST corresponding to the raw, state-level +// tracebacks. +bool LatticeSimpleDecoder::GetRawLattice(Lattice *ofst, + bool use_final_probs) const { + typedef LatticeArc Arc; + typedef Arc::StateId StateId; + typedef Arc::Weight Weight; + typedef Arc::Label Label; + + // Note: you can't use the old interface (Decode()) if you want to + // get the lattice with use_final_probs = false. You'd have to do + // InitDecoding() and then AdvanceDecoding(). + if (decoding_finalized_ && !use_final_probs) + KALDI_ERR << "You cannot call FinalizeDecoding() and then call " + << "GetRawLattice() with use_final_probs == false"; + + unordered_map final_costs_local; + + const unordered_map &final_costs = + (decoding_finalized_ ? final_costs_ : final_costs_local); + + if (!decoding_finalized_ && use_final_probs) + ComputeFinalCosts(&final_costs_local, NULL, NULL); + + ofst->DeleteStates(); + int32 num_frames = NumFramesDecoded(); + KALDI_ASSERT(num_frames > 0); + const int32 bucket_count = num_toks_/2 + 3; + unordered_map tok_map(bucket_count); + // First create all states. + for (int32 f = 0; f <= num_frames; f++) { + if (active_toks_[f].toks == NULL) { + KALDI_WARN << "GetRawLattice: no tokens active on frame " << f + << ": not producing lattice.\n"; + return false; + } + for (Token *tok = active_toks_[f].toks; tok != NULL; tok = tok->next) + tok_map[tok] = ofst->AddState(); + // The next statement sets the start state of the output FST. + // Because we always add new states to the head of the list + // active_toks_[f].toks, and the start state was the first one + // added, it will be the last one added to ofst. + if (f == 0 && ofst->NumStates() > 0) + ofst->SetStart(ofst->NumStates()-1); + } + StateId cur_state = 0; // we rely on the fact that we numbered these + // consecutively (AddState() returns the numbers in order..) + for (int32 f = 0; f <= num_frames; f++) { + for (Token *tok = active_toks_[f].toks; tok != NULL; tok = tok->next, + cur_state++) { + for (ForwardLink *l = tok->links; + l != NULL; + l = l->next) { + unordered_map::const_iterator iter = + tok_map.find(l->next_tok); + StateId nextstate = iter->second; + KALDI_ASSERT(iter != tok_map.end()); + Arc arc(l->ilabel, l->olabel, + Weight(l->graph_cost, l->acoustic_cost), + nextstate); + ofst->AddArc(cur_state, arc); + } + if (f == num_frames) { + if (use_final_probs && !final_costs.empty()) { + unordered_map::const_iterator iter = + final_costs.find(tok); + if (iter != final_costs.end()) + ofst->SetFinal(cur_state, LatticeWeight(iter->second, 0)); + } else { + ofst->SetFinal(cur_state, LatticeWeight::One()); + } + } + } + } + KALDI_ASSERT(cur_state == ofst->NumStates()); + return (cur_state != 0); +} + +// This function is now deprecated, since now we do determinization from outside +// the LatticeSimpleDecoder class. +// Outputs an FST corresponding to the lattice-determinized +// lattice (one path per word sequence). +bool LatticeSimpleDecoder::GetLattice( + CompactLattice *ofst, + bool use_final_probs) const { + Lattice raw_fst; + GetRawLattice(&raw_fst, use_final_probs); + Invert(&raw_fst); // make it so word labels are on the input. + if (!TopSort(&raw_fst)) // topological sort makes lattice-determinization more efficient + KALDI_WARN << "Topological sorting of state-level lattice failed " + "(probably your lexicon has empty words or your LM has epsilon cycles; this " + " is a bad idea.)"; + // (in phase where we get backward-costs). + fst::ILabelCompare ilabel_comp; + ArcSort(&raw_fst, ilabel_comp); // sort on ilabel; makes + // lattice-determinization more efficient. + + fst::DeterminizeLatticePrunedOptions lat_opts; + lat_opts.max_mem = config_.det_opts.max_mem; + + DeterminizeLatticePruned(raw_fst, config_.lattice_beam, ofst, lat_opts); + raw_fst.DeleteStates(); // Free memory-- raw_fst no longer needed. + Connect(ofst); // Remove unreachable states... there might be + // a small number of these, in some cases. + // Note: if something went wrong and the raw lattice was empty, + // we should still get to this point in the code without warnings or failures. + return (ofst->NumStates() != 0); +} + + + +// FindOrAddToken either locates a token in cur_toks_, or if necessary inserts a new, +// empty token (i.e. with no forward links) for the current frame. [note: it's +// inserted if necessary into cur_toks_ and also into the singly linked list +// of tokens active on this frame (whose head is at active_toks_[frame]). +// +// Returns the Token pointer. Sets "changed" (if non-NULL) to true +// if the token was newly created or the cost changed. +inline LatticeSimpleDecoder::Token *LatticeSimpleDecoder::FindOrAddToken( + StateId state, int32 frame, BaseFloat tot_cost, + bool emitting, bool *changed) { + KALDI_ASSERT(frame < active_toks_.size()); + Token *&toks = active_toks_[frame].toks; + + unordered_map::iterator find_iter = cur_toks_.find(state); + if (find_iter == cur_toks_.end()) { // no such token presently. + // Create one. + const BaseFloat extra_cost = 0.0; + // tokens on the currently final frame have zero extra_cost + // as any of them could end up + // on the winning path. + Token *new_tok = new Token (tot_cost, extra_cost, NULL, toks); + toks = new_tok; + num_toks_++; + cur_toks_[state] = new_tok; + if (changed) *changed = true; + return new_tok; + } else { + Token *tok = find_iter->second; // There is an existing Token for this state. + if (tok->tot_cost > tot_cost) { + tok->tot_cost = tot_cost; + if (changed) *changed = true; + } else { + if (changed) *changed = false; + } + return tok; + } +} + +// delta is the amount by which the extra_costs must +// change before it sets "extra_costs_changed" to true. If delta is larger, +// we'll tend to go back less far toward the beginning of the file. +void LatticeSimpleDecoder::PruneForwardLinks( + int32 frame, bool *extra_costs_changed, + bool *links_pruned, BaseFloat delta) { + // We have to iterate until there is no more change, because the links + // are not guaranteed to be in topological order. + + *extra_costs_changed = false; + *links_pruned = false; + KALDI_ASSERT(frame >= 0 && frame < active_toks_.size()); + if (active_toks_[frame].toks == NULL ) { // empty list; this should + // not happen. + if (!warned_) { + KALDI_WARN << "No tokens alive [doing pruning].. warning first " + "time only for each utterance\n"; + warned_ = true; + } + } + + bool changed = true; + while (changed) { + changed = false; + for (Token *tok = active_toks_[frame].toks; tok != NULL; tok = tok->next) { + ForwardLink *link, *prev_link = NULL; + // will recompute tok_extra_cost. + BaseFloat tok_extra_cost = std::numeric_limits::infinity(); + for (link = tok->links; link != NULL; ) { + // See if we need to excise this link... + Token *next_tok = link->next_tok; + BaseFloat link_extra_cost = next_tok->extra_cost + + ((tok->tot_cost + link->acoustic_cost + link->graph_cost) + - next_tok->tot_cost); + KALDI_ASSERT(link_extra_cost == link_extra_cost); // check for NaN + if (link_extra_cost > config_.lattice_beam) { // excise link + ForwardLink *next_link = link->next; + if (prev_link != NULL) prev_link->next = next_link; + else tok->links = next_link; + delete link; + link = next_link; // advance link but leave prev_link the same. + *links_pruned = true; + } else { // keep the link and update the tok_extra_cost if needed. + if (link_extra_cost < 0.0) { // this is just a precaution. + if (link_extra_cost < -0.01) + KALDI_WARN << "Negative extra_cost: " << link_extra_cost; + link_extra_cost = 0.0; + } + if (link_extra_cost < tok_extra_cost) + tok_extra_cost = link_extra_cost; + prev_link = link; + link = link->next; + } + } + if (fabs(tok_extra_cost - tok->extra_cost) > delta) + changed = true; + tok->extra_cost = tok_extra_cost; // will be +infinity or <= lattice_beam_. + } + if (changed) *extra_costs_changed = true; + + // Note: it's theoretically possible that aggressive compiler + // optimizations could cause an infinite loop here for small delta and + // high-dynamic-range scores. + } +} + + +void LatticeSimpleDecoder::ComputeFinalCosts( + unordered_map *final_costs, + BaseFloat *final_relative_cost, + BaseFloat *final_best_cost) const { + KALDI_ASSERT(!decoding_finalized_); + if (final_costs != NULL) + final_costs->clear(); + + BaseFloat infinity = std::numeric_limits::infinity(); + BaseFloat best_cost = infinity, + best_cost_with_final = infinity; + + for (unordered_map::const_iterator iter = cur_toks_.begin(); + iter != cur_toks_.end(); ++iter) { + StateId state = iter->first; + Token *tok = iter->second; + BaseFloat final_cost = fst_.Final(state).Value(); + BaseFloat cost = tok->tot_cost, + cost_with_final = cost + final_cost; + best_cost = std::min(cost, best_cost); + best_cost_with_final = std::min(cost_with_final, best_cost_with_final); + if (final_costs != NULL && final_cost != infinity) + (*final_costs)[tok] = final_cost; + } + if (final_relative_cost != NULL) { + if (best_cost == infinity && best_cost_with_final == infinity) { + // Likely this will only happen if there are no tokens surviving. + // This seems the least bad way to handle it. + *final_relative_cost = infinity; + } else { + *final_relative_cost = best_cost_with_final - best_cost; + } + } + if (final_best_cost != NULL) { + if (best_cost_with_final != infinity) { // final-state exists. + *final_best_cost = best_cost_with_final; + } else { // no final-state exists. + *final_best_cost = best_cost; + } + } +} + + +// PruneForwardLinksFinal is a version of PruneForwardLinks that we call +// on the final frame. If there are final tokens active, it uses the final-probs +// for pruning, otherwise it treats all tokens as final. +void LatticeSimpleDecoder::PruneForwardLinksFinal() { + KALDI_ASSERT(!active_toks_.empty()); + int32 frame_plus_one = active_toks_.size() - 1; + + if (active_toks_[frame_plus_one].toks == NULL) // empty list; should not happen. + KALDI_WARN << "No tokens alive at end of file\n"; + + typedef unordered_map::const_iterator IterType; + ComputeFinalCosts(&final_costs_, &final_relative_cost_, &final_best_cost_); + decoding_finalized_ = true; + // We're about to delete some of the tokens active on the final frame, so we + // clear cur_toks_ because otherwise it would then contain dangling pointers. + cur_toks_.clear(); + + // Now go through tokens on this frame, pruning forward links... may have to + // iterate a few times until there is no more change, because the list is not + // in topological order. This is a modified version of the code in + // PruneForwardLinks, but here we also take account of the final-probs. + bool changed = true; + BaseFloat delta = 1.0e-05; + while (changed) { + changed = false; + for (Token *tok = active_toks_[frame_plus_one].toks; + tok != NULL; tok = tok->next) { + ForwardLink *link, *prev_link=NULL; + // will recompute tok_extra_cost. It has a term in it that corresponds + // to the "final-prob", so instead of initializing tok_extra_cost to infinity + // below we set it to the difference between the (score+final_prob) of this token, + // and the best such (score+final_prob). + + BaseFloat final_cost; + if (final_costs_.empty()) { + final_cost = 0.0; + } else { + IterType iter = final_costs_.find(tok); + if (iter != final_costs_.end()) + final_cost = iter->second; + else + final_cost = std::numeric_limits::infinity(); + } + BaseFloat tok_extra_cost = tok->tot_cost + final_cost - final_best_cost_; + // tok_extra_cost will be a "min" over either directly being final, or + // being indirectly final through other links, and the loop below may + // decrease its value: + for (link = tok->links; link != NULL; ) { + // See if we need to excise this link... + Token *next_tok = link->next_tok; + BaseFloat link_extra_cost = next_tok->extra_cost + + ((tok->tot_cost + link->acoustic_cost + link->graph_cost) + - next_tok->tot_cost); + if (link_extra_cost > config_.lattice_beam) { // excise link + ForwardLink *next_link = link->next; + if (prev_link != NULL) prev_link->next = next_link; + else tok->links = next_link; + delete link; + link = next_link; // advance link but leave prev_link the same. + } else { // keep the link and update the tok_extra_cost if needed. + if (link_extra_cost < 0.0) { // this is just a precaution. + if (link_extra_cost < -0.01) + KALDI_WARN << "Negative extra_cost: " << link_extra_cost; + link_extra_cost = 0.0; + } + if (link_extra_cost < tok_extra_cost) + tok_extra_cost = link_extra_cost; + prev_link = link; + link = link->next; + } + } + // prune away tokens worse than lattice_beam above best path. This step + // was not necessary in the non-final case because then, this case + // showed up as having no forward links. Here, the tok_extra_cost has + // an extra component relating to the final-prob. + if (tok_extra_cost > config_.lattice_beam) + tok_extra_cost = std::numeric_limits::infinity(); + // to be pruned in PruneTokensForFrame + + if (!ApproxEqual(tok->extra_cost, tok_extra_cost, delta)) + changed = true; + tok->extra_cost = tok_extra_cost; // will be +infinity or <= lattice_beam_. + } + } // while changed +} + +BaseFloat LatticeSimpleDecoder::FinalRelativeCost() const { + if (!decoding_finalized_) { + BaseFloat relative_cost; + ComputeFinalCosts(NULL, &relative_cost, NULL); + return relative_cost; + } else { + // we're not allowed to call that function if FinalizeDecoding() has + // been called; return a cached value. + return final_relative_cost_; + } +} + +// Prune away any tokens on this frame that have no forward links. [we don't do +// this in PruneForwardLinks because it would give us a problem with dangling +// pointers]. +void LatticeSimpleDecoder::PruneTokensForFrame(int32 frame) { + KALDI_ASSERT(frame >= 0 && frame < active_toks_.size()); + Token *&toks = active_toks_[frame].toks; + if (toks == NULL) + KALDI_WARN << "No tokens alive [doing pruning]"; + Token *tok, *next_tok, *prev_tok = NULL; + for (tok = toks; tok != NULL; tok = next_tok) { + next_tok = tok->next; + if (tok->extra_cost == std::numeric_limits::infinity()) { + // Next token is unreachable from end of graph; excise tok from list + // and delete tok. + if (prev_tok != NULL) prev_tok->next = tok->next; + else toks = tok->next; + delete tok; + num_toks_--; + } else { + prev_tok = tok; + } + } +} + +// Go backwards through still-alive tokens, pruning them, starting not from +// the current frame (where we want to keep all tokens) but from the frame before +// that. We go backwards through the frames and stop when we reach a point +// where the delta-costs are not changing (and the delta controls when we consider +// a cost to have "not changed"). +void LatticeSimpleDecoder::PruneActiveTokens(BaseFloat delta) { + int32 cur_frame_plus_one = NumFramesDecoded(); + int32 num_toks_begin = num_toks_; + // The index "f" below represents a "frame plus one", i.e. you'd have to subtract + // one to get the corresponding index for the decodable object. + for (int32 f = cur_frame_plus_one - 1; f >= 0; f--) { + // Reason why we need to prune forward links in this situation: + // (1) we have never pruned them + // (2) we never pruned the forward links on the next frame, which + // + if (active_toks_[f].must_prune_forward_links) { + bool extra_costs_changed = false, links_pruned = false; + PruneForwardLinks(f, &extra_costs_changed, &links_pruned, delta); + if (extra_costs_changed && f > 0) + active_toks_[f-1].must_prune_forward_links = true; + if (links_pruned) + active_toks_[f].must_prune_tokens = true; + active_toks_[f].must_prune_forward_links = false; + } + if (f+1 < cur_frame_plus_one && + active_toks_[f+1].must_prune_tokens) { + PruneTokensForFrame(f+1); + active_toks_[f+1].must_prune_tokens = false; + } + } + KALDI_VLOG(3) << "PruneActiveTokens: pruned tokens from " << num_toks_begin + << " to " << num_toks_; +} + + +// FinalizeDecoding() is a version of PruneActiveTokens that we call +// (optionally) on the final frame. Takes into account the final-prob of +// tokens. This function used to be called PruneActiveTokensFinal(). +void LatticeSimpleDecoder::FinalizeDecoding() { + int32 final_frame_plus_one = NumFramesDecoded(); + int32 num_toks_begin = num_toks_; + PruneForwardLinksFinal(); + for (int32 f = final_frame_plus_one - 1; f >= 0; f--) { + bool b1, b2; // values not used. + BaseFloat dontcare = 0.0; + PruneForwardLinks(f, &b1, &b2, dontcare); + PruneTokensForFrame(f + 1); + } + PruneTokensForFrame(0); + KALDI_VLOG(3) << "pruned tokens from " << num_toks_begin + << " to " << num_toks_; +} + +void LatticeSimpleDecoder::ProcessEmitting(DecodableInterface *decodable) { + int32 frame = active_toks_.size() - 1; // frame is the frame-index + // (zero-based) used to get likelihoods + // from the decodable object. + active_toks_.resize(active_toks_.size() + 1); + prev_toks_.clear(); + cur_toks_.swap(prev_toks_); + + // Processes emitting arcs for one frame. Propagates from + // prev_toks_ to cur_toks_. + BaseFloat cutoff = std::numeric_limits::infinity(); + for (unordered_map::iterator iter = prev_toks_.begin(); + iter != prev_toks_.end(); + ++iter) { + StateId state = iter->first; + Token *tok = iter->second; + for (fst::ArcIterator > aiter(fst_, state); + !aiter.Done(); + aiter.Next()) { + const Arc &arc = aiter.Value(); + if (arc.ilabel != 0) { // propagate.. + BaseFloat ac_cost = -decodable->LogLikelihood(frame, arc.ilabel), + graph_cost = arc.weight.Value(), + cur_cost = tok->tot_cost, + tot_cost = cur_cost + ac_cost + graph_cost; + if (tot_cost >= cutoff) continue; + else if (tot_cost + config_.beam < cutoff) + cutoff = tot_cost + config_.beam; + // AddToken adds the next_tok to cur_toks_ (if not already present). + Token *next_tok = FindOrAddToken(arc.nextstate, frame + 1, tot_cost, + true, NULL); + + // Add ForwardLink from tok to next_tok (put on head of list tok->links) + tok->links = new ForwardLink(next_tok, arc.ilabel, arc.olabel, + graph_cost, ac_cost, tok->links); + } + } + } +} + +void LatticeSimpleDecoder::ProcessNonemitting() { + KALDI_ASSERT(!active_toks_.empty()); + int32 frame = static_cast(active_toks_.size()) - 2; + // Note: "frame" is the time-index we just processed, or -1 if + // we are processing the nonemitting transitions before the + // first frame (called from InitDecoding()). + + // Processes nonemitting arcs for one frame. Propagates within + // cur_toks_. Note-- this queue structure is is not very optimal as + // it may cause us to process states unnecessarily (e.g. more than once), + // but in the baseline code, turning this vector into a set to fix this + // problem did not improve overall speed. + std::vector queue; + BaseFloat best_cost = std::numeric_limits::infinity(); + for (unordered_map::iterator iter = cur_toks_.begin(); + iter != cur_toks_.end(); + ++iter) { + StateId state = iter->first; + if (fst_.NumInputEpsilons(state) != 0) + queue.push_back(state); + best_cost = std::min(best_cost, iter->second->tot_cost); + } + if (queue.empty()) { + if (!warned_) { + KALDI_ERR << "Error in ProcessEmitting: no surviving tokens: frame is " + << frame; + warned_ = true; + } + } + BaseFloat cutoff = best_cost + config_.beam; + + while (!queue.empty()) { + StateId state = queue.back(); + queue.pop_back(); + Token *tok = cur_toks_[state]; + // If "tok" has any existing forward links, delete them, + // because we're about to regenerate them. This is a kind + // of non-optimality (remember, this is the simple decoder), + // but since most states are emitting it's not a huge issue. + tok->DeleteForwardLinks(); + tok->links = NULL; + for (fst::ArcIterator > aiter(fst_, state); + !aiter.Done(); + aiter.Next()) { + const Arc &arc = aiter.Value(); + if (arc.ilabel == 0) { // propagate nonemitting only... + BaseFloat graph_cost = arc.weight.Value(), + cur_cost = tok->tot_cost, + tot_cost = cur_cost + graph_cost; + if (tot_cost < cutoff) { + bool changed; + Token *new_tok = FindOrAddToken(arc.nextstate, frame + 1, tot_cost, + false, &changed); + + tok->links = new ForwardLink(new_tok, 0, arc.olabel, + graph_cost, 0, tok->links); + + // "changed" tells us whether the new token has a different + // cost from before, or is new [if so, add into queue]. + if (changed && fst_.NumInputEpsilons(arc.nextstate) != 0) + queue.push_back(arc.nextstate); + } + } + } + } +} + +void LatticeSimpleDecoder::ClearActiveTokens() { // a cleanup routine, at utt end/begin + for (size_t i = 0; i < active_toks_.size(); i++) { + // Delete all tokens alive on this frame, and any forward + // links they may have. + for (Token *tok = active_toks_[i].toks; tok != NULL; ) { + tok->DeleteForwardLinks(); + Token *next_tok = tok->next; + delete tok; + num_toks_--; + tok = next_tok; + } + } + active_toks_.clear(); + KALDI_ASSERT(num_toks_ == 0); +} + +// PruneCurrentTokens deletes the tokens from the "toks" map, but not +// from the active_toks_ list, which could cause dangling forward pointers +// (will delete it during regular pruning operation). +void LatticeSimpleDecoder::PruneCurrentTokens(BaseFloat beam, unordered_map *toks) { + if (toks->empty()) { + KALDI_VLOG(2) << "No tokens to prune.\n"; + return; + } + BaseFloat best_cost = 1.0e+10; // positive == high cost == bad. + for (unordered_map::iterator iter = toks->begin(); + iter != toks->end(); ++iter) { + best_cost = + std::min(best_cost, + static_cast(iter->second->tot_cost)); + } + std::vector retained; + BaseFloat cutoff = best_cost + beam; + for (unordered_map::iterator iter = toks->begin(); + iter != toks->end(); ++iter) { + if (iter->second->tot_cost < cutoff) + retained.push_back(iter->first); + } + unordered_map tmp; + for (size_t i = 0; i < retained.size(); i++) { + tmp[retained[i]] = (*toks)[retained[i]]; + } + KALDI_VLOG(2) << "Pruned to "<<(retained.size())<<" toks.\n"; + tmp.swap(*toks); +} + + +} // end namespace kaldi. + + diff --git a/runtime/onnxruntime/third_party/kaldi/decoder/lattice-simple-decoder.h b/runtime/onnxruntime/third_party/kaldi/decoder/lattice-simple-decoder.h new file mode 100644 index 000000000..dd8de81e1 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/decoder/lattice-simple-decoder.h @@ -0,0 +1,318 @@ +// decoder/lattice-simple-decoder.h + +// Copyright 2009-2012 Microsoft Corporation +// 2012-2014 Johns Hopkins University (Author: Daniel Povey) +// 2014 Guoguo Chen + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#ifndef KALDI_DECODER_LATTICE_SIMPLE_DECODER_H_ +#define KALDI_DECODER_LATTICE_SIMPLE_DECODER_H_ + + +#include "util/stl-utils.h" +#include "fst/fstlib.h" +#include "itf/decodable-itf.h" +#include "fstext/fstext-lib.h" +#include "lat/determinize-lattice-pruned.h" +#include "lat/kaldi-lattice.h" + +#include + +namespace kaldi { + +struct LatticeSimpleDecoderConfig { + BaseFloat beam; + BaseFloat lattice_beam; + int32 prune_interval; + bool determinize_lattice; // not inspected by this class... used in + // command-line program. + bool prune_lattice; + BaseFloat beam_ratio; + BaseFloat prune_scale; // Note: we don't make this configurable on the command line, + // it's not a very important parameter. It affects the + // algorithm that prunes the tokens as we go. + fst::DeterminizeLatticePhonePrunedOptions det_opts; + + LatticeSimpleDecoderConfig(): beam(16.0), + lattice_beam(10.0), + prune_interval(25), + determinize_lattice(true), + beam_ratio(0.9), + prune_scale(0.1) { } + void Register(OptionsItf *opts) { + det_opts.Register(opts); + opts->Register("beam", &beam, "Decoding beam."); + opts->Register("lattice-beam", &lattice_beam, "Lattice generation beam"); + opts->Register("prune-interval", &prune_interval, "Interval (in frames) at " + "which to prune tokens"); + opts->Register("determinize-lattice", &determinize_lattice, "If true, " + "determinize the lattice (in a special sense, keeping only " + "best pdf-sequence for each word-sequence)."); + } + void Check() const { + KALDI_ASSERT(beam > 0.0 && lattice_beam > 0.0 && prune_interval > 0); + } +}; + + +/** Simplest possible decoder, included largely for didactic purposes and as a + means to debug more highly optimized decoders. See \ref decoders_simple + for more information. + */ +class LatticeSimpleDecoder { + public: + typedef fst::StdArc Arc; + typedef Arc::Label Label; + typedef Arc::StateId StateId; + typedef Arc::Weight Weight; + // instantiate this class once for each thing you have to decode. + LatticeSimpleDecoder(const fst::Fst &fst, + const LatticeSimpleDecoderConfig &config): + fst_(fst), config_(config), num_toks_(0) { config.Check(); } + + ~LatticeSimpleDecoder() { ClearActiveTokens(); } + + const LatticeSimpleDecoderConfig &GetOptions() const { + return config_; + } + + // Returns true if any kind of traceback is available (not necessarily from + // a final state). + bool Decode(DecodableInterface *decodable); + + + /// says whether a final-state was active on the last frame. If it was not, the + /// lattice (or traceback) will end with states that are not final-states. + bool ReachedFinal() const { + return FinalRelativeCost() != std::numeric_limits::infinity(); + } + + /// InitDecoding initializes the decoding, and should only be used if you + /// intend to call AdvanceDecoding(). If you call Decode(), you don't need + /// to call this. You can call InitDecoding if you have already decoded an + /// utterance and want to start with a new utterance. + void InitDecoding(); + + /// This function may be optionally called after AdvanceDecoding(), when you + /// do not plan to decode any further. It does an extra pruning step that + /// will help to prune the lattices output by GetLattice and (particularly) + /// GetRawLattice more accurately, particularly toward the end of the + /// utterance. It does this by using the final-probs in pruning (if any + /// final-state survived); it also does a final pruning step that visits all + /// states (the pruning that is done during decoding may fail to prune states + /// that are within kPruningScale = 0.1 outside of the beam). If you call + /// this, you cannot call AdvanceDecoding again (it will fail), and you + /// cannot call GetLattice() and related functions with use_final_probs = + /// false. + /// Used to be called PruneActiveTokensFinal(). + void FinalizeDecoding(); + + /// FinalRelativeCost() serves the same purpose as ReachedFinal(), but gives + /// more information. It returns the difference between the best (final-cost + /// plus cost) of any token on the final frame, and the best cost of any token + /// on the final frame. If it is infinity it means no final-states were + /// present on the final frame. It will usually be nonnegative. If it not + /// too positive (e.g. < 5 is my first guess, but this is not tested) you can + /// take it as a good indication that we reached the final-state with + /// reasonable likelihood. + BaseFloat FinalRelativeCost() const; + + // Outputs an FST corresponding to the single best path + // through the lattice. Returns true if result is nonempty + // (using the return status is deprecated, it will become void). + // If "use_final_probs" is true AND we reached the final-state + // of the graph then it will include those as final-probs, else + // it will treat all final-probs as one. + bool GetBestPath(Lattice *lat, + bool use_final_probs = true) const; + + // Outputs an FST corresponding to the raw, state-level + // tracebacks. Returns true if result is nonempty + // (using the return status is deprecated, it will become void). + // If "use_final_probs" is true AND we reached the final-state + // of the graph then it will include those as final-probs, else + // it will treat all final-probs as one. + bool GetRawLattice(Lattice *lat, + bool use_final_probs = true) const; + + // This function is now deprecated, since now we do determinization from + // outside the LatticeTrackingDecoder class. + // Outputs an FST corresponding to the lattice-determinized + // lattice (one path per word sequence). [will become deprecated, + // users should determinize themselves.] + bool GetLattice(CompactLattice *clat, + bool use_final_probs = true) const; + + inline int32 NumFramesDecoded() const { return active_toks_.size() - 1; } + private: + struct Token; + // ForwardLinks are the links from a token to a token on the next frame. + // or sometimes on the current frame (for input-epsilon links). + struct ForwardLink { + Token *next_tok; // the next token [or NULL if represents final-state] + Label ilabel; // ilabel on link. + Label olabel; // olabel on link. + BaseFloat graph_cost; // graph cost of traversing link (contains LM, etc.) + BaseFloat acoustic_cost; // acoustic cost (pre-scaled) of traversing link + ForwardLink *next; // next in singly-linked list of forward links from a + // token. + ForwardLink(Token *next_tok, Label ilabel, Label olabel, + BaseFloat graph_cost, BaseFloat acoustic_cost, + ForwardLink *next): + next_tok(next_tok), ilabel(ilabel), olabel(olabel), + graph_cost(graph_cost), acoustic_cost(acoustic_cost), + next(next) { } + }; + + // Token is what's resident in a particular state at a particular time. + // In this decoder a Token actually contains *forward* links. + // When first created, a Token just has the (total) cost. We add forward + // links from it when we process the next frame. + struct Token { + BaseFloat tot_cost; // would equal weight.Value()... cost up to this point. + BaseFloat extra_cost; // >= 0. After calling PruneForwardLinks, this equals + // the minimum difference between the cost of the best path this is on, + // and the cost of the absolute best path, under the assumption + // that any of the currently active states at the decoding front may + // eventually succeed (e.g. if you were to take the currently active states + // one by one and compute this difference, and then take the minimum). + + ForwardLink *links; // Head of singly linked list of ForwardLinks + + Token *next; // Next in list of tokens for this frame. + + Token(BaseFloat tot_cost, BaseFloat extra_cost, ForwardLink *links, + Token *next): tot_cost(tot_cost), extra_cost(extra_cost), links(links), + next(next) { } + Token() {} + void DeleteForwardLinks() { + ForwardLink *l = links, *m; + while (l != NULL) { + m = l->next; + delete l; + l = m; + } + links = NULL; + } + }; + + // head and tail of per-frame list of Tokens (list is in topological order), + // and something saying whether we ever pruned it using PruneForwardLinks. + struct TokenList { + Token *toks; + bool must_prune_forward_links; + bool must_prune_tokens; + TokenList(): toks(NULL), must_prune_forward_links(true), + must_prune_tokens(true) { } + }; + + + // FindOrAddToken either locates a token in cur_toks_, or if necessary inserts a new, + // empty token (i.e. with no forward links) for the current frame. [note: it's + // inserted if necessary into cur_toks_ and also into the singly linked list + // of tokens active on this frame (whose head is at active_toks_[frame]). + // + // Returns the Token pointer. Sets "changed" (if non-NULL) to true + // if the token was newly created or the cost changed. + inline Token *FindOrAddToken(StateId state, int32 frame_plus_one, + BaseFloat tot_cost, bool emitting, bool *changed); + + // delta is the amount by which the extra_costs must + // change before it sets "extra_costs_changed" to true. If delta is larger, + // we'll tend to go back less far toward the beginning of the file. + void PruneForwardLinks(int32 frame, bool *extra_costs_changed, + bool *links_pruned, + BaseFloat delta); + + // PruneForwardLinksFinal is a version of PruneForwardLinks that we call + // on the final frame. If there are final tokens active, it uses the final-probs + // for pruning, otherwise it treats all tokens as final. + void PruneForwardLinksFinal(); + + // Prune away any tokens on this frame that have no forward links. [we don't do + // this in PruneForwardLinks because it would give us a problem with dangling + // pointers]. + void PruneTokensForFrame(int32 frame); + + // Go backwards through still-alive tokens, pruning them if the + // forward+backward cost is more than lat_beam away from the best path. It's + // possible to prove that this is "correct" in the sense that we won't lose + // anything outside of lat_beam, regardless of what happens in the future. + // delta controls when it considers a cost to have changed enough to continue + // going backward and propagating the change. larger delta -> will recurse + // less far. + void PruneActiveTokens(BaseFloat delta); + + void ProcessEmitting(DecodableInterface *decodable); + + void ProcessNonemitting(); + + void ClearActiveTokens(); // a cleanup routine, at utt end/begin + + // This function computes the final-costs for tokens active on the final + // frame. It outputs to final-costs, if non-NULL, a map from the Token* + // pointer to the final-prob of the corresponding state, or zero for all states if + // none were final. It outputs to final_relative_cost, if non-NULL, the + // difference between the best forward-cost including the final-prob cost, and + // the best forward-cost without including the final-prob cost (this will + // usually be positive), or infinity if there were no final-probs. It outputs + // to final_best_cost, if non-NULL, the lowest for any token t active on the + // final frame, of t + final-cost[t], where final-cost[t] is the final-cost + // in the graph of the state corresponding to token t, or zero if there + // were no final-probs active on the final frame. + // You cannot call this after FinalizeDecoding() has been called; in that + // case you should get the answer from class-member variables. + void ComputeFinalCosts(unordered_map *final_costs, + BaseFloat *final_relative_cost, + BaseFloat *final_best_cost) const; + + + // PruneCurrentTokens deletes the tokens from the "toks" map, but not + // from the active_toks_ list, which could cause dangling forward pointers + // (will delete it during regular pruning operation). + void PruneCurrentTokens(BaseFloat beam, unordered_map *toks); + + unordered_map cur_toks_; + unordered_map prev_toks_; + std::vector active_toks_; // Lists of tokens, indexed by + // frame_plus_one + const fst::Fst &fst_; + LatticeSimpleDecoderConfig config_; + int32 num_toks_; // current total #toks allocated... + bool warned_; + + + /// decoding_finalized_ is true if someone called FinalizeDecoding(). [note, + /// calling this is optional]. If true, it's forbidden to decode more. Also, + /// if this is set, then the output of ComputeFinalCosts() is in the next + /// three variables. The reason we need to do this is that after + /// FinalizeDecoding() calls PruneTokensForFrame() for the final frame, some + /// of the tokens on the last frame are freed, so we free the list from + /// cur_toks_ to avoid having dangling pointers hanging around. + bool decoding_finalized_; + /// For the meaning of the next 3 variables, see the comment for + /// decoding_finalized_ above., and ComputeFinalCosts(). + unordered_map final_costs_; + BaseFloat final_relative_cost_; + BaseFloat final_best_cost_; +}; + + +} // end namespace kaldi. + + +#endif diff --git a/runtime/onnxruntime/third_party/kaldi/decoder/simple-decoder.cc b/runtime/onnxruntime/third_party/kaldi/decoder/simple-decoder.cc new file mode 100644 index 000000000..f6d799aa8 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/decoder/simple-decoder.cc @@ -0,0 +1,293 @@ +// decoder/simple-decoder.cc + +// Copyright 2009-2011 Microsoft Corporation +// 2012-2013 Johns Hopkins University (author: Daniel Povey) + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#include "decoder/simple-decoder.h" +#include "fstext/remove-eps-local.h" +#include + +namespace kaldi { + +SimpleDecoder::~SimpleDecoder() { + ClearToks(cur_toks_); + ClearToks(prev_toks_); +} + + +bool SimpleDecoder::Decode(DecodableInterface *decodable) { + InitDecoding(); + AdvanceDecoding(decodable); + return (!cur_toks_.empty()); +} + +void SimpleDecoder::InitDecoding() { + // clean up from last time: + ClearToks(cur_toks_); + ClearToks(prev_toks_); + // initialize decoding: + StateId start_state = fst_.Start(); + KALDI_ASSERT(start_state != fst::kNoStateId); + StdArc dummy_arc(0, 0, StdWeight::One(), start_state); + cur_toks_[start_state] = new Token(dummy_arc, 0.0, NULL); + num_frames_decoded_ = 0; + ProcessNonemitting(); +} + +void SimpleDecoder::AdvanceDecoding(DecodableInterface *decodable, + int32 max_num_frames) { + KALDI_ASSERT(num_frames_decoded_ >= 0 && + "You must call InitDecoding() before AdvanceDecoding()"); + int32 num_frames_ready = decodable->NumFramesReady(); + // num_frames_ready must be >= num_frames_decoded, or else + // the number of frames ready must have decreased (which doesn't + // make sense) or the decodable object changed between calls + // (which isn't allowed). + KALDI_ASSERT(num_frames_ready >= num_frames_decoded_); + int32 target_frames_decoded = num_frames_ready; + if (max_num_frames >= 0) + target_frames_decoded = std::min(target_frames_decoded, + num_frames_decoded_ + max_num_frames); + while (num_frames_decoded_ < target_frames_decoded) { + // note: ProcessEmitting() increments num_frames_decoded_ + ClearToks(prev_toks_); + cur_toks_.swap(prev_toks_); + ProcessEmitting(decodable); + ProcessNonemitting(); + PruneToks(beam_, &cur_toks_); + } +} + +bool SimpleDecoder::ReachedFinal() const { + for (unordered_map::const_iterator iter = cur_toks_.begin(); + iter != cur_toks_.end(); + ++iter) { + if (iter->second->cost_ != std::numeric_limits::infinity() && + fst_.Final(iter->first) != StdWeight::Zero()) + return true; + } + return false; +} + +BaseFloat SimpleDecoder::FinalRelativeCost() const { + // as a special case, if there are no active tokens at all (e.g. some kind of + // pruning failure), return infinity. + double infinity = std::numeric_limits::infinity(); + if (cur_toks_.empty()) + return infinity; + double best_cost = infinity, + best_cost_with_final = infinity; + for (unordered_map::const_iterator iter = cur_toks_.begin(); + iter != cur_toks_.end(); + ++iter) { + // Note: Plus is taking the minimum cost, since we're in the tropical + // semiring. + best_cost = std::min(best_cost, iter->second->cost_); + best_cost_with_final = std::min(best_cost_with_final, + iter->second->cost_ + + fst_.Final(iter->first).Value()); + } + BaseFloat extra_cost = best_cost_with_final - best_cost; + if (extra_cost != extra_cost) { // NaN. This shouldn't happen; it indicates some + // kind of error, most likely. + KALDI_WARN << "Found NaN (likely search failure in decoding)"; + return infinity; + } + // Note: extra_cost will be infinity if no states were final. + return extra_cost; +} + +// Outputs an FST corresponding to the single best path +// through the lattice. +bool SimpleDecoder::GetBestPath(Lattice *fst_out, bool use_final_probs) const { + fst_out->DeleteStates(); + Token *best_tok = NULL; + bool is_final = ReachedFinal(); + if (!is_final) { + for (unordered_map::const_iterator iter = cur_toks_.begin(); + iter != cur_toks_.end(); + ++iter) + if (best_tok == NULL || *best_tok < *(iter->second) ) + best_tok = iter->second; + } else { + double infinity =std::numeric_limits::infinity(), + best_cost = infinity; + for (unordered_map::const_iterator iter = cur_toks_.begin(); + iter != cur_toks_.end(); + ++iter) { + double this_cost = iter->second->cost_ + fst_.Final(iter->first).Value(); + if (this_cost != infinity && this_cost < best_cost) { + best_cost = this_cost; + best_tok = iter->second; + } + } + } + if (best_tok == NULL) return false; // No output. + + std::vector arcs_reverse; // arcs in reverse order. + for (Token *tok = best_tok; tok != NULL; tok = tok->prev_) + arcs_reverse.push_back(tok->arc_); + KALDI_ASSERT(arcs_reverse.back().nextstate == fst_.Start()); + arcs_reverse.pop_back(); // that was a "fake" token... gives no info. + + StateId cur_state = fst_out->AddState(); + fst_out->SetStart(cur_state); + for (ssize_t i = static_cast(arcs_reverse.size())-1; i >= 0; i--) { + LatticeArc arc = arcs_reverse[i]; + arc.nextstate = fst_out->AddState(); + fst_out->AddArc(cur_state, arc); + cur_state = arc.nextstate; + } + if (is_final && use_final_probs) + fst_out->SetFinal(cur_state, + LatticeWeight(fst_.Final(best_tok->arc_.nextstate).Value(), + 0.0)); + else + fst_out->SetFinal(cur_state, LatticeWeight::One()); + fst::RemoveEpsLocal(fst_out); + return true; +} + + +void SimpleDecoder::ProcessEmitting(DecodableInterface *decodable) { + int32 frame = num_frames_decoded_; + // Processes emitting arcs for one frame. Propagates from + // prev_toks_ to cur_toks_. + double cutoff = std::numeric_limits::infinity(); + for (unordered_map::iterator iter = prev_toks_.begin(); + iter != prev_toks_.end(); + ++iter) { + StateId state = iter->first; + Token *tok = iter->second; + KALDI_ASSERT(state == tok->arc_.nextstate); + for (fst::ArcIterator > aiter(fst_, state); + !aiter.Done(); + aiter.Next()) { + const StdArc &arc = aiter.Value(); + if (arc.ilabel != 0) { // propagate.. + BaseFloat acoustic_cost = -decodable->LogLikelihood(frame, arc.ilabel); + double total_cost = tok->cost_ + arc.weight.Value() + acoustic_cost; + + if (total_cost >= cutoff) continue; + if (total_cost + beam_ < cutoff) + cutoff = total_cost + beam_; + Token *new_tok = new Token(arc, acoustic_cost, tok); + unordered_map::iterator find_iter + = cur_toks_.find(arc.nextstate); + if (find_iter == cur_toks_.end()) { + cur_toks_[arc.nextstate] = new_tok; + } else { + if ( *(find_iter->second) < *new_tok ) { + Token::TokenDelete(find_iter->second); + find_iter->second = new_tok; + } else { + Token::TokenDelete(new_tok); + } + } + } + } + } + num_frames_decoded_++; +} + +void SimpleDecoder::ProcessNonemitting() { + // Processes nonemitting arcs for one frame. Propagates within + // cur_toks_. + std::vector queue; + double infinity = std::numeric_limits::infinity(); + double best_cost = infinity; + for (unordered_map::iterator iter = cur_toks_.begin(); + iter != cur_toks_.end(); + ++iter) { + queue.push_back(iter->first); + best_cost = std::min(best_cost, iter->second->cost_); + } + double cutoff = best_cost + beam_; + + while (!queue.empty()) { + StateId state = queue.back(); + queue.pop_back(); + Token *tok = cur_toks_[state]; + KALDI_ASSERT(tok != NULL && state == tok->arc_.nextstate); + for (fst::ArcIterator > aiter(fst_, state); + !aiter.Done(); + aiter.Next()) { + const StdArc &arc = aiter.Value(); + if (arc.ilabel == 0) { // propagate nonemitting only... + const BaseFloat acoustic_cost = 0.0; + Token *new_tok = new Token(arc, acoustic_cost, tok); + if (new_tok->cost_ > cutoff) { + Token::TokenDelete(new_tok); + } else { + unordered_map::iterator find_iter + = cur_toks_.find(arc.nextstate); + if (find_iter == cur_toks_.end()) { + cur_toks_[arc.nextstate] = new_tok; + queue.push_back(arc.nextstate); + } else { + if ( *(find_iter->second) < *new_tok ) { + Token::TokenDelete(find_iter->second); + find_iter->second = new_tok; + queue.push_back(arc.nextstate); + } else { + Token::TokenDelete(new_tok); + } + } + } + } + } + } +} + +// static +void SimpleDecoder::ClearToks(unordered_map &toks) { + for (unordered_map::iterator iter = toks.begin(); + iter != toks.end(); ++iter) { + Token::TokenDelete(iter->second); + } + toks.clear(); +} + +// static +void SimpleDecoder::PruneToks(BaseFloat beam, unordered_map *toks) { + if (toks->empty()) { + KALDI_VLOG(2) << "No tokens to prune.\n"; + return; + } + double best_cost = std::numeric_limits::infinity(); + for (unordered_map::iterator iter = toks->begin(); + iter != toks->end(); ++iter) + best_cost = std::min(best_cost, iter->second->cost_); + std::vector retained; + double cutoff = best_cost + beam; + for (unordered_map::iterator iter = toks->begin(); + iter != toks->end(); ++iter) { + if (iter->second->cost_ < cutoff) + retained.push_back(iter->first); + else + Token::TokenDelete(iter->second); + } + unordered_map tmp; + for (size_t i = 0; i < retained.size(); i++) { + tmp[retained[i]] = (*toks)[retained[i]]; + } + KALDI_VLOG(2) << "Pruned to " << (retained.size()) << " toks.\n"; + tmp.swap(*toks); +} + +} // end namespace kaldi. diff --git a/runtime/onnxruntime/third_party/kaldi/decoder/simple-decoder.h b/runtime/onnxruntime/third_party/kaldi/decoder/simple-decoder.h new file mode 100644 index 000000000..cd3a690bf --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/decoder/simple-decoder.h @@ -0,0 +1,156 @@ +// decoder/simple-decoder.h + +// Copyright 2009-2013 Microsoft Corporation; Lukas Burget; +// Saarland University (author: Arnab Ghoshal); +// Johns Hopkins University (author: Daniel Povey) + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#ifndef KALDI_DECODER_SIMPLE_DECODER_H_ +#define KALDI_DECODER_SIMPLE_DECODER_H_ + + +#include "util/stl-utils.h" +#include "fst/fstlib.h" +#include "lat/kaldi-lattice.h" +#include "itf/decodable-itf.h" + +namespace kaldi { + +/** Simplest possible decoder, included largely for didactic purposes and as a + means to debug more highly optimized decoders. See \ref decoders_simple + for more information. + */ +class SimpleDecoder { + public: + typedef fst::StdArc StdArc; + typedef StdArc::Weight StdWeight; + typedef StdArc::Label Label; + typedef StdArc::StateId StateId; + + SimpleDecoder(const fst::Fst &fst, BaseFloat beam): fst_(fst), beam_(beam) { } + + ~SimpleDecoder(); + + /// Decode this utterance. + /// Returns true if any tokens reached the end of the file (regardless of + /// whether they are in a final state); query ReachedFinal() after Decode() + /// to see whether we reached a final state. + bool Decode(DecodableInterface *decodable); + + bool ReachedFinal() const; + + // GetBestPath gets the decoding traceback. If "use_final_probs" is true + // AND we reached a final state, it limits itself to final states; + // otherwise it gets the most likely token not taking into account final-probs. + // fst_out will be empty (Start() == kNoStateId) if nothing was available due to + // search error. + // If Decode() returned true, it is safe to assume GetBestPath will return true. + // It returns true if the output lattice was nonempty (i.e. had states in it); + // using the return value is deprecated. + bool GetBestPath(Lattice *fst_out, bool use_final_probs = true) const; + + /// *** The next functions are from the "new interface". *** + + /// FinalRelativeCost() serves the same function as ReachedFinal(), but gives + /// more information. It returns the difference between the best (final-cost plus + /// cost) of any token on the final frame, and the best cost of any token + /// on the final frame. If it is infinity it means no final-states were present + /// on the final frame. It will usually be nonnegative. + BaseFloat FinalRelativeCost() const; + + /// InitDecoding initializes the decoding, and should only be used if you + /// intend to call AdvanceDecoding(). If you call Decode(), you don't need + /// to call this. You can call InitDecoding if you have already decoded an + /// utterance and want to start with a new utterance. + void InitDecoding(); + + /// This will decode until there are no more frames ready in the decodable + /// object, but if max_num_frames is >= 0 it will decode no more than + /// that many frames. If it returns false, then no tokens are alive, + /// which is a kind of error state. + void AdvanceDecoding(DecodableInterface *decodable, + int32 max_num_frames = -1); + + /// Returns the number of frames already decoded. + int32 NumFramesDecoded() const { return num_frames_decoded_; } + + private: + + class Token { + public: + LatticeArc arc_; // We use LatticeArc so that we can separately + // store the acoustic and graph cost, in case + // we need to produce lattice-formatted output. + Token *prev_; + int32 ref_count_; + double cost_; // accumulated total cost up to this point. + Token(const StdArc &arc, + BaseFloat acoustic_cost, + Token *prev): prev_(prev), ref_count_(1) { + arc_.ilabel = arc.ilabel; + arc_.olabel = arc.olabel; + arc_.weight = LatticeWeight(arc.weight.Value(), acoustic_cost); + arc_.nextstate = arc.nextstate; + if (prev) { + prev->ref_count_++; + cost_ = prev->cost_ + (arc.weight.Value() + acoustic_cost); + } else { + cost_ = arc.weight.Value() + acoustic_cost; + } + } + bool operator < (const Token &other) { + return cost_ > other.cost_; + } + + static void TokenDelete(Token *tok) { + while (--tok->ref_count_ == 0) { + Token *prev = tok->prev_; + delete tok; + if (prev == NULL) return; + else tok = prev; + } +#ifdef KALDI_PARANOID + KALDI_ASSERT(tok->ref_count_ > 0); +#endif + } + }; + + // ProcessEmitting decodes the frame num_frames_decoded_ of the + // decodable object, then increments num_frames_decoded_. + void ProcessEmitting(DecodableInterface *decodable); + + void ProcessNonemitting(); + + unordered_map cur_toks_; + unordered_map prev_toks_; + const fst::Fst &fst_; + BaseFloat beam_; + // Keep track of the number of frames decoded in the current file. + int32 num_frames_decoded_; + + static void ClearToks(unordered_map &toks); + + static void PruneToks(BaseFloat beam, unordered_map *toks); + + KALDI_DISALLOW_COPY_AND_ASSIGN(SimpleDecoder); +}; + + +} // end namespace kaldi. + + +#endif diff --git a/runtime/onnxruntime/third_party/kaldi/decoder/training-graph-compiler.cc b/runtime/onnxruntime/third_party/kaldi/decoder/training-graph-compiler.cc new file mode 100644 index 000000000..f610ce51c --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/decoder/training-graph-compiler.cc @@ -0,0 +1,182 @@ +// decoder/training-graph-compiler.cc + +// Copyright 2009-2011 Microsoft Corporation +// 2018 Johns Hopkins University (author: Daniel Povey) +// 2021 Xiaomi Corporation (Author: Junbo Zhang) + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. +#include "decoder/training-graph-compiler.h" +#include "hmm/hmm-utils.h" // for GetHTransducer + +namespace kaldi { + + +TrainingGraphCompiler::TrainingGraphCompiler(const TransitionModel &trans_model, + const ContextDependency &ctx_dep, // Does not maintain reference to this. + fst::VectorFst *lex_fst, + const std::vector &disambig_syms, + const TrainingGraphCompilerOptions &opts): + trans_model_(trans_model), ctx_dep_(ctx_dep), lex_fst_(lex_fst), + disambig_syms_(disambig_syms), opts_(opts) { + using namespace fst; + const std::vector &phone_syms = trans_model_.GetPhones(); // needed to create context fst. + + KALDI_ASSERT(!phone_syms.empty()); + KALDI_ASSERT(IsSortedAndUniq(phone_syms)); + SortAndUniq(&disambig_syms_); + for (int32 i = 0; i < disambig_syms_.size(); i++) + if (std::binary_search(phone_syms.begin(), phone_syms.end(), + disambig_syms_[i])) + KALDI_ERR << "Disambiguation symbol " << disambig_syms_[i] + << " is also a phone."; + + subsequential_symbol_ = 1 + phone_syms.back(); + if (!disambig_syms_.empty() && subsequential_symbol_ <= disambig_syms_.back()) + subsequential_symbol_ = 1 + disambig_syms_.back(); + + if (lex_fst == NULL) return; + + { + int32 N = ctx_dep.ContextWidth(), + P = ctx_dep.CentralPosition(); + if (P != N-1) + AddSubsequentialLoop(subsequential_symbol_, lex_fst_); // This is needed for + // systems with right-context or we will not successfully compose + // with C. + } + + { // make sure lexicon is olabel sorted. + fst::OLabelCompare olabel_comp; + fst::ArcSort(lex_fst_, olabel_comp); + } +} + +bool TrainingGraphCompiler::CompileGraphFromText( + const std::vector &transcript, + fst::VectorFst *out_fst) { + using namespace fst; + VectorFst word_fst; + MakeLinearAcceptor(transcript, &word_fst); + return CompileGraph(word_fst, out_fst); +} + +bool TrainingGraphCompiler::CompileGraphFromLG(const fst::VectorFst &phone2word_fst, + fst::VectorFst *out_fst) { + using namespace fst; + + KALDI_ASSERT(phone2word_fst.Start() != kNoStateId); + + const std::vector &phone_syms = trans_model_.GetPhones(); // needed to create context fst. + + // inv_cfst will be expanded on the fly, as needed. + InverseContextFst inv_cfst(subsequential_symbol_, + phone_syms, + disambig_syms_, + ctx_dep_.ContextWidth(), + ctx_dep_.CentralPosition()); + + + VectorFst ctx2word_fst; + ComposeDeterministicOnDemandInverse(phone2word_fst, &inv_cfst, &ctx2word_fst); + // now ctx2word_fst is C * LG, assuming phone2word_fst is written as LG. + KALDI_ASSERT(ctx2word_fst.Start() != kNoStateId); + + HTransducerConfig h_cfg; + h_cfg.transition_scale = opts_.transition_scale; + + std::vector disambig_syms_h; // disambiguation symbols on + // input side of H. + VectorFst *H = GetHTransducer(inv_cfst.IlabelInfo(), + ctx_dep_, + trans_model_, + h_cfg, + &disambig_syms_h); + + VectorFst &trans2word_fst = *out_fst; // transition-id to word. + TableCompose(*H, ctx2word_fst, &trans2word_fst); + + KALDI_ASSERT(trans2word_fst.Start() != kNoStateId); + + // Epsilon-removal and determinization combined. This will fail if not determinizable. + DeterminizeStarInLog(&trans2word_fst); + + if (!disambig_syms_h.empty()) { + RemoveSomeInputSymbols(disambig_syms_h, &trans2word_fst); + // we elect not to remove epsilons after this phase, as it is + // a little slow. + if (opts_.rm_eps) + RemoveEpsLocal(&trans2word_fst); + } + + + // Encoded minimization. + MinimizeEncoded(&trans2word_fst); + + std::vector disambig; + bool check_no_self_loops = true; + AddSelfLoops(trans_model_, + disambig, + opts_.self_loop_scale, + opts_.reorder, + check_no_self_loops, + &trans2word_fst); + + delete H; + return true; +} + +bool TrainingGraphCompiler::CompileGraph(const fst::VectorFst &word_fst, + fst::VectorFst *out_fst) { + using namespace fst; + KALDI_ASSERT(lex_fst_ !=NULL); + KALDI_ASSERT(out_fst != NULL); + + VectorFst phone2word_fst; + // TableCompose more efficient than compose. + TableCompose(*lex_fst_, word_fst, &phone2word_fst, &lex_cache_); + return CompileGraphFromLG(phone2word_fst, out_fst); +} + +bool TrainingGraphCompiler::CompileGraphsFromText( + const std::vector > &transcripts, + std::vector*> *out_fsts) { + using namespace fst; + std::vector* > word_fsts(transcripts.size()); + for (size_t i = 0; i < transcripts.size(); i++) { + VectorFst *word_fst = new VectorFst(); + MakeLinearAcceptor(transcripts[i], word_fst); + word_fsts[i] = word_fst; + } + bool ans = CompileGraphs(word_fsts, out_fsts); + for (size_t i = 0; i < transcripts.size(); i++) + delete word_fsts[i]; + return ans; +} + +bool TrainingGraphCompiler::CompileGraphs( + const std::vector* > &word_fsts, + std::vector* > *out_fsts) { + out_fsts->resize(word_fsts.size(), NULL); + for (size_t i = 0; i < word_fsts.size(); i++) { + fst::VectorFst trans2word_fst; + if (!CompileGraph(*(word_fsts[i]), &trans2word_fst)) return false; + (*out_fsts)[i] = trans2word_fst.Copy(); + } + return true; +} + + +} // end namespace kaldi diff --git a/runtime/onnxruntime/third_party/kaldi/decoder/training-graph-compiler.h b/runtime/onnxruntime/third_party/kaldi/decoder/training-graph-compiler.h new file mode 100644 index 000000000..f6128d82b --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/decoder/training-graph-compiler.h @@ -0,0 +1,117 @@ +// decoder/training-graph-compiler.h + +// Copyright 2009-2011 Microsoft Corporation +// 2018 Johns Hopkins University (author: Daniel Povey) + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at + +// http://www.apache.org/licenses/LICENSE-2.0 + +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. +#ifndef KALDI_DECODER_TRAINING_GRAPH_COMPILER_H_ +#define KALDI_DECODER_TRAINING_GRAPH_COMPILER_H_ + +#include "base/kaldi-common.h" +#include "hmm/transition-model.h" +#include "fst/fstlib.h" +#include "fstext/fstext-lib.h" +#include "tree/context-dep.h" + + +namespace kaldi { + +struct TrainingGraphCompilerOptions { + + BaseFloat transition_scale; + BaseFloat self_loop_scale; + bool rm_eps; + bool reorder; // (Dan-style graphs) + + explicit TrainingGraphCompilerOptions(BaseFloat transition_scale = 1.0, + BaseFloat self_loop_scale = 1.0, + bool b = true) : + transition_scale(transition_scale), + self_loop_scale(self_loop_scale), + rm_eps(false), + reorder(b) { } + + void Register(OptionsItf *opts) { + opts->Register("transition-scale", &transition_scale, "Scale of transition " + "probabilities (excluding self-loops)"); + opts->Register("self-loop-scale", &self_loop_scale, "Scale of self-loop vs. " + "non-self-loop probability mass "); + opts->Register("reorder", &reorder, "Reorder transition ids for greater decoding efficiency."); + opts->Register("rm-eps", &rm_eps, "Remove [most] epsilons before minimization (only applicable " + "if disambig symbols present)"); + } +}; + + +class TrainingGraphCompiler { + public: + TrainingGraphCompiler(const TransitionModel &trans_model, // Maintains reference to this object. + const ContextDependency &ctx_dep, // And this. + fst::VectorFst *lex_fst, // Takes ownership of this object. + // It should not contain disambiguation symbols or subsequential symbol, + // but it should contain optional silence. + const std::vector &disambig_syms, // disambig symbols in phone symbol table. + const TrainingGraphCompilerOptions &opts); + + + // CompileGraph compiles a single training graph its input is a + // weighted acceptor (G) at the word level, its output is HCLG. + // Note: G could actually be a transducer, it would also work. + // This function is not const for technical reasons involving the cache. + // if not for "table_compose" we could make it const. + bool CompileGraph(const fst::VectorFst &word_grammar, + fst::VectorFst *out_fst); + + // Same as `CompileGraph`, but uses an external LG fst. + bool CompileGraphFromLG(const fst::VectorFst &phone2word_fst, + fst::VectorFst * out_fst); + + // CompileGraphs allows you to compile a number of graphs at the same + // time. This consumes more memory but is faster. + bool CompileGraphs( + const std::vector *> &word_fsts, + std::vector *> *out_fsts); + + // This version creates an FST from the text and calls CompileGraph. + bool CompileGraphFromText(const std::vector &transcript, + fst::VectorFst *out_fst); + + // This function creates FSTs from the text and calls CompileGraphs. + bool CompileGraphsFromText( + const std::vector > &word_grammar, + std::vector *> *out_fsts); + + + ~TrainingGraphCompiler() { delete lex_fst_; } + private: + const TransitionModel &trans_model_; + const ContextDependency &ctx_dep_; + fst::VectorFst *lex_fst_; // lexicon FST (an input; we take + // ownership as we need to modify it). + std::vector disambig_syms_; // disambig symbols (if any) in the phone + int32 subsequential_symbol_; // search in ../fstext/context-fst.h for more info. + // symbol table. + fst::TableComposeCache > lex_cache_; // stores matcher.. + // this is one of Dan's extensions. + + TrainingGraphCompilerOptions opts_; +}; + + + +} // end namespace kaldi. + +#endif diff --git a/runtime/onnxruntime/third_party/kaldi/fstbin/Makefile b/runtime/onnxruntime/third_party/kaldi/fstbin/Makefile new file mode 100644 index 000000000..a22c014a7 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/fstbin/Makefile @@ -0,0 +1,31 @@ + +# make "all" the target. +all: + +# Disable linking math libs because not needed here. Just for compilation speed. +# no, it's now needed for context-fst-test. +# MATHLIB = NONE + +EXTRA_CXXFLAGS = -Wno-sign-compare + +include ../kaldi.mk + +BINFILES = fstdeterminizestar \ + fstrmsymbols fstisstochastic fstminimizeencoded fstmakecontextfst \ + fstmakecontextsyms fstaddsubsequentialloop fstaddselfloops \ + fstrmepslocal fstcomposecontext fsttablecompose fstrand \ + fstdeterminizelog fstphicompose fstcopy \ + fstpushspecial fsts-to-transcripts fsts-project fsts-union \ + fsts-concat make-grammar-fst + +OBJFILES = + +TESTFILES = + +# actually, this library is currently empty. Everything is a header. +LIBFILE = + +ADDLIBS = ../decoder/kaldi-decoder.a ../fstext/kaldi-fstext.a \ + ../util/kaldi-util.a ../matrix/kaldi-matrix.a ../base/kaldi-base.a + +include ../makefiles/default_rules.mk diff --git a/runtime/onnxruntime/third_party/kaldi/fstbin/fstaddselfloops.cc b/runtime/onnxruntime/third_party/kaldi/fstbin/fstaddselfloops.cc new file mode 100644 index 000000000..a84b316d4 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/fstbin/fstaddselfloops.cc @@ -0,0 +1,93 @@ +// fstbin/fstaddselfloops.cc + +// Copyright 2009-2011 Microsoft Corporation + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + + +#include "base/kaldi-common.h" +#include "fst/fstlib.h" +#include "fstext/determinize-star.h" +#include "fstext/fstext-utils.h" +#include "fstext/kaldi-fst-io.h" +#include "util/parse-options.h" +#include "util/simple-io-funcs.h" +/* some test examples: + pushd ~/tmpdir + ( echo 3; echo 4) > in.list + ( echo 5; echo 6) > out.list + ( echo "0 0 0 0"; echo "0 0" ) | fstcompile | fstaddselfloops in.list out.list | fstprint + ( echo "0 1 0 1"; echo " 0 2 1 0"; echo "1 0"; echo "2 0"; ) | fstcompile | fstaddselfloops in.list out.list | fstprint +*/ + +int main(int argc, char *argv[]) { + try { + using namespace kaldi; + using namespace fst; + using kaldi::int32; + + const char *usage = + "Adds self-loops to states of an FST to propagate disambiguation symbols through it\n" + "They are added on each final state and each state with non-epsilon output symbols\n" + "on at least one arc out of the state. Useful in conjunction with predeterminize\n" + "\n" + "Usage: fstaddselfloops in-disambig-list out-disambig-list [in.fst [out.fst] ]\n" + "E.g: fstaddselfloops in.list out.list < in.fst > withloops.fst\n" + "in.list and out.list are lists of integers, one per line, of the\n" + "same length.\n"; + + ParseOptions po(usage); + po.Read(argc, argv); + + if (po.NumArgs() < 2 || po.NumArgs() > 4) { + po.PrintUsage(); + exit(1); + } + + std::string disambig_in_rxfilename = po.GetArg(1), + disambig_out_rxfilename = po.GetArg(2), + fst_in_filename = po.GetOptArg(3), + fst_out_filename = po.GetOptArg(4); + + VectorFst *fst = ReadFstKaldi(fst_in_filename); + + std::vector disambig_in; + if (!ReadIntegerVectorSimple(disambig_in_rxfilename, &disambig_in)) + KALDI_ERR << "fstaddselfloops: Could not read disambiguation symbols from " + << kaldi::PrintableRxfilename(disambig_in_rxfilename); + + std::vector disambig_out; + if (!ReadIntegerVectorSimple(disambig_out_rxfilename, &disambig_out)) + KALDI_ERR << "fstaddselfloops: Could not read disambiguation symbols from " + << kaldi::PrintableRxfilename(disambig_out_rxfilename); + + if (disambig_in.size() != disambig_out.size()) + KALDI_ERR << "fstaddselfloops: mismatch in size of disambiguation symbols"; + + AddSelfLoops(fst, disambig_in, disambig_out); + + WriteFstKaldi(*fst, fst_out_filename); + + delete fst; + + return 0; + } catch(const std::exception &e) { + std::cerr << e.what(); + return -1; + } + return 0; +} + diff --git a/runtime/onnxruntime/third_party/kaldi/fstbin/fstaddsubsequentialloop.cc b/runtime/onnxruntime/third_party/kaldi/fstbin/fstaddsubsequentialloop.cc new file mode 100644 index 000000000..7f8133400 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/fstbin/fstaddsubsequentialloop.cc @@ -0,0 +1,85 @@ +// fstbin/fstaddsubsequentialloop.cc + +// Copyright 2009-2011 Microsoft Corporation + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + + +#include "base/kaldi-common.h" +#include "util/kaldi-io.h" +#include "util/parse-options.h" +#include "util/text-utils.h" +#include "fst/fstlib.h" +#include "fstext/fstext-utils.h" +#include "fstext/context-fst.h" +#include "fstext/kaldi-fst-io.h" + + + +/* some test examples: + ( echo "0 0 0 0"; echo "0 0" ) | fstcompile | fstaddsubsequentialloop 1 | fstprint + ( echo "0 1 0 0"; echo " 0 2 0 0"; echo "1 0"; echo "2 0"; ) | fstcompile | fstaddsubsequentialloop 1 | fstprint +*/ + +int main(int argc, char *argv[]) { + try { + using namespace kaldi; + using namespace fst; + using kaldi::int32; + + const char *usage = + "Minimizes FST after encoding [this algorithm applicable to all FSTs in tropical semiring]\n" + "\n" + "Usage: fstaddsubsequentialloop subseq_sym [in.fst [out.fst] ]\n" + "E.g.: fstaddsubsequentialloop 52 < LG.fst > LG_sub.fst\n"; + + float delta = kDelta; + ParseOptions po(usage); + po.Register("delta", &delta, + "Delta likelihood used for quantization of weights"); + po.Read(argc, argv); + + if (po.NumArgs() < 1 || po.NumArgs() > 3) { + po.PrintUsage(); + exit(1); + } + + int32 subseq_sym; + if (!ConvertStringToInteger(po.GetArg(1), &subseq_sym)) + KALDI_ERR << "Invalid subsequential symbol "< *fst = ReadFstKaldi(fst_in_filename); + + int32 h = HighestNumberedInputSymbol(*fst); + if (subseq_sym <= h) { + std::cerr << "fstaddsubsequentialloop.cc: subseq symbol does not seem right, "< tmp.fst + ( echo " 0"; echo "a 1"; echo "b 2"; echo "c 3" ) > phones.txt + fstmakecontextsyms phones.txt ilabels.sym > context.txt + fstprint --isymbols=context.txt --osymbols=phones.txt tmp.fst + # and the result is: + +WARNING (fstcomposecontext[5.4]:main():fstcomposecontext.cc:130) Disambiguation symbols list is empty; this likely indicates an error in data preparation. +0 1 a +1 2 /a/b b +2 3 a/b/c c +3 4 b/c/ +4 + + + # (2) with disambig syms: + ( echo 4; echo 5) > disambig.list + ( echo " 0"; echo "a 1"; echo "b 2"; echo "c 3"; echo "#0 4"; echo "#1 5") > phones.txt + ( echo "0 1 1 1"; echo "1 2 2 2"; echo " 2 3 4 4"; echo "3 4 3 3"; echo "4 5 5 5"; echo "5 0" ) | fstcompile > in.fst + fstcomposecontext --read-disambig-syms=disambig.list ilabels.sym in.fst tmp.fst + fstmakecontextsyms phones.txt ilabels.sym > context.txt + cp phones.txt phones_disambig.txt; ( echo "#0 4"; echo "#1 5" ) >> phones_disambig.txt + fstprint --isymbols=context.txt --osymbols=phones_disambig.txt tmp.fst + +0 1 #-1 a +1 2 /a/b b +2 3 #0 #0 +3 4 a/b/c c +4 5 #1 #1 +5 6 b/c/ + +*/ + +int main(int argc, char *argv[]) { + try { + using namespace kaldi; + using namespace fst; + using kaldi::int32; + /* + # fstcomposecontext composes efficiently with a context fst + # that it generates. Without --disambig-syms specified, it + # assumes that all input symbols of in.fst are phones. + # It adds the subsequential symbol itself (it does not + # appear in the output so doesn't need to be specified by the user). + # the disambig.list is a list of disambiguation symbols on the LHS + # of in.fst. The symbols on the LHS of out.fst are indexes into + # the ilabels.list file, which is a kaldi-format file containing a + # vector >, which specifies what the labels mean in + # terms of windows of symbols. + fstcomposecontext ilabels.sym [ in.fst [ out.fst ] ] + --disambig-syms=disambig.list + --context-size=3 + --central-position=1 + --binary=false + */ + + const char *usage = + "Composes on the left with a dynamically created context FST\n" + "\n" + "Usage: fstcomposecontext [ [] ]\n" + "E.g: fstcomposecontext ilabels.sym < LG.fst > CLG.fst\n"; + + + ParseOptions po(usage); + bool binary = true; + std::string disambig_rxfilename, + disambig_wxfilename; + int32 context_width = 3, central_position = 1; + int32 nonterm_phones_offset = -1; + po.Register("binary", &binary, + "If true, output ilabels-output-file in binary format"); + po.Register("read-disambig-syms", &disambig_rxfilename, + "List of disambiguation symbols on input of in.fst"); + po.Register("write-disambig-syms", &disambig_wxfilename, + "List of disambiguation symbols on input of out.fst"); + po.Register("context-size", &context_width, "Size of phone context window"); + po.Register("central-position", ¢ral_position, + "Designated central position in context window"); + po.Register("nonterm-phones-offset", &nonterm_phones_offset, + "The integer id of #nonterm_bos in your phones.txt, if present " + "(only relevant for grammar-FST construction, see " + "doc/grammar.dox"); + + po.Read(argc, argv); + + if (po.NumArgs() < 1 || po.NumArgs() > 3) { + po.PrintUsage(); + exit(1); + } + + std::string ilabels_out_filename = po.GetArg(1), + fst_in_filename = po.GetOptArg(2), + fst_out_filename = po.GetOptArg(3); + + VectorFst *fst = ReadFstKaldi(fst_in_filename); + + if ( (disambig_wxfilename != "") && (disambig_rxfilename == "") ) + KALDI_ERR << "fstcomposecontext: cannot specify --write-disambig-syms if " + "not specifying --read-disambig-syms\n"; + + std::vector disambig_in; + if (disambig_rxfilename != "") + if (!ReadIntegerVectorSimple(disambig_rxfilename, &disambig_in)) + KALDI_ERR << "fstcomposecontext: Could not read disambiguation symbols from " + << PrintableRxfilename(disambig_rxfilename); + + if (disambig_in.empty()) { + KALDI_WARN << "Disambiguation symbols list is empty; this likely " + << "indicates an error in data preparation."; + } + + std::vector > ilabels; + VectorFst composed_fst; + + // Work gets done here (see context-fst.h) + if (nonterm_phones_offset < 0) { + // The normal case. + ComposeContext(disambig_in, context_width, central_position, + fst, &composed_fst, &ilabels); + } else { + // The grammar-FST case. See ../doc/grammar.dox for an intro. + if (context_width != 2 || central_position != 1) { + KALDI_ERR << "Grammar-fst graph creation only supports models with left-" + "biphone context. (--nonterm-phones-offset option was supplied)."; + } + ComposeContextLeftBiphone(nonterm_phones_offset, disambig_in, + *fst, &composed_fst, &ilabels); + } + WriteILabelInfo(Output(ilabels_out_filename, binary).Stream(), + binary, ilabels); + + if (disambig_wxfilename != "") { + std::vector disambig_out; + for (size_t i = 0; i < ilabels.size(); i++) + if (ilabels[i].size() == 1 && ilabels[i][0] <= 0) + disambig_out.push_back(static_cast(i)); + if (!WriteIntegerVectorSimple(disambig_wxfilename, disambig_out)) { + std::cerr << "fstcomposecontext: Could not write disambiguation symbols to " + << PrintableWxfilename(disambig_wxfilename) << '\n'; + return 1; + } + } + + WriteFstKaldi(composed_fst, fst_out_filename); + delete fst; + return 0; + } catch(const std::exception &e) { + std::cerr << e.what(); + return -1; + } +} diff --git a/runtime/onnxruntime/third_party/kaldi/fstbin/fstcopy.cc b/runtime/onnxruntime/third_party/kaldi/fstbin/fstcopy.cc new file mode 100644 index 000000000..a22b148d6 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/fstbin/fstcopy.cc @@ -0,0 +1,78 @@ +// fstbin/fstcopy.cc + +// Copyright 2009-2011 Microsoft Corporation + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + + +#include "base/kaldi-common.h" +#include "util/common-utils.h" +#include "fst/fstlib.h" +#include "fstext/table-matcher.h" +#include "fstext/fstext-utils.h" +#include "fstext/kaldi-fst-io.h" + +/* +Test: +the command below should print out something similar to the given +input. and if you remove ,t from the output side it should print something +binary. +cat < \n"; + + ParseOptions po(usage); + + po.Read(argc, argv); + + if (po.NumArgs() != 2) { + po.PrintUsage(); + exit(1); + } + + std::string fst_rspecifier = po.GetArg(1), + fst_wspecifier = po.GetArg(2); + + SequentialTableReader fst_reader(fst_rspecifier); + TableWriter fst_writer(fst_wspecifier); + int32 n_done = 0; + + for (; !fst_reader.Done(); fst_reader.Next(), n_done++) + fst_writer.Write(fst_reader.Key(), fst_reader.Value()); + + KALDI_LOG << "Copied " << n_done << " FSTs."; + return (n_done != 0 ? 0 : 1); + } catch(const std::exception &e) { + std::cerr << e.what(); + return -1; + } +} + diff --git a/runtime/onnxruntime/third_party/kaldi/fstbin/fstdeterminizelog.cc b/runtime/onnxruntime/third_party/kaldi/fstbin/fstdeterminizelog.cc new file mode 100644 index 000000000..f44624180 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/fstbin/fstdeterminizelog.cc @@ -0,0 +1,65 @@ +// fstbin/fstdeterminizelog.cc + +// Copyright 2009-2011 Microsoft Corporation + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + + +#include "base/kaldi-common.h" +#include "util/kaldi-io.h" +#include "util/parse-options.h" +#include "util/text-utils.h" +#include "fst/fstlib.h" +#include "fstext/fstext-utils.h" +#include "fstext/kaldi-fst-io.h" + +int main(int argc, char *argv[]) { + try { + using namespace kaldi; + using namespace fst; + using kaldi::int32; + + const char *usage = + "Determinizes in the log semiring\n" + "\n" + "Usage: fstdeterminizelog [in.fst [out.fst] ]\n" + "\n" + "See also fstdeterminizestar\n"; + + ParseOptions po(usage); + po.Read(argc, argv); + + if (po.NumArgs() > 2) { + po.PrintUsage(); + exit(1); + } + + std::string fst_in_filename = po.GetOptArg(1), + fst_out_filename = po.GetOptArg(2); + + VectorFst *fst = ReadFstKaldi(fst_in_filename); + + DeterminizeInLog(fst); + + WriteFstKaldi(*fst, fst_out_filename); + delete fst; + return 0; + } catch(const std::exception &e) { + std::cerr << e.what(); + return -1; + } +} + diff --git a/runtime/onnxruntime/third_party/kaldi/fstbin/fstdeterminizestar.cc b/runtime/onnxruntime/third_party/kaldi/fstbin/fstdeterminizestar.cc new file mode 100644 index 000000000..82cc7f6cf --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/fstbin/fstdeterminizestar.cc @@ -0,0 +1,119 @@ +// fstbin/fstdeterminizestar.cc + +// Copyright 2009-2011 Microsoft Corporation + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + + +#include "base/kaldi-common.h" +#include "fst/fstlib.h" +#include "fstext/determinize-star.h" +#include "fstext/fstext-utils.h" +#include "fstext/kaldi-fst-io.h" +#include "util/parse-options.h" + +#if !defined(_MSC_VER) && !defined(__APPLE__) +#include // Comment this line and the call to signal below if +// it causes compilation problems. It is only to enable a debugging procedure +// when determinization does not terminate. We are disabling this code if +// compiling on Windows because signal.h is not available there, and on +// MacOS due to a problem with in the initial release of Sierra. +#endif + +/* some test examples: + ( echo "0 0 0 0"; echo "0 0" ) | fstcompile | fstdeterminizestar | fstprint + ( echo "0 0 1 0"; echo "0 0" ) | fstcompile | fstdeterminizestar | fstprint + ( echo "0 0 1 0"; echo "0 1 1 0"; echo "0 0" ) | fstcompile | fstdeterminizestar | fstprint + # this last one fails [correctly]: + ( echo "0 0 0 1"; echo "0 0" ) | fstcompile | fstdeterminizestar | fstprint + + cd ~/tmpdir + while true; do + fstrand > 1.fst + fstpredeterminize out.lst 1.fst | fstdeterminizestar | fstrmsymbols out.lst > 2.fst + fstequivalent --random=true 1.fst 2.fst || echo "Test failed" + echo -n "." + done + + Test of debugging [with non-determinizable input]: + ( echo " 0 0 1 0 1.0"; echo "0 1 1 0"; echo "1 1 1 0 0"; echo "0 2 2 0"; echo "2"; echo "1" ) | fstcompile | fstdeterminizestar + kill -SIGUSR1 [the process-id of fstdeterminizestar] + # prints out a bunch of debugging output showing the mess it got itself into. +*/ + + +bool debug_location = false; +void signal_handler(int) { + debug_location = true; +} + + + +int main(int argc, char *argv[]) { + try { + using namespace kaldi; + using namespace fst; + using kaldi::int32; + + const char *usage = + "Removes epsilons and determinizes in one step\n" + "\n" + "Usage: fstdeterminizestar [in.fst [out.fst] ]\n" + "\n" + "See also: fstdeterminizelog, lattice-determinize\n"; + + float delta = kDelta; + int max_states = -1; + bool use_log = false; + ParseOptions po(usage); + po.Register("use-log", &use_log, "Determinize in log semiring."); + po.Register("delta", &delta, "Delta value used to determine equivalence of weights."); + po.Register("max-states", &max_states, "Maximum number of states in determinized FST before it will abort."); + po.Read(argc, argv); + + if (po.NumArgs() > 2) { + po.PrintUsage(); + exit(1); + } + + std::string fst_in_str = po.GetOptArg(1), + fst_out_str = po.GetOptArg(2); + + // This enables us to get traceback info from determinization that is + // not seeming to terminate. +#if !defined(_MSC_VER) && !defined(__APPLE__) + signal(SIGUSR1, signal_handler); +#endif + // Normal case: just files. + VectorFst *fst = ReadFstKaldi(fst_in_str); + + ArcSort(fst, ILabelCompare()); // improves speed. + if (use_log) { + DeterminizeStarInLog(fst, delta, &debug_location, max_states); + } else { + VectorFst det_fst; + DeterminizeStar(*fst, &det_fst, delta, &debug_location, max_states); + *fst = det_fst; // will do shallow copy and then det_fst goes + // out of scope anyway. + } + WriteFstKaldi(*fst, fst_out_str); + delete fst; + return 0; + } catch(const std::exception &e) { + std::cerr << e.what(); + return -1; + } +} diff --git a/runtime/onnxruntime/third_party/kaldi/fstbin/fstisstochastic.cc b/runtime/onnxruntime/third_party/kaldi/fstbin/fstisstochastic.cc new file mode 100644 index 000000000..3e0f1651b --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/fstbin/fstisstochastic.cc @@ -0,0 +1,85 @@ +// fstbin/fstisstochastic.cc + +// Copyright 2009-2011 Microsoft Corporation + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + + +#include "base/kaldi-common.h" +#include "util/kaldi-io.h" +#include "util/parse-options.h" +#include "fst/fstlib.h" +#include "fstext/fstext-utils.h" +#include "fstext/kaldi-fst-io.h" + +// e.g. of test: +// echo " 0 0" | fstcompile | fstisstochastic +// should return 0 and print "0 0" [meaning, min and +// max weight are one = exp(0)] +// echo " 0 1" | fstcompile | fstisstochastic +// should return 1, not stochastic, and print 1 1 +// (echo "0 0 0 0 0.693147 "; echo "0 1 0 0 0.693147 "; echo "1 0" ) | fstcompile | fstisstochastic +// should return 0, stochastic; it prints "0 -1.78e-07" for me +// (echo "0 0 0 0 0.693147 "; echo "0 1 0 0 0.693147 "; echo "1 0" ) | fstcompile | fstisstochastic --test-in-log=false +// should return 1, not stochastic in tropical; it prints "0 0.693147" for me +// (echo "0 0 0 0 0 "; echo "0 1 0 0 0 "; echo "1 0" ) | fstcompile | fstisstochastic --test-in-log=false +// should return 0, stochastic in tropical; it prints "0 0" for me +// (echo "0 0 0 0 0.693147 "; echo "0 1 0 0 0.693147 "; echo "1 0" ) | fstcompile | fstisstochastic --test-in-log=false --delta=1 +// returns 0 even though not stochastic because we gave it an absurdly large delta. + +int main(int argc, char *argv[]) { + try { + using namespace kaldi; + using namespace fst; + using kaldi::int32; + + const char *usage = + "Checks whether an FST is stochastic and exits with success if so.\n" + "Prints out maximum error (in log units).\n" + "\n" + "Usage: fstisstochastic [ in.fst ]\n"; + + float delta = 0.01; + bool test_in_log = true; + + ParseOptions po(usage); + po.Register("delta", &delta, "Maximum error to accept."); + po.Register("test-in-log", &test_in_log, "Test stochasticity in log semiring."); + po.Read(argc, argv); + + if (po.NumArgs() > 1) { + po.PrintUsage(); + exit(1); + } + + std::string fst_in_filename = po.GetOptArg(1); + + Fst *fst = ReadFstKaldiGeneric(fst_in_filename); + + bool ans; + StdArc::Weight min, max; + if (test_in_log) ans = IsStochasticFstInLog(*fst, delta, &min, &max); + else ans = IsStochasticFst(*fst, delta, &min, &max); + + std::cout << min.Value() << " " << max.Value() << '\n'; + delete fst; + if (ans) return 0; // success; + else return 1; + } catch(const std::exception &e) { + std::cerr << e.what(); + return -1; + } +} diff --git a/runtime/onnxruntime/third_party/kaldi/fstbin/fstmakecontextfst.cc b/runtime/onnxruntime/third_party/kaldi/fstbin/fstmakecontextfst.cc new file mode 100644 index 000000000..818bf00f5 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/fstbin/fstmakecontextfst.cc @@ -0,0 +1,143 @@ +// fstbin/fstmakecontextfst.cc + +// Copyright 2009-2011 Microsoft Corporation + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + + +#include "base/kaldi-common.h" +#include "util/kaldi-io.h" +#include "util/common-utils.h" +#include "fst/fstlib.h" +#include "fstext/fstext-utils.h" +#include "fstext/context-fst.h" +#include "fstext/kaldi-fst-io.h" + + +/* for example of testing setup, see fstmakecontextsymbols.cc */ + +int main(int argc, char *argv[]) { + try { + using namespace kaldi; + using namespace fst; + using kaldi::int32; + + const char *usage = + "Constructs a context FST with a specified context-width and context-position.\n" + "Outputs the context FST, and a file in Kaldi format that describes what the\n" + "input labels mean. Note: this is very inefficient if there are a lot of phones,\n" + "better to use fstcomposecontext instead\n" + "\n" + "Usage: fstmakecontextfst []\n" + "E.g.: fstmakecontextfst phones.txt 42 ilabels.sym > C.fst\n"; + + bool binary = true; // binary output to ilabels_output_file. + std::string disambig_rxfilename, disambig_wxfilename; + int32 context_width = 3, central_position = 1; + + ParseOptions po(usage); + po.Register("read-disambig-syms", &disambig_rxfilename, + "List of disambiguation symbols to read"); + po.Register("write-disambig-syms", &disambig_wxfilename, + "List of disambiguation symbols to write"); + po.Register("context-size", &context_width, "Size of phonetic context window"); + po.Register("central-position", ¢ral_position, + "Designated central position in context window"); + po.Register("binary", &binary, + "Write ilabels output file in binary Kaldi format"); + + po.Read(argc, argv); + + if (po.NumArgs() < 3 || po.NumArgs() > 4) { + po.PrintUsage(); + exit(1); + } + + std::string phones_symtab_filename = po.GetArg(1); + int32 subseq_sym; + if (!ConvertStringToInteger(po.GetArg(2), &subseq_sym)) + KALDI_ERR << "Invalid subsequential symbol " << po.GetArg(2); + std::string ilabels_out_filename = po.GetArg(3); + std::string fst_out_filename = po.GetOptArg(4); + + + std::vector phone_syms; + { + fst::SymbolTable *phones_symtab = NULL; + { // read phone symbol table. + std::ifstream is(phones_symtab_filename.c_str()); + phones_symtab = fst::SymbolTable::ReadText(is, phones_symtab_filename); + if (!phones_symtab) KALDI_ERR << "Could not read phones symbol-table file "< disambig_in; + if (disambig_rxfilename != "") { + if (!ReadIntegerVectorSimple(disambig_rxfilename, &disambig_in)) + KALDI_ERR << "fstcomposecontext: Could not read disambiguation symbols from " + << PrintableRxfilename(disambig_rxfilename); + } + + if (std::binary_search(phone_syms.begin(), phone_syms.end(), subseq_sym) + || std::binary_search(disambig_in.begin(), disambig_in.end(), subseq_sym)) + KALDI_ERR << "Invalid subsequential symbol " << subseq_sym + << ", already a phone or disambiguation symbol."; + + // 'loop_fst' will be an acceptor FST with single (initial and final) state, with + // a loop for each phone and disambiguation symbol. + StdVectorFst loop_fst; + loop_fst.AddState(); // Add state zero. + loop_fst.SetStart(0); + loop_fst.SetFinal(0, TropicalWeight::One()); + for (size_t i = 0; i < phone_syms.size(); i++) { + int32 sym = phone_syms[i]; + loop_fst.AddArc(0, StdArc(sym, sym, TropicalWeight::One(), 0)); + } + + std::vector > ilabels; + VectorFst context_fst; + + ComposeContext(disambig_in, context_width, central_position, + &loop_fst, &context_fst, &ilabels, true); + + WriteFstKaldi(context_fst, fst_out_filename); + + WriteILabelInfo(Output(ilabels_out_filename, binary).Stream(), + binary, ilabels); + + if (disambig_wxfilename != "") { + std::vector disambig_out; + for (size_t i = 0; i < ilabels.size(); i++) + if (ilabels[i].size() == 1 && ilabels[i][0] <= 0) + disambig_out.push_back(static_cast(i)); + if (!WriteIntegerVectorSimple(disambig_wxfilename, disambig_out)) + KALDI_ERR << "fstcomposecontext: Could not write disambiguation symbols to " + << PrintableWxfilename(disambig_wxfilename); + } + return 0; + } catch(const std::exception &e) { + std::cerr << e.what(); + return -1; + } +} diff --git a/runtime/onnxruntime/third_party/kaldi/fstbin/fstmakecontextsyms.cc b/runtime/onnxruntime/third_party/kaldi/fstbin/fstmakecontextsyms.cc new file mode 100644 index 000000000..c9d493975 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/fstbin/fstmakecontextsyms.cc @@ -0,0 +1,122 @@ +// fstbin/fstmakecontextsyms.cc + +// Copyright 2009-2011 Microsoft Corporation + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#include "tree/context-dep.h" +#include "util/common-utils.h" +#include "fst/fstlib.h" +#include "fstext/fstext-utils.h" +#include "fstext/context-fst.h" + + +/* + Test for this and makecontextfst: + mkdir -p ~/tmpdir + pushd ~/tmpdir + (echo " 0"; echo "a 1"; echo "b 2"; echo "#0 3"; echo "#1 4"; echo "#$ 5" ) > phones.txt + ( echo 3; echo 4 ) > disambig.list + fstmakecontextfst --read-disambig-syms=disambig.list <(grep -v '#' phones.txt) 5 ilabels.int > C.fst + fstmakecontextsyms phones.txt ilabels.int > context_syms.txt + fstprint --isymbols=context_syms.txt --osymbols=phones.txt C.fst > C.txt + + fstrandgen C.fst | fstprint --isymbols=context_syms.txt --osymbols=phones.txt + + Example output: + + fstrandgen C.fst | fstprint --isymbols=context_syms.txt --osymbols=phones.txt +0 1 #-1 b +1 2 /b/ #$ +2 3 #1 #1 +3 4 #0 #0 +4 5 #0 #0 +5 6 #0 #0 +6 7 #0 #0 +7 8 #0 #0 +8 9 #1 #1 +9 +*/ + + +int main(int argc, char *argv[]) { + try { + using namespace kaldi; + using namespace fst; + typedef fst::StdArc::Label Label; + const char *usage = "Create input symbols for CLG\n" + "Usage: fstmakecontextsyms phones-symtab ilabels_input_file [output-symtab.txt]\n" + "E.g.: fstmakecontextsyms phones.txt ilabels.sym > context_symbols.txt\n"; + + ParseOptions po(usage); + + std::string disambig_list_file = "", + phone_separator = "/", + initial_disambig = "#-1"; + + po.Register("phone-separator", &phone_separator, + "Separator for phones in phone-in-context symbols."); + po.Register("initial-disambig", &initial_disambig, + "Name for special disambiguation symbol that occurs at start " + "of context-dependent phone sequences"); + + po.Read(argc, argv); + + if (po.NumArgs() < 2 || po.NumArgs() > 3) { + po.PrintUsage(); + exit(1); + } + + std::string phones_symtab_filename = po.GetArg(1), + ilabel_info_filename = po.GetArg(2), + clg_symtab_filename = po.GetOptArg(3); + + std::vector > ilabel_info; + { + bool binary; + Input ki(ilabel_info_filename, &binary); + ReadILabelInfo(ki.Stream(), + binary, &ilabel_info); + } + + fst::SymbolTable *phones_symtab = NULL; + { // read phone symbol table. + std::ifstream is(phones_symtab_filename.c_str()); + phones_symtab = fst::SymbolTable::ReadText(is, phones_symtab_filename); + if (!phones_symtab) KALDI_ERR << "Could not read phones symbol-table file "<WriteText(std::cout)) + KALDI_ERR << "Cannot write symbol table to standard output."; + } else { + if (!clg_symtab->WriteText(clg_symtab_filename)) + KALDI_ERR << "Cannot open symbol table file "< 2) { + po.PrintUsage(); + exit(1); + } + + std::string fst_in_filename = po.GetOptArg(1), + fst_out_filename = po.GetOptArg(2); + + VectorFst *fst = ReadFstKaldi(fst_in_filename); + + MinimizeEncoded(fst, delta); + + WriteFstKaldi(*fst, fst_out_filename); + + delete fst; + return 0; + } catch(const std::exception &e) { + std::cerr << e.what(); + return -1; + } + return 0; +} + diff --git a/runtime/onnxruntime/third_party/kaldi/fstbin/fstphicompose.cc b/runtime/onnxruntime/third_party/kaldi/fstbin/fstphicompose.cc new file mode 100644 index 000000000..13ffcf39c --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/fstbin/fstphicompose.cc @@ -0,0 +1,183 @@ +// fstbin/fstphicompose.cc + +// Copyright 2009-2011 Microsoft Corporation + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + + +#include "base/kaldi-common.h" +#include "util/common-utils.h" +#include "fst/fstlib.h" +#include "fstext/table-matcher.h" +#include "fstext/fstext-utils.h" +#include "fstext/kaldi-fst-io.h" + +/* +The following commands represent a basic test of this program. + +cat < a.fst +0 1 10 10 +0 1 11 11 +1 +EOF + +cat < g.fst +0 1 10 10 2.0 +0 2 100 100 6.6 +2 1 10 10 0.0 +2 1 11 11 1.0 +1 +EOF +fstcompose a.fst g.fst | fstprint +# gives, as expected: +# 0 1 10 10 2 +# 1 +fstphicompose 100 a.fst g.fst | fstprint +# gives, again correctly: +#0 1 10 10 2 +#0 1 11 11 7.5999999 +#1 + +Next, test that it's working as desired for final-probs, +i.e. it takes the backoff arc when looking for a final-prob, +only if no final-prob present at current state. + +cat < a.fst +0 1 10 10 +0 1 11 11 +1 +EOF +cat < g.fst +0 1 10 10 2.0 +0 3 11 11 2.0 +1 2 100 110 6.6 +3 10.0 +3 2 100 110 0.0 +2 +EOF +fstphicompose 100 a.fst g.fst | fstprint +# output is: +#0 1 10 10 2 +#0 2 11 11 2 +#1 6.5999999 +#2 10 + +*/ + +int main(int argc, char *argv[]) { + try { + using namespace kaldi; + using namespace fst; + using kaldi::int32; + /* + fstphicompose does composition, but treats the second FST + specially (basically, like a backoff LM); whenever the + composition algorithm fails to find a match for a label + in the second FST, if it sees a phi transition it will + take it instead, and look for a match at the destination. + phi is the label on the input side of the backoff arc of + the LM (the label on the output side doesn't matter). + + Also modifies the second fst so that it treats final-probs + "correctly", i.e. takes the failure transition when looking + for a final-prob. This would not work if there were + epsilons. + */ + + const char *usage = + "Composition, where the right FST has \"failure\" (phi) transitions\n" + "that are only taken where there was no match of a \"real\" label\n" + "You supply the label corresponding to phi.\n" + "\n" + "Usage: fstphicompose phi-label (fst1-rxfilename|fst1-rspecifier) " + "(fst2-rxfilename|fst2-rspecifier) [(out-rxfilename|out-rspecifier)]\n" + "E.g.: fstphicompose 54 a.fst b.fst c.fst\n" + "or: fstphicompose 11 ark:a.fsts G.fst ark:b.fsts\n"; + + ParseOptions po(usage); + + po.Read(argc, argv); + + if (po.NumArgs() < 3 || po.NumArgs() > 4) { + po.PrintUsage(); + exit(1); + } + + std::string + phi_str = po.GetArg(1), + fst1_in_str = po.GetArg(2), + fst2_in_str = po.GetArg(3), + fst_out_str = po.GetOptArg(4); + + bool is_table_1 = + (ClassifyRspecifier(fst1_in_str, NULL, NULL) != kNoRspecifier), + is_table_2 = + (ClassifyRspecifier(fst2_in_str, NULL, NULL) != kNoRspecifier), + is_table_out = + (ClassifyWspecifier(fst_out_str, NULL, NULL, NULL) != kNoWspecifier); + + int32 phi_label; + if (!ConvertStringToInteger(phi_str, &phi_label) + || phi_label <= 0) + KALDI_ERR << "Invalid first argument (phi label), expect positive integer."; + + if (is_table_out != (is_table_1 || is_table_2)) + KALDI_ERR << "Incompatible combination of archives and files"; + + if (!is_table_1 && !is_table_2) { // Only dealing with files... + VectorFst *fst1 = ReadFstKaldi(fst1_in_str); + + VectorFst *fst2 = ReadFstKaldi(fst2_in_str); + + PropagateFinal(phi_label, fst2); // makes it work correctly + // w.r.t. final-probs. + + VectorFst composed_fst; + + PhiCompose(*fst1, *fst2, phi_label, &composed_fst); + + delete fst1; + delete fst2; + + WriteFstKaldi(composed_fst, fst_out_str); + return 0; + } else if (is_table_1 && !is_table_2) { + + VectorFst *fst2 = ReadFstKaldi(fst2_in_str); + PropagateFinal(phi_label, fst2); // makes it work correctly + // w.r.t. final-probs. + SequentialTableReader fst1_reader(fst1_in_str); + TableWriter fst_writer(fst_out_str); + int32 n_done = 0; + for (; !fst1_reader.Done(); fst1_reader.Next(), n_done++) { + VectorFst fst1(fst1_reader.Value()); + VectorFst fst_out; + PhiCompose(fst1, *fst2, phi_label, &fst_out); + fst_writer.Write(fst1_reader.Key(), fst_out); + } + KALDI_LOG << "Composed " << n_done << " FSTs."; + return (n_done != 0 ? 0 : 1); + } else { + KALDI_ERR << "The combination of tables/non-tables that you " + << "supplied is not currently supported. Either implement this, " + << "ask the maintainers to implement it, or call this program " + << "differently."; + } + } catch(const std::exception &e) { + std::cerr << e.what(); + return -1; + } +} diff --git a/runtime/onnxruntime/third_party/kaldi/fstbin/fstpushspecial.cc b/runtime/onnxruntime/third_party/kaldi/fstbin/fstpushspecial.cc new file mode 100644 index 000000000..b7428f2b9 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/fstbin/fstpushspecial.cc @@ -0,0 +1,72 @@ +// fstbin/fstpushspecial.cc + +// Copyright 2012 Daniel Povey + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + + +#include "base/kaldi-common.h" +#include "util/kaldi-io.h" +#include "util/parse-options.h" +#include "util/text-utils.h" +#include "fst/fstlib.h" +#include "fstext/fstext-utils.h" +#include "fstext/push-special.h" +#include "fstext/kaldi-fst-io.h" + +int main(int argc, char *argv[]) { + try { + using namespace kaldi; + using namespace fst; + using kaldi::int32; + + const char *usage = + "Pushes weights in an FST such that all the states\n" + "in the FST have arcs and final-probs with weights that\n" + "sum to the same amount (viewed as being in the log semiring).\n" + "Thus, the \"extra weight\" is distributed throughout the FST.\n" + "Tolerance parameter --delta controls how exact this is, and the\n" + "speed.\n" + "\n" + "Usage: fstpushspecial [options] [in.fst [out.fst] ]\n"; + + BaseFloat delta = kDelta; + ParseOptions po(usage); + po.Register("delta", &delta, "Delta cost: after pushing, all states will " + "have a total weight that differs from the average by no more " + "than this."); + po.Read(argc, argv); + + if (po.NumArgs() > 2) { + po.PrintUsage(); + exit(1); + } + + std::string fst_in_filename = po.GetOptArg(1), + fst_out_filename = po.GetOptArg(2); + + VectorFst *fst = ReadFstKaldi(fst_in_filename); + + PushSpecial(fst, delta); + + WriteFstKaldi(*fst, fst_out_filename); + delete fst; + return 0; + } catch(const std::exception &e) { + std::cerr << e.what(); + return -1; + } +} diff --git a/runtime/onnxruntime/third_party/kaldi/fstbin/fstrand.cc b/runtime/onnxruntime/third_party/kaldi/fstbin/fstrand.cc new file mode 100644 index 000000000..f0bc39380 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/fstbin/fstrand.cc @@ -0,0 +1,67 @@ +// fstbin/fstrand.cc + +// Copyright 2009-2011 Microsoft Corporation + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + + +#include "base/kaldi-common.h" +#include "util/kaldi-io.h" +#include "util/parse-options.h" +#include "util/text-utils.h" +#include "fstext/rand-fst.h" +#include "time.h" +#include "fstext/fstext-utils.h" +#include "fstext/kaldi-fst-io.h" + +int main(int argc, char *argv[]) { + try { + using namespace fst; + using kaldi::int32; + + const char *usage = + "Generate random FST\n" + "\n" + "Usage: fstrand [out.fst]\n"; + + srand(time(NULL)); + RandFstOptions opts; + + + kaldi::ParseOptions po(usage); + po.Register("allow-empty", &opts.allow_empty, + "If true, we may generate an empty FST."); + + po.Read(argc, argv); + + if (po.NumArgs() > 1) { + po.PrintUsage(); + exit(1); + } + + std::string fst_out_filename = po.GetOptArg(1); + + VectorFst *rand_fst = RandFst(opts); + + WriteFstKaldi(*rand_fst, fst_out_filename); + delete rand_fst; + return 0; + } catch(const std::exception &e) { + std::cerr << e.what(); + return -1; + } +} + diff --git a/runtime/onnxruntime/third_party/kaldi/fstbin/fstrmepslocal.cc b/runtime/onnxruntime/third_party/kaldi/fstbin/fstrmepslocal.cc new file mode 100644 index 000000000..3aedd6dd7 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/fstbin/fstrmepslocal.cc @@ -0,0 +1,101 @@ +// fstbin/fstrmepslocal.cc + +// Copyright 2009-2011 Microsoft Corporation + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + + +#include "base/kaldi-common.h" +#include "util/kaldi-io.h" +#include "util/parse-options.h" +#include "util/text-utils.h" +#include "fst/fstlib.h" +#include "fstext/determinize-star.h" +#include "fstext/fstext-utils.h" +#include "fstext/kaldi-fst-io.h" + + +/* + A test example: + ( echo "0 1 1 0"; echo "1 2 0 2"; echo "2 0"; ) | fstcompile | fstrmepslocal | fstprint +# prints: +# 0 1 1 2 +# 1 + ( echo "0 1 0 0"; echo "0 0"; echo "1 0" ) | fstcompile | fstrmepslocal | fstprint +# 0 + ( echo "0 1 0 0"; echo "0 0"; echo "1 0" ) | fstcompile | fstrmepslocal | fstprint + ( echo "0 1 0 0"; echo "0 0"; echo "1 0" ) | fstcompile | fstrmepslocal --use-log=true | fstprint +# 0 -0.693147182 + +*/ + +int main(int argc, char *argv[]) { + try { + using namespace kaldi; + using namespace fst; + using kaldi::int32; + + const char *usage = + "Removes some (but not all) epsilons in an algorithm that will always reduce the number of\n" + "arcs+states. Option to preserves equivalence in tropical or log semiring, and\n" + "if in tropical, stochasticit in either log or tropical.\n" + "\n" + "Usage: fstrmepslocal [in.fst [out.fst] ]\n"; + + ParseOptions po(usage); + bool use_log = false; + bool stochastic_in_log = true; + po.Register("use-log", &use_log, + "Preserve equivalence in log semiring [false->tropical]\n"); + po.Register("stochastic-in-log", &stochastic_in_log, + "Preserve stochasticity in log semiring [false->tropical]\n"); + po.Read(argc, argv); + + if (po.NumArgs() > 2) { + po.PrintUsage(); + exit(1); + } + + std::string fst_in_filename = po.GetOptArg(1), + fst_out_filename = po.GetOptArg(2); + + VectorFst *fst = ReadFstKaldi(fst_in_filename); + + if (!use_log && stochastic_in_log) { + RemoveEpsLocalSpecial(fst); + } else if (use_log && !stochastic_in_log) { + std::cerr << "fstrmsymbols: invalid combination of flags\n"; + return 1; + } else if (use_log) { + VectorFst log_fst; + Cast(*fst, &log_fst); + delete fst; + RemoveEpsLocal(&log_fst); + fst = new VectorFst; + Cast(log_fst, fst); + } else { + RemoveEpsLocal(fst); + } + + WriteFstKaldi(*fst, fst_out_filename); + delete fst; + return 0; + } catch(const std::exception &e) { + std::cerr << e.what(); + return -1; + } +} + diff --git a/runtime/onnxruntime/third_party/kaldi/fstbin/fstrmsymbols.cc b/runtime/onnxruntime/third_party/kaldi/fstbin/fstrmsymbols.cc new file mode 100644 index 000000000..a365b016e --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/fstbin/fstrmsymbols.cc @@ -0,0 +1,193 @@ +// fstbin/fstrmsymbols.cc + +// Copyright 2009-2011 Microsoft Corporation + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + + +#include "base/kaldi-common.h" +#include "util/common-utils.h" +#include "fst/fstlib.h" +#include "fstext/determinize-star.h" +#include "fstext/fstext-utils.h" +#include "fstext/kaldi-fst-io.h" + +namespace fst { +// we can move these functions elsewhere later, if they are needed in other +// places. + +template +void RemoveArcsWithSomeInputSymbols(const std::vector &symbols_in, + VectorFst *fst) { + typedef typename Arc::StateId StateId; + + kaldi::ConstIntegerSet symbol_set(symbols_in); + + StateId num_states = fst->NumStates(); + StateId dead_state = fst->AddState(); + for (StateId s = 0; s < num_states; s++) { + for (MutableArcIterator > iter(fst, s); + !iter.Done(); iter.Next()) { + if (symbol_set.count(iter.Value().ilabel) != 0) { + Arc arc = iter.Value(); + arc.nextstate = dead_state; + iter.SetValue(arc); + } + } + } + // Connect() will actually remove the arcs, and the dead state. + Connect(fst); + if (fst->NumStates() == 0) + KALDI_WARN << "After Connect(), fst was empty."; +} + +template +void PenalizeArcsWithSomeInputSymbols(const std::vector &symbols_in, + float penalty, + VectorFst *fst) { + typedef typename Arc::StateId StateId; + typedef typename Arc::Label Label; + typedef typename Arc::Weight Weight; + + Weight penalty_weight(penalty); + + kaldi::ConstIntegerSet symbol_set(symbols_in); + + StateId num_states = fst->NumStates(); + for (StateId s = 0; s < num_states; s++) { + for (MutableArcIterator > iter(fst, s); + !iter.Done(); iter.Next()) { + if (symbol_set.count(iter.Value().ilabel) != 0) { + Arc arc = iter.Value(); + arc.weight = Times(arc.weight, penalty_weight); + iter.SetValue(arc); + } + } + } +} + +} + + +int main(int argc, char *argv[]) { + try { + using namespace kaldi; + using namespace fst; + using kaldi::int32; + + bool apply_to_output = false; + bool remove_arcs = false; + float penalty = -std::numeric_limits::infinity(); + + const char *usage = + "With no options, replaces a subset of symbols with epsilon, wherever\n" + "they appear on the input side of an FST." + "With --remove-arcs=true, will remove arcs that contain these symbols\n" + "on the input\n" + "With --penalty=, will add the specified penalty to the\n" + "cost of any arc that has one of the given symbols on its input side\n" + "In all cases, the option --apply-to-output=true (or for\n" + "back-compatibility, --remove-from-output=true) makes this apply\n" + "to the output side.\n" + "\n" + "Usage: fstrmsymbols [options] [ []]\n" + "E.g: fstrmsymbols in.list < in.fst > out.fst\n" + " is an rxfilename specifying a file containing list of integers\n" + "representing symbols, in text form, one per line.\n"; + + ParseOptions po(usage); + po.Register("remove-from-output", &apply_to_output, "If true, this applies to symbols " + "on the output, not the input, side. (For back compatibility; use " + "--apply-to-output insead)"); + po.Register("apply-to-output", &apply_to_output, "If true, this applies to symbols " + "on the output, not the input, side."); + po.Register("remove-arcs", &remove_arcs, "If true, instead of converting the symbol " + "to , remove the arcs."); + po.Register("penalty", &penalty, "If specified, instead of converting " + "the symbol to , penalize the arc it is on by adding this " + "value to its cost."); + + + po.Read(argc, argv); + + if (remove_arcs && + penalty != -std::numeric_limits::infinity()) + KALDI_ERR << "--remove-arc and --penalty options are mutually exclusive"; + + if (po.NumArgs() < 1 || po.NumArgs() > 3) { + po.PrintUsage(); + exit(1); + } + + std::string disambig_rxfilename = po.GetArg(1), + fst_rxfilename = po.GetOptArg(2), + fst_wxfilename = po.GetOptArg(3); + + VectorFst *fst = CastOrConvertToVectorFst( + ReadFstKaldiGeneric(fst_rxfilename)); + + std::vector disambig_in; + if (!ReadIntegerVectorSimple(disambig_rxfilename, &disambig_in)) + KALDI_ERR << "fstrmsymbols: Could not read disambiguation symbols from " + << (disambig_rxfilename == "" ? "standard input" : disambig_rxfilename); + + if (apply_to_output) Invert(fst); + if (remove_arcs) { + RemoveArcsWithSomeInputSymbols(disambig_in, fst); + } else if (penalty != -std::numeric_limits::infinity()) { + PenalizeArcsWithSomeInputSymbols(disambig_in, penalty, fst); + } else { + RemoveSomeInputSymbols(disambig_in, fst); + } + if (apply_to_output) Invert(fst); + + WriteFstKaldi(*fst, fst_wxfilename); + + delete fst; + return 0; + } catch(const std::exception &e) { + std::cerr << e.what(); + return -1; + } +} + +/* some test examples: + + ( echo "0 0 1 1"; echo " 0 0 3 2"; echo "0 0"; ) | fstcompile | fstrmsymbols "echo 3; echo 4|" | fstprint + # should produce: + # 0 0 1 1 + # 0 0 0 2 + # 0 + + ( echo "0 0 1 1"; echo " 0 0 3 2"; echo "0 0"; ) | fstcompile | fstrmsymbols --apply-to-output=true "echo 2; echo 3|" | fstprint + # should produce: + # 0 0 1 1 + # 0 0 3 0 + # 0 + + + ( echo "0 0 1 1"; echo " 0 0 3 2"; echo "0 0"; ) | fstcompile | fstrmsymbols --remove-arcs=true "echo 3; echo 4|" | fstprint + # should produce: + # 0 0 1 1 + # 0 + + ( echo "0 0 1 1"; echo " 0 0 3 2"; echo "0 0"; ) | fstcompile | fstrmsymbols --penalty=2 "echo 3; echo 4; echo 5|" | fstprint +# should produce: + # 0 0 1 1 + # 0 0 3 2 2 + # 0 + +*/ diff --git a/runtime/onnxruntime/third_party/kaldi/fstbin/fsts-concat.cc b/runtime/onnxruntime/third_party/kaldi/fstbin/fsts-concat.cc new file mode 100644 index 000000000..2a217eda7 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/fstbin/fsts-concat.cc @@ -0,0 +1,112 @@ +// fstbin/fsts-concat.cc + +// Copyright 2016 Johns Hopkins University (Authors: Jan "Yenda" Trmal) +// 2018 Soapbox Labs (Author: Karel Vesely) + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + + +#include "base/kaldi-common.h" +#include "util/common-utils.h" +#include "fstext/fstext-utils.h" +#include "fstext/kaldi-fst-io.h" + + +int main(int argc, char *argv[]) { + try { + using namespace kaldi; + using namespace fst; + typedef kaldi::int32 int32; + typedef kaldi::uint64 uint64; + + const char *usage = + "Reads kaldi archives with FSTs. Concatenates the fsts from all the rspecifiers.\n" + "The fsts to concatenate must have same key. The sequencing is given by the position of arguments.\n" + "\n" + "Usage: fsts-concat [options] ... \n" + " e.g.: fsts-concat scp:fsts1.scp scp:fsts2.scp ... ark:fsts_out.ark\n" + "\n" + "see also: fstconcat (from the OpenFst toolkit)\n"; + + ParseOptions po(usage); + + po.Read(argc, argv); + + if (po.NumArgs() < 3) { + po.PrintUsage(); + exit(1); + } + + std::string fsts_rspecifier = po.GetArg(1), + fsts_wspecifier = po.GetArg(po.NumArgs()); + + SequentialTableReader fst_reader(fsts_rspecifier); + std::vector*> fst_readers; + TableWriter fst_writer(fsts_wspecifier); + + for (int32 i = 2; i < po.NumArgs(); i++) + fst_readers.push_back(new RandomAccessTableReader(po.GetArg(i))); + const int32 num_fst_readers = fst_readers.size(); + + int32 n_done = 0, + n_skipped = 0; + + for (; !fst_reader.Done(); fst_reader.Next()) { + std::string key = fst_reader.Key(); + + // Check that the key exists in all 'fst_readers'. + bool skip_key = false; + for (int32 i = 0; i < num_fst_readers; i++) { + if (!fst_readers[i]->HasKey(key)) { + KALDI_WARN << "Skipping '" << key << "'" + << " due to missing the fst in " << (i+2) << "th : " + << "'" << po.GetArg(i+2) << "'"; + skip_key = true; + } + } + if (skip_key) { + n_skipped++; + continue; + } + + // Concatenate! + VectorFst fst_out = fst_readers.back()->Value(key); + // Loop from (last-1) to first, as 'prepending' the fsts is faster, + // see: http://www.openfst.org/twiki/bin/view/FST/ConcatDoc + for (int32 i = num_fst_readers-2; i >= 0; i--) { + fst::Concat(fst_readers[i]->Value(key), &fst_out); + } + // Finally, prepend the fst from the 'Sequential' reader. + fst::Concat(fst_reader.Value(), &fst_out); + + // Write the output. + fst_writer.Write(key, fst_out); + n_done++; + } + + // Cleanup. + for (int32 i = 0; i < num_fst_readers; i++) + delete fst_readers[i]; + fst_readers.clear(); + + KALDI_LOG << "Produced " << n_done << " FSTs by concatenating " << po.NumArgs()-1 + << " streams " << "(" << n_skipped << " keys skipped)."; + return (n_done != 0 ? 0 : 1); + } catch(const std::exception &e) { + std::cerr << e.what(); + return -1; + } +} diff --git a/runtime/onnxruntime/third_party/kaldi/fstbin/fsts-project.cc b/runtime/onnxruntime/third_party/kaldi/fstbin/fsts-project.cc new file mode 100644 index 000000000..015f14317 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/fstbin/fsts-project.cc @@ -0,0 +1,82 @@ +// fstbin/fsts-project.cc + +// Copyright 2016 Johns Hopkins University (Authors: Jan "Yenda" Trmal) + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + + +#include "base/kaldi-common.h" +#include "util/common-utils.h" +#include "fstext/fstext-utils.h" +#include "fstext/kaldi-fst-io.h" + + +int main(int argc, char *argv[]) { + try { + using namespace kaldi; + using namespace fst; + typedef kaldi::int32 int32; + typedef kaldi::uint64 uint64; + + const char *usage = + "Reads kaldi archive of FSTs; for each element, performs the project\n" + "operation either on input (default) or on the output (if the option\n" + "--project-output is true).\n" + "\n" + "Usage: fsts-project [options] \n" + " e.g.: fsts-project ark:train.fsts ark,t:train.fsts\n" + "\n" + "see also: fstproject (from the OpenFst toolkit)\n"; + + ParseOptions po(usage); + + bool project_output = false; + + po.Register("project-output", &project_output, + "If true, project output vs input"); + + po.Read(argc, argv); + + if (po.NumArgs() != 2) { + po.PrintUsage(); + exit(1); + } + + std::string fsts_rspecifier = po.GetArg(1), + fsts_wspecifier = po.GetArg(2); + + + SequentialTableReader fst_reader(fsts_rspecifier); + TableWriter fst_writer(fsts_wspecifier); + + int32 n_done = 0; + for (; !fst_reader.Done(); fst_reader.Next()) { + std::string key = fst_reader.Key(); + VectorFst fst(fst_reader.Value()); + + Project(&fst, project_output ? PROJECT_OUTPUT : PROJECT_INPUT); + + fst_writer.Write(key, fst); + n_done++; + } + + KALDI_LOG << "Projected " << n_done << " FSTs"; + return (n_done != 0 ? 0 : 1); + } catch(const std::exception &e) { + std::cerr << e.what(); + return -1; + } +} diff --git a/runtime/onnxruntime/third_party/kaldi/fstbin/fsts-to-transcripts.cc b/runtime/onnxruntime/third_party/kaldi/fstbin/fsts-to-transcripts.cc new file mode 100644 index 000000000..ae74d5de6 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/fstbin/fsts-to-transcripts.cc @@ -0,0 +1,103 @@ +// fstbin/fsts-to-transcripts.cc + +// Copyright 2012-2013 Johns Hopkins University (Authors: Guoguo Chen, +// Daniel Povey) + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + + +#include "base/kaldi-common.h" +#include "util/common-utils.h" +#include "fstext/fstext-utils.h" +#include "fstext/kaldi-fst-io.h" + + +int main(int argc, char *argv[]) { + try { + using namespace kaldi; + using namespace fst; + typedef kaldi::int32 int32; + typedef kaldi::uint64 uint64; + + const char *usage = + "Reads a table of FSTs; for each element, finds the best path and \n" + "prints out the output-symbol sequence (if --output-side=true), or \n" + "input-symbol sequence otherwise.\n" + "\n" + "Usage:\n" + " fsts-to-transcripts [options] " + " \n" + "e.g.:\n" + " fsts-to-transcripts ark:train.fsts ark,t:train.text\n"; + + ParseOptions po(usage); + + bool output_side = true; + + po.Register("output-side", &output_side, "If true, extract the symbols on " + "the output side of the FSTs, else the input side."); + + po.Read(argc, argv); + + if (po.NumArgs() != 2) { + po.PrintUsage(); + exit(1); + } + + std::string fst_rspecifier = po.GetArg(1), + transcript_wspecifier = po.GetArg(2); + + + SequentialTableReader fst_reader(fst_rspecifier); + Int32VectorWriter transcript_writer(transcript_wspecifier); + + int32 n_done = 0, n_err = 0; + for (; !fst_reader.Done(); fst_reader.Next()) { + std::string key = fst_reader.Key(); + const VectorFst &fst = fst_reader.Value(); + + + VectorFst shortest_path; + ShortestPath(fst, &shortest_path); // the OpenFst algorithm ShortestPath. + + if (shortest_path.NumStates() == 0) { + KALDI_WARN << "Input FST (after shortest path) was empty. Producing " + << "no output for key " << key; + n_err++; + continue; + } + + std::vector transcript; + bool ans; + if (output_side) ans = fst::GetLinearSymbolSequence( + shortest_path, NULL, &transcript, NULL); + else + ans = fst::GetLinearSymbolSequence( + shortest_path, &transcript, NULL, NULL); + if (!ans) { + KALDI_ERR << "GetLinearSymbolSequence returned false (code error);"; + } + transcript_writer.Write(key, transcript); + n_done++; + } + + KALDI_LOG << "Converted " << n_done << " FSTs, " << n_err << " with errors"; + return (n_done != 0 ? 0 : 1); + } catch(const std::exception &e) { + std::cerr << e.what(); + return -1; + } +} diff --git a/runtime/onnxruntime/third_party/kaldi/fstbin/fsts-union.cc b/runtime/onnxruntime/third_party/kaldi/fstbin/fsts-union.cc new file mode 100644 index 000000000..ed68cea76 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/fstbin/fsts-union.cc @@ -0,0 +1,100 @@ +// fstbin/fsts-union.cc + +// Copyright 2016 Johns Hopkins University (Authors: Jan "Yenda" Trmal) + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + + +#include "base/kaldi-common.h" +#include "util/common-utils.h" +#include "fstext/fstext-utils.h" +#include "fstext/kaldi-fst-io.h" + + +int main(int argc, char *argv[]) { + try { + using namespace kaldi; + using namespace fst; + typedef kaldi::int32 int32; + typedef kaldi::uint64 uint64; + + const char *usage = + "Reads a kaldi archive of FSTs. Performs the FST operation union on\n" + "all fsts sharing the same key. Assumes the archive is sorted by key.\n" + "\n" + "Usage: fsts-union [options] \n" + " e.g.: fsts-union ark:keywords_tmp.fsts ark,t:keywords.fsts\n" + "\n" + "see also: fstunion (from the OpenFst toolkit)\n"; + + ParseOptions po(usage); + + po.Read(argc, argv); + + if (po.NumArgs() != 2) { + po.PrintUsage(); + exit(1); + } + + std::string fsts_rspecifier = po.GetArg(1), + fsts_wspecifier = po.GetArg(2); + + + SequentialTableReader fst_reader(fsts_rspecifier); + TableWriter fst_writer(fsts_wspecifier); + + int32 n_out_done = 0, + n_in_done = 0; + std::string res_key = ""; + VectorFst res_fst; + + for (; !fst_reader.Done(); fst_reader.Next()) { + std::string key = fst_reader.Key(); + VectorFst fst(fst_reader.Value()); + + n_in_done++; + if (key == res_key) { + fst::Union(&res_fst, fst); + } else { + if (res_key != "") { + VectorFst out_fst; + fst::Determinize(res_fst, &out_fst); + fst::Minimize(&out_fst); + fst::RmEpsilon(&out_fst); + fst_writer.Write(res_key, out_fst); + n_out_done++; + } + res_fst = fst; + res_key = key; + } + } + if (res_key != "") { + VectorFst out_fst; + fst::Determinize(res_fst, &out_fst); + fst::Minimize(&out_fst); + fst::RmEpsilon(&out_fst); + fst_writer.Write(res_key, out_fst); + n_out_done++; + } + + KALDI_LOG << "Applied fst union on " << n_in_done + << " FSTs, produced " << n_out_done << " FSTs"; + return (n_out_done != 0 ? 0 : 1); + } catch(const std::exception &e) { + std::cerr << e.what(); + return -1; + } +} diff --git a/runtime/onnxruntime/third_party/kaldi/fstbin/fsttablecompose.cc b/runtime/onnxruntime/third_party/kaldi/fstbin/fsttablecompose.cc new file mode 100644 index 000000000..6652a8c90 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/fstbin/fsttablecompose.cc @@ -0,0 +1,202 @@ +// fstbin/fsttablecompose.cc + +// Copyright 2009-2011 Microsoft Corporation +// 2013 Johns Hopkins University (author: Daniel Povey) + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + + +#include "base/kaldi-common.h" +//#include "util/common-utils.h" +#include "fst/fstlib.h" +#include "fstext/table-matcher.h" +#include "fstext/fstext-utils.h" +#include "fstext/kaldi-fst-io.h" +#include "util/parse-options.h" + +/* + cd ~/tmpdir + while true; do + fstrand | fstarcsort --sort_type=olabel > 1.fst; fstrand | fstarcsort > 2.fst + fstcompose 1.fst 2.fst > 3a.fst + fsttablecompose 1.fst 2.fst > 3b.fst + fstequivalent --random=true 3a.fst 3b.fst || echo "Test failed" + echo -n "." + done + +*/ + +int main(int argc, char *argv[]) { + try { + using namespace kaldi; + using namespace fst; + using kaldi::int32; + /* + fsttablecompose should always give equivalent results to compose, + but it is more efficient for certain kinds of inputs. + In particular, it is useful when, say, the left FST has states + that typically either have epsilon olabels, or + one transition out for each of the possible symbols (as the + olabel). The same with the input symbols of the right-hand FST + is possible. + */ + + const char *usage = + "Composition algorithm [between two FSTs of standard type, in tropical\n" + "semiring] that is more efficient for certain cases-- in particular,\n" + "where one of the FSTs (the left one, if --match-side=left) has large\n" + "out-degree\n" + "\n" + "Usage: fsttablecompose (fst1-rxfilename|fst1-rspecifier) " + "(fst2-rxfilename|fst2-rspecifier) [(out-rxfilename|out-rspecifier)]\n"; + + ParseOptions po(usage); + + TableComposeOptions opts; + std::string match_side = "left"; + std::string compose_filter = "sequence"; + + po.Register("connect", &opts.connect, "If true, trim FST before output."); + po.Register("match-side", &match_side, "Side of composition to do table " + "match, one of: \"left\" or \"right\"."); + po.Register("compose-filter", &compose_filter, "Composition filter to use, " + "one of: \"alt_sequence\", \"auto\", \"match\", \"sequence\""); + + po.Read(argc, argv); + + if (match_side == "left") { + opts.table_match_type = MATCH_OUTPUT; + } else if (match_side == "right") { + opts.table_match_type = MATCH_INPUT; + } else { + KALDI_ERR << "Invalid match-side option: " << match_side; + } + + if (compose_filter == "alt_sequence") { + opts.filter_type = ALT_SEQUENCE_FILTER; + } else if (compose_filter == "auto") { + opts.filter_type = AUTO_FILTER; + } else if (compose_filter == "match") { + opts.filter_type = MATCH_FILTER; + } else if (compose_filter == "sequence") { + opts.filter_type = SEQUENCE_FILTER; + } else { + KALDI_ERR << "Invalid compose-filter option: " << compose_filter; + } + + if (po.NumArgs() < 2 || po.NumArgs() > 3) { + po.PrintUsage(); + exit(1); + } + + std::string fst1_in_str = po.GetArg(1), + fst2_in_str = po.GetArg(2), + fst_out_str = po.GetOptArg(3); + + + //// Note: the "table" in is_table_1 and similar variables has nothing + //// to do with the "table" in "fsttablecompose"; is_table_1 relates to + //// whether we are dealing with a single FST or a whole set of FSTs. + //bool is_table_1 = + // (ClassifyRspecifier(fst1_in_str, NULL, NULL) != kNoRspecifier), + // is_table_2 = + // (ClassifyRspecifier(fst2_in_str, NULL, NULL) != kNoRspecifier), + // is_table_out = + // (ClassifyWspecifier(fst_out_str, NULL, NULL, NULL) != kNoWspecifier); + //if (is_table_out != (is_table_1 || is_table_2)) + // KALDI_ERR << "Incompatible combination of archives and files"; + // + //if (!is_table_1 && !is_table_2) { // Only dealing with files... + VectorFst *fst1 = ReadFstKaldi(fst1_in_str); + + VectorFst *fst2 = ReadFstKaldi(fst2_in_str); + + // Checks if is olabel sorted and is ilabel sorted. + if (fst1->Properties(fst::kOLabelSorted, true) == 0) { + KALDI_WARN << "The first FST is not olabel sorted."; + } + if (fst2->Properties(fst::kILabelSorted, true) == 0) { + KALDI_WARN << "The second FST is not ilabel sorted."; + } + + VectorFst composed_fst; + + TableCompose(*fst1, *fst2, &composed_fst, opts); + + delete fst1; + delete fst2; + + WriteFstKaldi(composed_fst, fst_out_str); + return 0; + //} else if (!is_table_1 && is_table_2 + // && opts.table_match_type == MATCH_OUTPUT) { + // // second arg is an archive, and match-side=left (default). + // TableComposeCache > cache(opts); + // VectorFst *fst1 = ReadFstKaldi(fst1_in_str); + // SequentialTableReader fst2_reader(fst2_in_str); + // TableWriter fst_writer(fst_out_str); + // int32 n_done = 0; + + // // Checks if is olabel sorted. + // if (fst1->Properties(fst::kOLabelSorted, true) == 0) { + // KALDI_WARN << "The first FST is not olabel sorted."; + // } + // for (; !fst2_reader.Done(); fst2_reader.Next(), n_done++) { + // VectorFst fst2(fst2_reader.Value()); + // VectorFst fst_out; + // TableCompose(*fst1, fst2, &fst_out, &cache); + // fst_writer.Write(fst2_reader.Key(), fst_out); + // } + // KALDI_LOG << "Composed " << n_done << " FSTs."; + // return (n_done != 0 ? 0 : 1); + //} else if (is_table_1 && is_table_2) { + // SequentialTableReader fst1_reader(fst1_in_str); + // RandomAccessTableReader fst2_reader(fst2_in_str); + // TableWriter fst_writer(fst_out_str); + // int32 n_done = 0, n_err = 0; + // for (; !fst1_reader.Done(); fst1_reader.Next()) { + // std::string key = fst1_reader.Key(); + // if (!fst2_reader.HasKey(key)) { + // KALDI_WARN << "No such key " << key << " in second table."; + // n_err++; + // } else { + // const VectorFst &fst1(fst1_reader.Value()), + // &fst2(fst2_reader.Value(key)); + // VectorFst result; + // TableCompose(fst1, fst2, &result, opts); + // if (result.NumStates() == 0) { + // KALDI_WARN << "Empty output for key " << key; + // n_err++; + // } else { + // fst_writer.Write(key, result); + // n_done++; + // } + // } + // } + // KALDI_LOG << "Successfully composed " << n_done << " FSTs, errors or " + // << "empty output on " << n_err; + //} else { + // KALDI_ERR << "The combination of tables/non-tables and match-type that you " + // << "supplied is not currently supported. Either implement this, " + // << "ask the maintainers to implement it, or call this program " + // << "differently."; + //} + } catch(const std::exception &e) { + std::cerr << e.what(); + return -1; + } +} + diff --git a/runtime/onnxruntime/third_party/kaldi/fstbin/make-grammar-fst.cc b/runtime/onnxruntime/third_party/kaldi/fstbin/make-grammar-fst.cc new file mode 100644 index 000000000..564f25531 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/fstbin/make-grammar-fst.cc @@ -0,0 +1,167 @@ +// fstbin/make-grammar-fst.cc + +// Copyright 2018 Johns Hopkins University (author: Daniel Povey) + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + + +#include "base/kaldi-common.h" +#include "util/common-utils.h" +#include "fst/fstlib.h" +#include "fstext/table-matcher.h" +#include "fstext/kaldi-fst-io.h" +#include "decoder/grammar-fst.h" + +template +void MakeGrammarFst(kaldi::ParseOptions po, + int32 nonterm_phones_offset, + bool write_as_grammar){ + using namespace kaldi; + using namespace fst; + using kaldi::int32; + + std::string fst_out_str = po.GetArg(po.NumArgs()); + std::string top_fst_str = po.GetArg(1); + + ConstFst *top_tmp_fst = ConstFst::Read(top_fst_str); + std::shared_ptr top_fst(new FST(*top_tmp_fst)); + + std::vector > > pairs; + + int32 num_pairs = (po.NumArgs() - 2) / 2; + for (int32 i = 1; i <= num_pairs; i++) { + int32 nonterminal; + std::string nonterm_str = po.GetArg(2*i); + if (!ConvertStringToInteger(nonterm_str, &nonterminal) || + nonterminal <= 0) + KALDI_ERR << "Expected positive integer as nonterminal, got: " + << nonterm_str; + std::string fst_str = po.GetArg(2*i + 1); + + ConstFst *tmp_fst = ConstFst::Read(fst_str); + std::shared_ptr this_fst(new FST(*tmp_fst)); + + pairs.push_back(std::pair >( + nonterminal, this_fst)); + }; + + GrammarFstTpl *grammar_fst = new GrammarFstTpl(nonterm_phones_offset, + top_fst, + pairs); + + if (write_as_grammar) { + bool binary = true; // GrammarFst does not support non-binary write. + WriteKaldiObject(*grammar_fst, fst_out_str, binary); + } else { + VectorFst vfst; + CopyToVectorFst(grammar_fst, &vfst); + ConstFst cfst(vfst); + // We don't have a wrapper in kaldi-fst-io.h for writing type + // ConstFst, so do it manually. + bool binary = true, write_binary_header = false; // suppress the ^@B + Output ko(fst_out_str, binary, write_binary_header); + FstWriteOptions wopts(kaldi::PrintableWxfilename(fst_out_str)); + cfst.Write(ko.Stream(), wopts); + } + + KALDI_LOG << "Created grammar FST and wrote it to " + << fst_out_str; +} + + +int main(int argc, char *argv[]) { + try { + using namespace kaldi; + using namespace fst; + using kaldi::int32; + + const char *usage = + "Construct GrammarFst and write it to disk (or convert it to ConstFst\n" + "and write that to disk instead). Mostly intended for demonstration\n" + "and testing purposes (since it may be more convenient to construct\n" + "GrammarFst from code). See kaldi-asr.org/doc/grammar.html\n" + "Can also be used to prepares FSTs for this use, by calling\n" + "PrepareForGrammarFst(), which does things like adding final-probs and\n" + "making small structural tweaks to the FST\n" + "\n" + "Usage (1): make-grammar-fst [options] \\\n" + " [ ...]] \n" + "\n" + ", are the integer ids of the corresponding\n" + " user-defined nonterminal symbols (e.g. #nonterm:contact_list) in the\n" + " phones.txt file.\n" + "e.g.: make-grammar-fst --nonterm-phones-offset=317 HCLG.fst \\\n" + " 320 HCLG1.fst HCLG_grammar.fst\n" + "\n" + "Usage (2): make-grammar-fst \n" + " Prepare individual FST for compilation into GrammarFst.\n" + " E.g. make-grammar-fst HCLG.fst HCLGmod.fst. The outputs of this\n" + " will then become the arguments , , ... for usage\n" + " pattern (1).\n" + "\n" + "The --nonterm-phones-offset option is required for both usage patterns.\n"; + + + ParseOptions po(usage); + + + int32 nonterm_phones_offset = -1; + bool write_as_grammar = true; + bool make_mutable = false; + + po.Register("nonterm-phones-offset", &nonterm_phones_offset, + "Integer id of #nonterm_bos in phones.txt"); + po.Register("write-as-grammar", &write_as_grammar, "If true, " + "write as GrammarFst object; if false, convert to " + "ConstFst (readable by standard decoders) " + "and write that."); + po.Register("make-mutable", &make_mutable, "If true, " + "Make a GrammarFst using StdVectorFst as the " + "underlying FST instance type." + "Use const ConstFst otherwise."); + + po.Read(argc, argv); + + if (po.NumArgs() < 2 || po.NumArgs() % 2 != 0) { + po.PrintUsage(); + exit(1); + } + + if (nonterm_phones_offset < 0) + KALDI_ERR << "The --nonterm-phones-offset option must be supplied " + "and positive."; + + if (po.NumArgs() == 2) { + // this usage pattern calls PrepareForGrammarFst(). + VectorFst *fst = ReadFstKaldi(po.GetArg(1)); + PrepareForGrammarFst(nonterm_phones_offset, fst); + // This will write it as VectorFst; to avoid it having to be converted to + // ConstFst when read again by make-grammar-fst, you may want to pipe + // through fstconvert --fst_type=const. + WriteFstKaldi(*fst, po.GetArg(2)); + exit(0); + } + + if (make_mutable) { + MakeGrammarFst(po, nonterm_phones_offset, write_as_grammar); + } else { + MakeGrammarFst >(po, nonterm_phones_offset, write_as_grammar); + } + } catch(const std::exception &e) { + std::cerr << e.what(); + return -1; + } +} diff --git a/runtime/onnxruntime/third_party/kaldi/fstext/Makefile b/runtime/onnxruntime/third_party/kaldi/fstext/Makefile new file mode 100644 index 000000000..b76bd413c --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/fstext/Makefile @@ -0,0 +1,30 @@ + +# make "all" the target. +all: + +# Disable linking math libs because not needed here. Just for compilation speed. +# no, it's now needed for context-fst-test. +# MATHLIB = NONE + +EXTRA_CXXFLAGS = -Wno-sign-compare + +include ../kaldi.mk + +TESTFILES = determinize-star-test \ + pre-determinize-test trivial-factor-weight-test \ + context-fst-test factor-test table-matcher-test fstext-utils-test \ + remove-eps-local-test lattice-weight-test \ + determinize-lattice-test lattice-utils-test deterministic-fst-test \ + push-special-test epsilon-property-test prune-special-test + +OBJFILES = push-special.o kaldi-fst-io.o context-fst.o grammar-context-fst.o + + +LIBNAME = kaldi-fstext + +# tree and matrix archives needed for test-context-fst +# matrix archive needed for push-special. +ADDLIBS = ../tree/kaldi-tree.a ../util/kaldi-util.a ../matrix/kaldi-matrix.a \ + ../base/kaldi-base.a + +include ../makefiles/default_rules.mk diff --git a/runtime/onnxruntime/third_party/kaldi/fstext/context-fst-test.cc b/runtime/onnxruntime/third_party/kaldi/fstext/context-fst-test.cc new file mode 100644 index 000000000..65da1bb07 --- /dev/null +++ b/runtime/onnxruntime/third_party/kaldi/fstext/context-fst-test.cc @@ -0,0 +1,256 @@ +// fstext/context-fst-test.cc + +// Copyright 2009-2011 Microsoft Corporation + +// See ../../COPYING for clarification regarding multiple authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED +// WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// See the Apache 2 License for the specific language governing permissions and +// limitations under the License. + +#include "fstext/context-fst.h" +#include "fstext/fst-test-utils.h" +#include "tree/context-dep.h" +#include "util/kaldi-io.h" +#include "base/kaldi-math.h" + +namespace fst +{ +using std::vector; +using std::cout; + +// GenAcceptorFromSequence generates a linear acceptor (identical input+output symbols) that has this +// sequence of symbols, and +template +static VectorFst *GenAcceptorFromSequence(const vector &symbols, float cost) { + typedef typename Arc::Weight Weight; + typedef typename Arc::StateId StateId; + + vector split_cost(symbols.size()+1, 0.0); // for #-arcs + end-state. + { // compute split_cost. it must sum to "cost". + std::set indices; + size_t num_indices = 1 + (kaldi::Rand() % split_cost.size()); + while (indices.size() < num_indices) indices.insert(kaldi::Rand() % split_cost.size()); + for (std::set::iterator iter = indices.begin(); iter != indices.end(); ++iter) { + split_cost[*iter] = cost / num_indices; + } + } + + VectorFst *fst = new VectorFst(); + StateId cur_state = fst->AddState(); + fst->SetStart(cur_state); + for (size_t i = 0; i < symbols.size(); i++) { + StateId next_state = fst->AddState(); + Arc arc; + arc.ilabel = symbols[i]; + arc.olabel = symbols[i]; + arc.nextstate = next_state; + arc.weight = (Weight) split_cost[i]; + fst->AddArc(cur_state, arc); + cur_state = next_state; + + } + fst->SetFinal(cur_state, (Weight)split_cost[symbols.size()]); + return fst; +} + + + +// CheckPhones is used to test the correctness of an FST that is the result of +// composition with a ContextFst. +template +static float CheckPhones(const VectorFst &linear_fst, + const vector &phone_ids, + const vector &disambig_ids, + const vector &phone_seq, + const vector > &ilabel_info, + int N, int P) { + typedef typename Arc::Label Label; + typedef typename Arc::StateId StateId; + typedef typename Arc::Weight Weight; + + assert(kaldi::IsSorted(phone_ids)); // so we can do binary_search. + + + vector input_syms; + vector output_syms; + Weight tot_cost; + bool ans = GetLinearSymbolSequence(linear_fst, &input_syms, + &output_syms, &tot_cost); + assert(ans); // should be linear. + + vector phone_seq_check; + for (size_t i = 0; i < output_syms.size(); i++) + if (std::binary_search(phone_ids.begin(), phone_ids.end(), output_syms[i])) + phone_seq_check.push_back(output_syms[i]); + + assert(phone_seq_check == phone_seq); + + vector > input_syms_long; + for (size_t i = 0; i < input_syms.size(); i++) { + Label isym = input_syms[i]; + if (ilabel_info[isym].size() == 0) continue; // epsilon. + if ( (ilabel_info[isym].size() == 1 && + ilabel_info[isym][0] <= 0) ) continue; // disambig. + input_syms_long.push_back(ilabel_info[isym]); + } + + for (size_t i = 0; i < input_syms_long.size(); i++) { + vector phone_context_window(N); // phone at pos i will be at pos P in this window. + int pos = ((int)i) - P; // pos of first phone in window [ may be out of range] . + for (int j = 0; j < N; j++, pos++) { + if (static_cast(pos) < phone_seq.size()) phone_context_window[j] = phone_seq[pos]; + else phone_context_window[j] = 0; // 0 is a special symbol that context-dep-itf expects to see + // when no phone is present due to out-of-window. context-fst knows about this too. + } + assert(input_syms_long[i] == phone_context_window); + } + return tot_cost.Value(); +} + + + + +template +static VectorFst *GenRandPhoneSeq(vector &phone_syms, + vector &disambig_syms, + typename Arc::Label subsequential_symbol, + int num_subseq_syms, + float seq_prob, + vector *phoneseq_out) { + KALDI_ASSERT(phoneseq_out != NULL); + typedef typename Arc::Label Label; + // Generate an FST that is a random phone sequence, ending + // with "num_subseq_syms" subsequential symbols. It will + // have disambiguation symbols randomly interspersed throughout. + // The number of phones is random (possibly zero). + size_t len = (kaldi::Rand() % 4) * (kaldi::Rand() % 3); // up to 3*2=6 phones. + float disambig_prob = 0.33; + phoneseq_out->clear(); + vector