From 10ac8cf39e7dae4aa8f3bc95307d2bdfc364cc88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Tue, 19 May 2026 14:51:12 +0000 Subject: [PATCH] feat: support SenseVoice speaker diarization (#2706) - Auto-enable output_timestamp when spk_model is present in inference_with_vad - Add SenseVoice + SPK demo and update README news/quick-start --- README.md | 23 ++++++- README_zh.md | 2 +- .../sense_voice/demo_spk.py | 61 +++++++++++++++++++ funasr/auto/auto_model.py | 2 + 4 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 examples/industrial_data_pretraining/sense_voice/demo_spk.py diff --git a/README.md b/README.md index 4330e05b7..91dc7eb1c 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ ## What's new: -- 2026/05/19: Fun-ASR-Nano now supports speaker diarization. Use with `vad_model` + `spk_model` + `punc_model` to get per-sentence speaker labels. See [demo](examples/industrial_data_pretraining/fun_asr_nano/demo_spk.py). +- 2026/05/19: Fun-ASR-Nano and SenseVoice now support speaker diarization. Use with `vad_model` + `spk_model` + `punc_model` to get per-sentence speaker labels. See [Fun-ASR-Nano demo](examples/industrial_data_pretraining/fun_asr_nano/demo_spk.py), [SenseVoice demo](examples/industrial_data_pretraining/sense_voice/demo_spk.py). - 2025/12/15: [Fun-ASR-Nano-2512](https://github.com/FunAudioLLM/Fun-ASR) is an end-to-end speech recognition large model trained on tens of millions of hours real speech data. It supports low-latency real-time transcription and covers 31 languages. - 2024/10/29: Real-time Transcription Service 1.12 released, The 2pass-offline mode supports the SensevoiceSmal model；([docs](runtime/readme.md)); - 2024/10/10：Added support for the Whisper-large-v3-turbo model, a multitasking model that can perform multilingual speech recognition, speech translation, and language identification. It can be downloaded from the [modelscope](examples/industrial_data_pretraining/whisper/demo.py), and [openai](examples/industrial_data_pretraining/whisper/demo_from_openai.py). @@ -231,6 +231,27 @@ Parameter Description: - `merge_vad`: Whether to merge short audio fragments segmented by the VAD model, with the merged length being `merge_length_s`, in seconds (s). - `ban_emo_unk`: Whether to ban the output of the `emo_unk` token. +#### SenseVoice with Speaker Diarization + +```python +from funasr import AutoModel +from funasr.utils.postprocess_utils import rich_transcription_postprocess + +model = AutoModel( + model="iic/SenseVoiceSmall", + vad_model="fsmn-vad", + vad_kwargs={"max_single_segment_time": 30000}, + spk_model="cam++", + punc_model="ct-punc", + device="cuda:0", +) +res = model.generate(input="example.wav", cache={}, language="auto", use_itn=True, + batch_size_s=60, merge_vad=True, merge_length_s=15) +# Result contains sentence_info with speaker labels +print(res[0]["sentence_info"]) +# [{'text': '...', 'start': 750, 'end': 5550, 'timestamp': [...], 'spk': 0}] +``` + #### Paraformer ```python diff --git a/README_zh.md b/README_zh.md index cc4d662ad..162f9b5c1 100644 --- a/README_zh.md +++ b/README_zh.md @@ -34,7 +34,7 @@ FunASR 希望在语音识别的学术研究和工业应用之间架起一座桥 ## 最新动态 -- 2026/05/19: Fun-ASR-Nano 现已支持说话人分离。配合 `vad_model` + `spk_model` + `punc_model` 使用，可获得带说话人标签的逐句结果。参见 [demo](examples/industrial_data_pretraining/fun_asr_nano/demo_spk.py)。 +- 2026/05/19: Fun-ASR-Nano 和 SenseVoice 现已支持说话人分离。配合 `vad_model` + `spk_model` + `punc_model` 使用，可获得带说话人标签的逐句结果。参见 [Fun-ASR-Nano demo](examples/industrial_data_pretraining/fun_asr_nano/demo_spk.py)、[SenseVoice demo](examples/industrial_data_pretraining/sense_voice/demo_spk.py)。 - 2025/12/15: [Fun-ASR-Nano-2512](https://github.com/FunAudioLLM/Fun-ASR) 是一款基于数千万小时真实语音数据训练的端到端语音识别大模型。它支持低延迟实时转写，并涵盖 31 种语言识别功能。 - 2024/10/29: 中文实时语音听写服务 1.12 发布，2pass-offline 模式支持 SensevoiceSmall 模型；详细信息参阅([部署文档](runtime/readme_cn.md)) - 2024/10/10：新增加 Whisper-large-v3-turbo 模型支持，多语言语音识别/翻译/语种识别，支持从 [modelscope](examples/industrial_data_pretraining/whisper/demo.py)仓库下载，也支持从 [openai](examples/industrial_data_pretraining/whisper/demo_from_openai.py)仓库下载模型。 diff --git a/examples/industrial_data_pretraining/sense_voice/demo_spk.py b/examples/industrial_data_pretraining/sense_voice/demo_spk.py new file mode 100644 index 000000000..e38a5c93f --- /dev/null +++ b/examples/industrial_data_pretraining/sense_voice/demo_spk.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +# -*- encoding: utf-8 -*- +# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. +# MIT License (https://opensource.org/licenses/MIT) + +""" +SenseVoice with Speaker Diarization + +This demo shows how to use SenseVoice with VAD + Speaker Model + Punctuation Model +to get per-sentence speaker labels. +""" + +import torch +from funasr import AutoModel +from funasr.utils.postprocess_utils import rich_transcription_postprocess + + +def main(): + model_dir = "iic/SenseVoiceSmall" + device = ( + "cuda:0" + if torch.cuda.is_available() + else "mps" if torch.backends.mps.is_available() else "cpu" + ) + + model = AutoModel( + model=model_dir, + vad_model="fsmn-vad", + vad_kwargs={"max_single_segment_time": 30000}, + spk_model="cam++", + punc_model="ct-punc", + device=device, + ) + + wav_path = f"{model.model_path}/example/zh.mp3" + res = model.generate( + input=wav_path, + cache={}, + language="auto", + use_itn=True, + batch_size_s=60, + merge_vad=True, + merge_length_s=15, + ) + + # Print full text + text = rich_transcription_postprocess(res[0]["text"]) + print("Text:", text) + print() + + # Print per-sentence results with speaker labels + print("Speaker Diarization Results:") + for sent in res[0]["sentence_info"]: + sent_text = rich_transcription_postprocess(sent["text"]) + print( + f" Speaker {sent['spk']}: [{sent['start']}ms - {sent['end']}ms] {sent_text}" + ) + + +if __name__ == "__main__": + main() diff --git a/funasr/auto/auto_model.py b/funasr/auto/auto_model.py index ddf19cc8c..dc7550c30 100644 --- a/funasr/auto/auto_model.py +++ b/funasr/auto/auto_model.py @@ -442,6 +442,8 @@ class AutoModel: def inference_with_vad(self, input, input_len=None, **cfg): self._reset_runtime_configs() + if self.spk_model is not None and "output_timestamp" not in cfg: + cfg["output_timestamp"] = True kwargs = self.kwargs # step.1: compute the vad model deep_update(self.vad_kwargs, cfg)