diff --git a/README.md b/README.md index fe73cb013..1ee25962e 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,7 @@ FunASR has open-sourced a large number of pre-trained models on industrial data. | fsmn-kws
( [⭐](https://modelscope.cn/models/iic/speech_charctc_kws_phone-xiaoyun/summary) ) | keyword spotting，streaming | 5000 hours, Mandarin | 0.7M | | fa-zh
( [⭐](https://modelscope.cn/models/damo/speech_timestamp_prediction-v1-16k-offline/summary) [🤗](https://huggingface.co/funasr/fa-zh) ) | timestamp prediction | 5000 hours, Mandarin | 38M | | cam++
( [⭐](https://modelscope.cn/models/iic/speech_campplus_sv_zh-cn_16k-common/summary) [🤗](https://huggingface.co/funasr/campplus) ) | speaker verification/diarization | 5000 hours | 7.2M | +| ERes2NetV2
([⭐](examples/industrial_data_pretraining/eres2netv2_sv/demo.py) [⭐](https://modelscope.cn/models/iic/speech_eres2netv2_sv_zh-cn_16k-common/summary) ) | speaker verification/diarization | 5000 hours | 17.8M | | Whisper-large-v3
([⭐](https://www.modelscope.cn/models/iic/Whisper-large-v3/summary) [🍀](https://github.com/openai/whisper) ) | speech recognition, with timestamps, non-streaming | multilingual | 1550 M | | Whisper-large-v3-turbo
([⭐](https://www.modelscope.cn/models/iic/Whisper-large-v3-turbo/summary) [🍀](https://github.com/openai/whisper) ) | speech recognition, with timestamps, non-streaming | multilingual | 809 M | | Qwen-Audio
([⭐](examples/industrial_data_pretraining/qwen_audio/demo.py) [🤗](https://huggingface.co/Qwen/Qwen-Audio) ) | audio-text multimodal models (pretraining) | multilingual | 8B | diff --git a/README_zh.md b/README_zh.md index 33dc13f0e..55a57eeef 100644 --- a/README_zh.md +++ b/README_zh.md @@ -123,6 +123,7 @@ FunASR 开源了大量在工业数据上预训练模型，您可以在[模型许 | fsmn-kws
( [⭐](https://modelscope.cn/models/iic/speech_charctc_kws_phone-xiaoyun/summary) ) | 语音唤醒，实时 | 5000 小时，中文 | 0.7M | | fa-zh
( [⭐](https://modelscope.cn/models/damo/speech_timestamp_prediction-v1-16k-offline/summary) [🤗](https://huggingface.co/funasr/fa-zh) ) | 字级别时间戳预测 | 50000 小时，中文 | 38M | | cam++
( [⭐](https://modelscope.cn/models/iic/speech_campplus_sv_zh-cn_16k-common/summary) [🤗](https://huggingface.co/funasr/campplus) ) | 说话人确认/分割 | 5000 小时 | 7.2M | +| ERes2NetV2
([⭐](examples/industrial_data_pretraining/eres2netv2_sv/demo.py) [⭐](https://modelscope.cn/models/iic/speech_eres2netv2_sv_zh-cn_16k-common/summary) ) | 说话人确认/分割 | 5000小时 | 17.8M | | Whisper-large-v3
([⭐](https://www.modelscope.cn/models/iic/Whisper-large-v3/summary) [🍀](https://github.com/openai/whisper) ) | 语音识别，带时间戳输出，非实时 | 多语言 | 1550 M | | Whisper-large-v3-turbo
([⭐](https://www.modelscope.cn/models/iic/Whisper-large-v3-turbo/summary) [🍀](https://github.com/openai/whisper) ) | 语音识别，带时间戳输出，非实时 | 多语言 | 809 M | | Qwen-Audio
([⭐](examples/industrial_data_pretraining/qwen_audio/demo.py) [🤗](https://huggingface.co/Qwen/Qwen-Audio) ) | 音频文本多模态大模型（预训练） | 多语言 | 8B | diff --git a/examples/industrial_data_pretraining/eres2netv2_sv/demo.py b/examples/industrial_data_pretraining/eres2netv2_sv/demo.py new file mode 100644 index 000000000..e44720f32 --- /dev/null +++ b/examples/industrial_data_pretraining/eres2netv2_sv/demo.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# -*- encoding: utf-8 -*- +# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. +# MIT License (https://opensource.org/licenses/MIT) + +from funasr import AutoModel + +# ASR with speaker diarization using ERes2NetV2 +model = AutoModel( + model="iic/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch", + vad_model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch", + vad_kwargs={"max_single_segment_time": 60000}, + punc_model="iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch", + spk_model="iic/speech_eres2netv2_sv_zh-cn_16k-common", + device="cuda:0", +) + +res = model.generate( + input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav", + batch_size_s=300, +) + +for sentence in res: + print(f"[Speaker {sentence.get('spk', '?')}] {sentence['text']}") diff --git a/examples/industrial_data_pretraining/eres2netv2_sv/demo_sv.py b/examples/industrial_data_pretraining/eres2netv2_sv/demo_sv.py new file mode 100644 index 000000000..506327021 --- /dev/null +++ b/examples/industrial_data_pretraining/eres2netv2_sv/demo_sv.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +# -*- encoding: utf-8 -*- +# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. +# MIT License (https://opensource.org/licenses/MIT) + +from funasr import AutoModel + +# Standalone speaker verification / embedding extraction +model = AutoModel( + model="iic/speech_eres2netv2_sv_zh-cn_16k-common", + device="cuda:0", +) + +# Extract speaker embedding +res = model.generate( + input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav", +) + +embedding = res[0]["spk_embedding"] +print(f"Speaker embedding shape: {embedding.shape}") diff --git a/funasr/models/eres2net/eres2netv2.py b/funasr/models/eres2net/eres2netv2.py new file mode 100644 index 000000000..4720ab8c9 --- /dev/null +++ b/funasr/models/eres2net/eres2netv2.py @@ -0,0 +1,226 @@ +# Copyright 3D-Speaker (https://github.com/alibaba-damo-academy/3D-Speaker). All Rights Reserved. +# Licensed under the Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0) + +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + +import funasr.models.sond.pooling.pooling_layers as pooling_layers +from funasr.models.eres2net.fusion import AFF + + +class ReLU(nn.Hardtanh): + + def __init__(self, inplace=False): + super(ReLU, self).__init__(0, 20, inplace) + + def __repr__(self): + inplace_str = "inplace" if self.inplace else "" + return self.__class__.__name__ + " (" + inplace_str + ")" + + +class BasicBlockERes2NetV2(nn.Module): + + def __init__(self, in_planes, planes, stride=1, baseWidth=26, scale=2, expansion=2): + super(BasicBlockERes2NetV2, self).__init__() + width = int(math.floor(planes * (baseWidth / 64.0))) + self.conv1 = nn.Conv2d(in_planes, width * scale, kernel_size=1, stride=stride, bias=False) + self.bn1 = nn.BatchNorm2d(width * scale) + self.nums = scale + self.expansion = expansion + + convs = [] + bns = [] + for i in range(self.nums): + convs.append(nn.Conv2d(width, width, kernel_size=3, padding=1, bias=False)) + bns.append(nn.BatchNorm2d(width)) + self.convs = nn.ModuleList(convs) + self.bns = nn.ModuleList(bns) + self.relu = ReLU(inplace=True) + + self.conv3 = nn.Conv2d(width * scale, planes * self.expansion, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion * planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(self.expansion * planes), + ) + self.stride = stride + self.width = width + self.scale = scale + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + spx = torch.split(out, self.width, 1) + for i in range(self.nums): + if i == 0: + sp = spx[i] + else: + sp = sp + spx[i] + sp = self.convs[i](sp) + sp = self.relu(self.bns[i](sp)) + if i == 0: + out = sp + else: + out = torch.cat((out, sp), 1) + + out = self.conv3(out) + out = self.bn3(out) + + residual = self.shortcut(x) + out += residual + out = self.relu(out) + + return out + + +class BasicBlockERes2NetV2AFF(nn.Module): + + def __init__(self, in_planes, planes, stride=1, baseWidth=26, scale=2, expansion=2): + super(BasicBlockERes2NetV2AFF, self).__init__() + width = int(math.floor(planes * (baseWidth / 64.0))) + self.conv1 = nn.Conv2d(in_planes, width * scale, kernel_size=1, stride=stride, bias=False) + self.bn1 = nn.BatchNorm2d(width * scale) + self.nums = scale + self.expansion = expansion + + convs = [] + fuse_models = [] + bns = [] + for i in range(self.nums): + convs.append(nn.Conv2d(width, width, kernel_size=3, padding=1, bias=False)) + bns.append(nn.BatchNorm2d(width)) + for j in range(self.nums - 1): + fuse_models.append(AFF(channels=width, r=4)) + + self.convs = nn.ModuleList(convs) + self.bns = nn.ModuleList(bns) + self.fuse_models = nn.ModuleList(fuse_models) + self.relu = ReLU(inplace=True) + + self.conv3 = nn.Conv2d(width * scale, planes * self.expansion, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion * planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(self.expansion * planes), + ) + self.stride = stride + self.width = width + self.scale = scale + + def forward(self, x): + residual = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + spx = torch.split(out, self.width, 1) + for i in range(self.nums): + if i == 0: + sp = spx[i] + else: + sp = self.fuse_models[i - 1](sp, spx[i]) + sp = self.convs[i](sp) + sp = self.relu(self.bns[i](sp)) + if i == 0: + out = sp + else: + out = torch.cat((out, sp), 1) + + out = self.conv3(out) + out = self.bn3(out) + + residual = self.shortcut(x) + out += residual + out = self.relu(out) + + return out + + +class ERes2NetV2(nn.Module): + def __init__( + self, + block=BasicBlockERes2NetV2, + block_fuse=BasicBlockERes2NetV2AFF, + num_blocks=[3, 4, 6, 3], + m_channels=64, + feat_dim=80, + embedding_size=192, + baseWidth=26, + scale=2, + expansion=2, + pooling_func="TSTP", + two_emb_layer=False, + ): + super(ERes2NetV2, self).__init__() + self.in_planes = m_channels + self.feat_dim = feat_dim + self.embedding_size = embedding_size + self.stats_dim = int(feat_dim / 8) * m_channels * 8 + self.two_emb_layer = two_emb_layer + self.baseWidth = baseWidth + self.scale = scale + self.expansion = expansion + + self.conv1 = nn.Conv2d(1, m_channels, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(m_channels) + self.layer1 = self._make_layer(block, m_channels, num_blocks[0], stride=1) + self.layer2 = self._make_layer(block, m_channels * 2, num_blocks[1], stride=2) + self.layer3 = self._make_layer(block_fuse, m_channels * 4, num_blocks[2], stride=2) + self.layer4 = self._make_layer(block_fuse, m_channels * 8, num_blocks[3], stride=2) + + self.layer3_ds = nn.Conv2d( + m_channels * 4 * self.expansion, m_channels * 8 * self.expansion, + kernel_size=3, padding=1, stride=2, bias=False, + ) + self.fuse34 = AFF(channels=m_channels * 8 * self.expansion, r=4) + + self.n_stats = 1 if pooling_func == "TAP" or pooling_func == "TSDP" else 2 + self.pool = getattr(pooling_layers, pooling_func)(in_dim=self.stats_dim * self.expansion) + self.seg_1 = nn.Linear(self.stats_dim * self.expansion * self.n_stats, embedding_size) + if self.two_emb_layer: + self.seg_bn_1 = nn.BatchNorm1d(embedding_size, affine=False) + self.seg_2 = nn.Linear(embedding_size, embedding_size) + else: + self.seg_bn_1 = nn.Identity() + self.seg_2 = nn.Identity() + + def _make_layer(self, block, planes, num_blocks, stride): + strides = [stride] + [1] * (num_blocks - 1) + layers = [] + for stride in strides: + layers.append( + block(self.in_planes, planes, stride, baseWidth=self.baseWidth, scale=self.scale, expansion=self.expansion) + ) + self.in_planes = planes * self.expansion + return nn.Sequential(*layers) + + def forward(self, x): + x = x.permute(0, 2, 1) # (B,T,F) => (B,F,T) + x = x.unsqueeze_(1) + out = F.relu(self.bn1(self.conv1(x))) + out1 = self.layer1(out) + out2 = self.layer2(out1) + out3 = self.layer3(out2) + out4 = self.layer4(out3) + out3_ds = self.layer3_ds(out3) + fuse_out34 = self.fuse34(out4, out3_ds) + stats = self.pool(fuse_out34) + + embed_a = self.seg_1(stats) + if self.two_emb_layer: + out = F.relu(embed_a) + out = self.seg_bn_1(out) + embed_b = self.seg_2(out) + return embed_b + else: + return embed_a diff --git a/funasr/models/eres2net/model.py b/funasr/models/eres2net/model.py new file mode 100644 index 000000000..e2549a2b7 --- /dev/null +++ b/funasr/models/eres2net/model.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +# -*- encoding: utf-8 -*- +# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. +# MIT License (https://opensource.org/licenses/MIT) +# Modified from 3D-Speaker (https://github.com/alibaba-damo-academy/3D-Speaker) + +import os +import time +import logging +import torch +import numpy as np + +from funasr.register import tables +from funasr.models.campplus.utils import extract_feature +from funasr.utils.load_utils import load_audio_text_image_video +from funasr.models.eres2net.eres2netv2 import ERes2NetV2 + + +@tables.register("model_classes", "ERes2NetV2") +@tables.register("model_classes", "iic/speech_eres2netv2_sv_zh-cn_16k-common") +class ERes2NetV2SV(torch.nn.Module): + def __init__( + self, + feat_dim=80, + embedding_size=192, + m_channels=64, + baseWidth=26, + scale=2, + expansion=2, + num_blocks=[3, 4, 6, 3], + pooling_func="TSTP", + two_emb_layer=False, + **kwargs, + ): + super().__init__() + self.model = ERes2NetV2( + feat_dim=feat_dim, + embedding_size=embedding_size, + m_channels=m_channels, + baseWidth=baseWidth, + scale=scale, + expansion=expansion, + num_blocks=num_blocks, + pooling_func=pooling_func, + two_emb_layer=two_emb_layer, + ) + self.embedding_size = embedding_size + + model_path = kwargs.get("model_path", None) + init_param = kwargs.get("init_param", None) + if init_param is None and model_path is not None: + ckpt = os.path.join(model_path, "pretrained_eres2netv2.ckpt") + if os.path.exists(ckpt): + init_param = ckpt + if init_param is not None and os.path.exists(init_param): + self._load_pretrained(init_param) + + def _load_pretrained(self, path): + state_dict = torch.load(path, map_location="cpu") + if "state_dict" in state_dict: + state_dict = state_dict["state_dict"] + missing, unexpected = self.model.load_state_dict(state_dict, strict=False) + if missing: + logging.warning(f"ERes2NetV2 missing keys: {missing[:5]}...") + logging.info(f"ERes2NetV2 loaded pretrained weights from {path}") + + def forward(self, x): + return self.model(x) + + def inference( + self, + data_in, + data_lengths=None, + key: list = None, + tokenizer=None, + frontend=None, + **kwargs, + ): + meta_data = {} + time1 = time.perf_counter() + audio_sample_list = load_audio_text_image_video( + data_in, fs=16000, audio_fs=kwargs.get("fs", 16000), data_type="sound" + ) + time2 = time.perf_counter() + meta_data["load_data"] = f"{time2 - time1:0.3f}" + speech, speech_lengths, speech_times = extract_feature(audio_sample_list) + speech = speech.to(device=kwargs["device"]) + time3 = time.perf_counter() + meta_data["extract_feat"] = f"{time3 - time2:0.3f}" + meta_data["batch_data_time"] = np.array(speech_times).sum().item() / 16000.0 + results = [{"spk_embedding": self.forward(speech.to(torch.float32))}] + return results, meta_data diff --git a/tests_models/run_all_tests.py b/tests_models/run_all_tests.py index 9ad5abb05..bf7ee8acc 100644 --- a/tests_models/run_all_tests.py +++ b/tests_models/run_all_tests.py @@ -18,6 +18,7 @@ tests = [ "test_paraformer_streaming.py", "test_qwen3_asr.py", "test_glm_asr.py", + "test_eres2netv2.py", ] SEP = "-" * 60 diff --git a/tests_models/test_eres2netv2.py b/tests_models/test_eres2netv2.py new file mode 100644 index 000000000..6651d5892 --- /dev/null +++ b/tests_models/test_eres2netv2.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Test ERes2NetV2: speaker verification/diarization with ASR""" +import sys +import time + + +def main(): + from funasr import AutoModel + + url_zh = "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav" + + # Test 1: ERes2NetV2 as spk_model combined with VAD + ASR + print("[ERes2NetV2] Loading model with VAD + ASR + SPK...") + t0 = time.time() + model = AutoModel( + model="iic/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch", + vad_model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch", + vad_kwargs={"max_single_segment_time": 60000}, + spk_model="iic/speech_eres2netv2_sv_zh-cn_16k-common", + device="cuda:0", + disable_update=True, + ) + print("[ERes2NetV2] Model loaded in %.1fs" % (time.time() - t0)) + + # Test with speaker diarization + print("[ERes2NetV2] Running inference with speaker diarization...") + t0 = time.time() + res = model.generate( + input=url_zh, + batch_size_s=300, + ) + print("[ERes2NetV2] Inference done in %.1fs" % (time.time() - t0)) + + if res and len(res) > 0 and "text" in res[0]: + print("[ERes2NetV2] Result: %s" % res[0]["text"]) + if "spk" in res[0]: + print("[ERes2NetV2] Speaker: %s" % res[0]["spk"]) + print("[ERes2NetV2] Test 1 PASSED") + else: + print("[ERes2NetV2] Test 1 FAILED - no text in result") + return 1 + + # Test 2: ERes2NetV2 standalone speaker embedding + print("[ERes2NetV2] Test 2: Standalone speaker embedding...") + t0 = time.time() + spk_model = AutoModel( + model="iic/speech_eres2netv2_sv_zh-cn_16k-common", + device="cuda:0", + disable_update=True, + ) + res = spk_model.generate(input=url_zh) + print("[ERes2NetV2] Embedding done in %.1fs" % (time.time() - t0)) + + if res and len(res) > 0 and "spk_embedding" in res[0]: + emb = res[0]["spk_embedding"] + print("[ERes2NetV2] Embedding shape: %s" % str(emb.shape)) + print("[ERes2NetV2] Test 2 PASSED") + else: + print("[ERes2NetV2] Test 2 FAILED - no spk_embedding in result") + return 1 + + print("[ERes2NetV2] All tests PASSED") + return 0 + + +if __name__ == "__main__": + sys.exit(main())