mirror of
https://github.com/modelscope/FunASR.git
synced 2026-10-11 22:39:57 +00:00
feat: add Qwen3-ASR speaker diarization; remove unnecessary punc_model from SPK examples
- Qwen3-ASR: support spk_model with forced_aligner for speaker diarization - Qwen3-ASR: fix list input handling (numpy/tensor to tuple format) - Qwen3-ASR: output timestamp as [start_ms, end_ms] for pipeline compatibility - Qwen3-ASR: graceful fallback when forced_aligner not configured - Remove punc_model from Fun-ASR-Nano/SenseVoice/Qwen3-ASR SPK demos and tests (these models output punctuation natively) - Fix test scripts to handle both text/sentence keys in sentence_info - auto_model: set return_time_stamps=True when spk_model present
This commit is contained in:
@@ -29,7 +29,6 @@ def main():
|
||||
vad_model="fsmn-vad",
|
||||
vad_kwargs={"max_single_segment_time": 30000},
|
||||
spk_model="cam++",
|
||||
punc_model="ct-punc",
|
||||
device=device,
|
||||
hub="ms",
|
||||
)
|
||||
|
||||
@@ -28,7 +28,6 @@ def main():
|
||||
vad_model="fsmn-vad",
|
||||
vad_kwargs={"max_single_segment_time": 30000},
|
||||
spk_model="cam++",
|
||||
punc_model="ct-punc",
|
||||
device=device,
|
||||
)
|
||||
|
||||
|
||||
@@ -491,6 +491,7 @@ class AutoModel:
|
||||
self._reset_runtime_configs()
|
||||
if self.spk_model is not None and "output_timestamp" not in cfg:
|
||||
cfg["output_timestamp"] = True
|
||||
cfg["return_time_stamps"] = True
|
||||
kwargs = self.kwargs
|
||||
# step.1: compute the vad model
|
||||
deep_update(self.vad_kwargs, cfg)
|
||||
|
||||
@@ -90,11 +90,18 @@ class Qwen3ASR(nn.Module):
|
||||
time1 = time.perf_counter()
|
||||
|
||||
language = kwargs.get("language", None)
|
||||
return_time_stamps = kwargs.get("return_time_stamps", False)
|
||||
return_time_stamps = kwargs.get("return_time_stamps", False) or kwargs.get("output_timestamp", False)
|
||||
context = kwargs.get("context", "")
|
||||
|
||||
if isinstance(data_in, (list, tuple)):
|
||||
audio_inputs = list(data_in)
|
||||
audio_inputs = []
|
||||
for item in data_in:
|
||||
if isinstance(item, np.ndarray):
|
||||
audio_inputs.append((item.astype(np.float32), 16000))
|
||||
elif isinstance(item, torch.Tensor):
|
||||
audio_inputs.append((item.cpu().numpy().astype(np.float32), 16000))
|
||||
else:
|
||||
audio_inputs.append(item)
|
||||
elif isinstance(data_in, str):
|
||||
audio_inputs = [data_in]
|
||||
elif isinstance(data_in, torch.Tensor):
|
||||
@@ -114,6 +121,12 @@ class Qwen3ASR(nn.Module):
|
||||
time2 = time.perf_counter()
|
||||
meta_data["load_data"] = f"{time2 - time1:0.3f}"
|
||||
|
||||
# If timestamps requested but forced_aligner not configured, fall back to no timestamps
|
||||
if return_time_stamps and self.qwen3_asr_model.forced_aligner is None:
|
||||
logging.warning("return_time_stamps requires forced_aligner. Skipping timestamps. "
|
||||
"Initialize with forced_aligner='Qwen/Qwen3-ForcedAligner-0.6B' to enable.")
|
||||
return_time_stamps = False
|
||||
|
||||
results = self.qwen3_asr_model.transcribe(
|
||||
audio=audio_inputs,
|
||||
context=context,
|
||||
@@ -132,7 +145,7 @@ class Qwen3ASR(nn.Module):
|
||||
result_dict["language"] = r.language
|
||||
if return_time_stamps and r.time_stamps is not None:
|
||||
result_dict["timestamp"] = [
|
||||
[ts.start_time, ts.end_time, ts.text]
|
||||
[int(ts.start_time), int(ts.end_time)]
|
||||
for ts in r.time_stamps.items
|
||||
]
|
||||
output.append(result_dict)
|
||||
|
||||
@@ -15,7 +15,6 @@ def main():
|
||||
vad_model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch",
|
||||
vad_kwargs={"max_single_segment_time": 30000},
|
||||
spk_model="iic/speech_campplus_sv_zh-cn_16k-common",
|
||||
punc_model="iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",
|
||||
device="cpu",
|
||||
disable_update=True,
|
||||
hub="hf",
|
||||
@@ -53,7 +52,7 @@ def main():
|
||||
|
||||
print("[Fun-ASR-Nano-SPK] sentence_info:")
|
||||
for s in si:
|
||||
print(" spk=%d | [%d-%d] %s" % (s["spk"], s["start"], s["end"], s["text"]))
|
||||
print(" spk=%d | [%d-%d] %s" % (s["spk"], s["start"], s["end"], s.get("text", s.get("sentence", ""))))
|
||||
|
||||
has_spk = all("spk" in s for s in si)
|
||||
if not has_spk:
|
||||
|
||||
@@ -14,7 +14,6 @@ def main():
|
||||
vad_model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch",
|
||||
vad_kwargs={"max_single_segment_time": 30000},
|
||||
spk_model="iic/speech_campplus_sv_zh-cn_16k-common",
|
||||
punc_model="iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",
|
||||
device="cpu",
|
||||
disable_update=True,
|
||||
)
|
||||
@@ -60,7 +59,7 @@ def main():
|
||||
|
||||
print("[SenseVoice-SPK] sentence_info:")
|
||||
for s in si:
|
||||
print(" spk=%d | [%d-%d] %s" % (s["spk"], s["start"], s["end"], rich_transcription_postprocess(s["text"])))
|
||||
print(" spk=%d | [%d-%d] %s" % (s["spk"], s["start"], s["end"], rich_transcription_postprocess(s.get("text", s.get("sentence", "")))))
|
||||
|
||||
has_spk = all("spk" in s for s in si)
|
||||
if not has_spk:
|
||||
|
||||
Reference in New Issue
Block a user