feat: add Qwen3-ASR speaker diarization; remove unnecessary punc_model from SPK examples

- Qwen3-ASR: support spk_model with forced_aligner for speaker diarization
- Qwen3-ASR: fix list input handling (numpy/tensor to tuple format)
- Qwen3-ASR: output timestamp as [start_ms, end_ms] for pipeline compatibility
- Qwen3-ASR: graceful fallback when forced_aligner not configured
- Remove punc_model from Fun-ASR-Nano/SenseVoice/Qwen3-ASR SPK demos and tests
  (these models output punctuation natively)
- Fix test scripts to handle both text/sentence keys in sentence_info
- auto_model: set return_time_stamps=True when spk_model present
This commit is contained in:
游雁
2026-05-21 04:38:21 +00:00
parent 2ca745e5d1
commit 4967c8bd8e
6 changed files with 19 additions and 9 deletions
@@ -29,7 +29,6 @@ def main():
vad_model="fsmn-vad",
vad_kwargs={"max_single_segment_time": 30000},
spk_model="cam++",
punc_model="ct-punc",
device=device,
hub="ms",
)
@@ -28,7 +28,6 @@ def main():
vad_model="fsmn-vad",
vad_kwargs={"max_single_segment_time": 30000},
spk_model="cam++",
punc_model="ct-punc",
device=device,
)
+1
View File
@@ -491,6 +491,7 @@ class AutoModel:
self._reset_runtime_configs()
if self.spk_model is not None and "output_timestamp" not in cfg:
cfg["output_timestamp"] = True
cfg["return_time_stamps"] = True
kwargs = self.kwargs
# step.1: compute the vad model
deep_update(self.vad_kwargs, cfg)
+16 -3
View File
@@ -90,11 +90,18 @@ class Qwen3ASR(nn.Module):
time1 = time.perf_counter()
language = kwargs.get("language", None)
return_time_stamps = kwargs.get("return_time_stamps", False)
return_time_stamps = kwargs.get("return_time_stamps", False) or kwargs.get("output_timestamp", False)
context = kwargs.get("context", "")
if isinstance(data_in, (list, tuple)):
audio_inputs = list(data_in)
audio_inputs = []
for item in data_in:
if isinstance(item, np.ndarray):
audio_inputs.append((item.astype(np.float32), 16000))
elif isinstance(item, torch.Tensor):
audio_inputs.append((item.cpu().numpy().astype(np.float32), 16000))
else:
audio_inputs.append(item)
elif isinstance(data_in, str):
audio_inputs = [data_in]
elif isinstance(data_in, torch.Tensor):
@@ -114,6 +121,12 @@ class Qwen3ASR(nn.Module):
time2 = time.perf_counter()
meta_data["load_data"] = f"{time2 - time1:0.3f}"
# If timestamps requested but forced_aligner not configured, fall back to no timestamps
if return_time_stamps and self.qwen3_asr_model.forced_aligner is None:
logging.warning("return_time_stamps requires forced_aligner. Skipping timestamps. "
"Initialize with forced_aligner='Qwen/Qwen3-ForcedAligner-0.6B' to enable.")
return_time_stamps = False
results = self.qwen3_asr_model.transcribe(
audio=audio_inputs,
context=context,
@@ -132,7 +145,7 @@ class Qwen3ASR(nn.Module):
result_dict["language"] = r.language
if return_time_stamps and r.time_stamps is not None:
result_dict["timestamp"] = [
[ts.start_time, ts.end_time, ts.text]
[int(ts.start_time), int(ts.end_time)]
for ts in r.time_stamps.items
]
output.append(result_dict)
+1 -2
View File
@@ -15,7 +15,6 @@ def main():
vad_model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch",
vad_kwargs={"max_single_segment_time": 30000},
spk_model="iic/speech_campplus_sv_zh-cn_16k-common",
punc_model="iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",
device="cpu",
disable_update=True,
hub="hf",
@@ -53,7 +52,7 @@ def main():
print("[Fun-ASR-Nano-SPK] sentence_info:")
for s in si:
print(" spk=%d | [%d-%d] %s" % (s["spk"], s["start"], s["end"], s["text"]))
print(" spk=%d | [%d-%d] %s" % (s["spk"], s["start"], s["end"], s.get("text", s.get("sentence", ""))))
has_spk = all("spk" in s for s in si)
if not has_spk:
+1 -2
View File
@@ -14,7 +14,6 @@ def main():
vad_model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch",
vad_kwargs={"max_single_segment_time": 30000},
spk_model="iic/speech_campplus_sv_zh-cn_16k-common",
punc_model="iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",
device="cpu",
disable_update=True,
)
@@ -60,7 +59,7 @@ def main():
print("[SenseVoice-SPK] sentence_info:")
for s in si:
print(" spk=%d | [%d-%d] %s" % (s["spk"], s["start"], s["end"], rich_transcription_postprocess(s["text"])))
print(" spk=%d | [%d-%d] %s" % (s["spk"], s["start"], s["end"], rich_transcription_postprocess(s.get("text", s.get("sentence", "")))))
has_spk = all("spk" in s for s in si)
if not has_spk: