mirror of
https://github.com/modelscope/FunASR.git
synced 2026-10-12 02:19:18 +00:00
fix: disable timestamps when CTC weights are missing (#3211)
Fail closed for native, remote-code, and vLLM Fun-ASR-Nano timestamp inference when checkpoint CTC weights are missing, partial, shape-mismatched, DDP-prefixed, or wrapped. Fixes #3208.
This commit is contained in:
@@ -64,6 +64,12 @@ pip install -r requirements.txt
|
||||
- [ ] Support speaker diarization
|
||||
- [x] Support model training
|
||||
|
||||
> [!NOTE]
|
||||
> Character-level timestamps require a checkpoint with complete `ctc_decoder.*` and
|
||||
> `ctc.*` weights. The current `Fun-ASR-MLT-Nano-2512` checkpoint does not include
|
||||
> those weights, so FunASR logs a warning and returns text without `timestamps` or
|
||||
> `ctc_timestamps` instead of aligning with uninitialized layers.
|
||||
|
||||
# Usage 🛠️
|
||||
|
||||
## Inference
|
||||
|
||||
@@ -64,6 +64,11 @@ pip install -r requirements.txt
|
||||
- [ ] 支持区分说话人识别
|
||||
- [x] 支持模型训练
|
||||
|
||||
> [!NOTE]
|
||||
> 字符级时间戳要求 checkpoint 同时包含完整的 `ctc_decoder.*` 和 `ctc.*` 权重。
|
||||
> 当前 `Fun-ASR-MLT-Nano-2512` checkpoint 未包含这些权重,因此 FunASR 会记录警告并仅返回文本,
|
||||
> 不再使用未初始化的层生成 `timestamps` 或 `ctc_timestamps`。
|
||||
|
||||
# 用法 🛠️
|
||||
|
||||
## 推理
|
||||
|
||||
@@ -15,6 +15,10 @@ from funasr.register import tables
|
||||
from funasr.train_utils.device_funcs import force_gatherable, to_device
|
||||
from funasr.utils.datadir_writer import DatadirWriter
|
||||
from funasr.utils.load_utils import extract_fbank, load_audio_text_image_video
|
||||
from funasr.models.fun_asr_nano.checkpoint_utils import (
|
||||
disable_incomplete_ctc,
|
||||
normalize_checkpoint_state,
|
||||
)
|
||||
from funasr.models.fun_asr_nano.device_utils import resolve_autocast_device_type
|
||||
|
||||
|
||||
@@ -109,6 +113,7 @@ class FunASRNano(nn.Module):
|
||||
|
||||
# ctc decoder
|
||||
self.ctc_decoder = None
|
||||
self._externally_loaded_ctc_keys = set()
|
||||
# TODO: fix table name
|
||||
ctc_decoder_class = tables.adaptor_classes.get(kwargs.get("ctc_decoder", None))
|
||||
if ctc_decoder_class is not None:
|
||||
@@ -135,8 +140,15 @@ class FunASRNano(nn.Module):
|
||||
self.ctc_decoder = ctc_decoder_class(**ctc_decoder_conf)
|
||||
init_param_path = ctc_decoder_conf.get("init_param_path", None)
|
||||
if init_param_path is not None:
|
||||
src_state = torch.load(init_param_path, map_location="cpu")
|
||||
src_state = normalize_checkpoint_state(
|
||||
torch.load(init_param_path, map_location="cpu")
|
||||
)
|
||||
flag = self.ctc_decoder.load_state_dict(src_state, strict=False)
|
||||
self._externally_loaded_ctc_keys.update(
|
||||
f"ctc_decoder.{key}"
|
||||
for key in self.ctc_decoder.state_dict()
|
||||
if key in src_state
|
||||
)
|
||||
logging.info(f"Loading ctc_decoder ckpt: {init_param_path}, status: {flag}")
|
||||
freeze = ctc_decoder_conf.get("freeze", False)
|
||||
if freeze:
|
||||
@@ -160,6 +172,11 @@ class FunASRNano(nn.Module):
|
||||
rank = int(os.environ.get("RANK", 0))
|
||||
logging.info(f"rank: {rank}, model is builded.")
|
||||
|
||||
def on_pretrained_model_loaded(self, loaded_keys):
|
||||
"""Fail closed when a checkpoint configures CTC without trained weights."""
|
||||
loaded_keys = set(loaded_keys).union(getattr(self, "_externally_loaded_ctc_keys", ()))
|
||||
disable_incomplete_ctc(self, loaded_keys, log=logging)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
speech: torch.Tensor = None,
|
||||
|
||||
Reference in New Issue
Block a user