fix: disable timestamps when CTC weights are missing (#3211)

Fail closed for native, remote-code, and vLLM Fun-ASR-Nano timestamp inference when checkpoint CTC weights are missing, partial, shape-mismatched, DDP-prefixed, or wrapped.

Fixes #3208.
This commit is contained in:
zhifu gao
2026-07-13 23:57:17 +08:00
committed by GitHub
parent 098654e8db
commit aefddc1088
8 changed files with 322 additions and 16 deletions
@@ -64,6 +64,12 @@ pip install -r requirements.txt
- [ ] Support speaker diarization
- [x] Support model training
> [!NOTE]
> Character-level timestamps require a checkpoint with complete `ctc_decoder.*` and
> `ctc.*` weights. The current `Fun-ASR-MLT-Nano-2512` checkpoint does not include
> those weights, so FunASR logs a warning and returns text without `timestamps` or
> `ctc_timestamps` instead of aligning with uninitialized layers.
# Usage 🛠️
## Inference
@@ -64,6 +64,11 @@ pip install -r requirements.txt
- [ ] 支持区分说话人识别
- [x] 支持模型训练
> [!NOTE]
> 字符级时间戳要求 checkpoint 同时包含完整的 `ctc_decoder.*` 和 `ctc.*` 权重。
> 当前 `Fun-ASR-MLT-Nano-2512` checkpoint 未包含这些权重,因此 FunASR 会记录警告并仅返回文本,
> 不再使用未初始化的层生成 `timestamps` 或 `ctc_timestamps`。
# 用法 🛠️
## 推理
@@ -15,6 +15,10 @@ from funasr.register import tables
from funasr.train_utils.device_funcs import force_gatherable, to_device
from funasr.utils.datadir_writer import DatadirWriter
from funasr.utils.load_utils import extract_fbank, load_audio_text_image_video
from funasr.models.fun_asr_nano.checkpoint_utils import (
disable_incomplete_ctc,
normalize_checkpoint_state,
)
from funasr.models.fun_asr_nano.device_utils import resolve_autocast_device_type
@@ -109,6 +113,7 @@ class FunASRNano(nn.Module):
# ctc decoder
self.ctc_decoder = None
self._externally_loaded_ctc_keys = set()
# TODO: fix table name
ctc_decoder_class = tables.adaptor_classes.get(kwargs.get("ctc_decoder", None))
if ctc_decoder_class is not None:
@@ -135,8 +140,15 @@ class FunASRNano(nn.Module):
self.ctc_decoder = ctc_decoder_class(**ctc_decoder_conf)
init_param_path = ctc_decoder_conf.get("init_param_path", None)
if init_param_path is not None:
src_state = torch.load(init_param_path, map_location="cpu")
src_state = normalize_checkpoint_state(
torch.load(init_param_path, map_location="cpu")
)
flag = self.ctc_decoder.load_state_dict(src_state, strict=False)
self._externally_loaded_ctc_keys.update(
f"ctc_decoder.{key}"
for key in self.ctc_decoder.state_dict()
if key in src_state
)
logging.info(f"Loading ctc_decoder ckpt: {init_param_path}, status: {flag}")
freeze = ctc_decoder_conf.get("freeze", False)
if freeze:
@@ -160,6 +172,11 @@ class FunASRNano(nn.Module):
rank = int(os.environ.get("RANK", 0))
logging.info(f"rank: {rank}, model is builded.")
def on_pretrained_model_loaded(self, loaded_keys):
"""Fail closed when a checkpoint configures CTC without trained weights."""
loaded_keys = set(loaded_keys).union(getattr(self, "_externally_loaded_ctc_keys", ()))
disable_incomplete_ctc(self, loaded_keys, log=logging)
def forward(
self,
speech: torch.Tensor = None,