From 8f8931dc9a8421083e9d4c51e509b9e422790f95 Mon Sep 17 00:00:00 2001 From: Mohammad Hijjawi <51020382+MohammadHijjawi97@users.noreply.github.com> Date: Thu, 8 Oct 2026 07:09:17 +0100 Subject: [PATCH] fix(paraformer): skip blank lines and read hotword files as UTF-8 (#3761) SeACo-Paraformer and Contextual Paraformer turn every line of a hotword .txt file into a hotword. A blank line (e.g. an extra newline at the end of the file) became an empty hotword, and its zero length made pack_padded_sequence raise "Length of all samples has to be greater than 0" in the bias encoder, so inference failed. The file was also opened with codecs.open() without an encoding, i.e. the locale encoding. On Windows a UTF-8 Chinese hotword file decoded into mojibake, so every hotword became (or reading raised UnicodeDecodeError on GBK locales). Read the local and downloaded hotword files as UTF-8 and skip blank lines, matching parse_hotword_file() in funasr/utils/postprocess_hotwords.py. --- funasr/models/contextual_paraformer/model.py | 9 ++- funasr/models/seaco_paraformer/model.py | 9 ++- tests/test_paraformer_hotword_file.py | 84 ++++++++++++++++++++ 3 files changed, 96 insertions(+), 6 deletions(-) create mode 100644 tests/test_paraformer_hotword_file.py diff --git a/funasr/models/contextual_paraformer/model.py b/funasr/models/contextual_paraformer/model.py index 4c4df739b..a8f23e926 100644 --- a/funasr/models/contextual_paraformer/model.py +++ b/funasr/models/contextual_paraformer/model.py @@ -7,7 +7,6 @@ import os import re import time import torch -import codecs import logging import tempfile import requests @@ -596,9 +595,11 @@ class ContextualParaformer(Paraformer): logging.info("Attempting to parse hotwords from local txt...") hotword_list = [] hotword_str_list = [] - with codecs.open(hotword_list_or_file, "r") as fin: + with open(hotword_list_or_file, "r", encoding="utf-8") as fin: for line in fin.readlines(): hw = line.strip() + if not hw: + continue hw_list = hw.split() if seg_dict is not None: hw_list = seg_tokenize(hw_list, seg_dict) @@ -623,9 +624,11 @@ class ContextualParaformer(Paraformer): hotword_list_or_file = text_file_path hotword_list = [] hotword_str_list = [] - with codecs.open(hotword_list_or_file, "r") as fin: + with open(hotword_list_or_file, "r", encoding="utf-8") as fin: for line in fin.readlines(): hw = line.strip() + if not hw: + continue hw_list = hw.split() if seg_dict is not None: hw_list = seg_tokenize(hw_list, seg_dict) diff --git a/funasr/models/seaco_paraformer/model.py b/funasr/models/seaco_paraformer/model.py index efa4745a3..6a5787530 100644 --- a/funasr/models/seaco_paraformer/model.py +++ b/funasr/models/seaco_paraformer/model.py @@ -8,7 +8,6 @@ import re import time import copy import torch -import codecs import logging import tempfile import requests @@ -629,9 +628,11 @@ class SeacoParaformer(BiCifParaformer, Paraformer): logging.info("Attempting to parse hotwords from local txt...") hotword_list = [] hotword_str_list = [] - with codecs.open(hotword_list_or_file, "r") as fin: + with open(hotword_list_or_file, "r", encoding="utf-8") as fin: for line in fin.readlines(): hw = line.strip() + if not hw: + continue hw_list = hw.split() if seg_dict is not None: hw_list = seg_tokenize(hw_list, seg_dict) @@ -656,9 +657,11 @@ class SeacoParaformer(BiCifParaformer, Paraformer): hotword_list_or_file = text_file_path hotword_list = [] hotword_str_list = [] - with codecs.open(hotword_list_or_file, "r") as fin: + with open(hotword_list_or_file, "r", encoding="utf-8") as fin: for line in fin.readlines(): hw = line.strip() + if not hw: + continue hw_list = hw.split() if seg_dict is not None: hw_list = seg_tokenize(hw_list, seg_dict) diff --git a/tests/test_paraformer_hotword_file.py b/tests/test_paraformer_hotword_file.py new file mode 100644 index 000000000..c12ff4557 --- /dev/null +++ b/tests/test_paraformer_hotword_file.py @@ -0,0 +1,84 @@ +"""Hotword ``.txt`` parsing for SeACo-Paraformer and Contextual Paraformer. + +``generate_hotwords_list`` turns every line of a hotword file into a list of +token ids. A blank line (for example an extra newline at the end of the file) +used to become an empty hotword, and the zero length then made +``pack_padded_sequence`` raise during decoding. The file was also read with the +platform's locale encoding, so a UTF-8 Chinese hotword file decoded into +```` tokens (or raised) on Windows. +""" + +import builtins +import os +import types + +import pytest +import torch + +from funasr.models.contextual_paraformer.model import ContextualParaformer +from funasr.models.seaco_paraformer.model import SeacoParaformer + +SOS = 1 +VOCAB = {"": 0, "": SOS, "阿": 2, "里": 3, "巴": 4, "云": 5} + + +class _Tokenizer: + def tokens2ids(self, tokens): + return [VOCAB.get(t, VOCAB[""]) for t in tokens] + + +def _generate(model_cls, tmp_path, content): + # Released Paraformer models ship a seg_dict next to am.mvn; it splits + # Chinese words into characters. + (tmp_path / "seg_dict").write_text("阿 阿\n里 里\n巴 巴\n云 云\n", encoding="utf-8") + hotword_file = tmp_path / "hotwords.txt" + hotword_file.write_text(content, encoding="utf-8") + frontend = types.SimpleNamespace(cmvn_file=os.path.join(str(tmp_path), "am.mvn")) + return model_cls.generate_hotwords_list( + types.SimpleNamespace(sos=SOS), + str(hotword_file), + tokenizer=_Tokenizer(), + frontend=frontend, + ) + + +@pytest.fixture(autouse=True) +def _fresh_seg_dict_cache(): + from funasr.models.seaco_paraformer import model as seaco_model + + seaco_model.load_seg_dict.cache_clear() + yield + seaco_model.load_seg_dict.cache_clear() + + +@pytest.mark.parametrize("model_cls", [SeacoParaformer, ContextualParaformer]) +def test_blank_lines_in_hotword_file_are_skipped(model_cls, tmp_path): + hotword_list = _generate(model_cls, tmp_path, "阿里巴巴\n\n \n阿里云\n\n") + + assert hotword_list == [[2, 3, 4, 4], [2, 3, 5], [SOS]] + # Every hotword must be packable by the bias encoder. + lengths = torch.tensor([len(hw) for hw in hotword_list]) + torch.nn.utils.rnn.pack_padded_sequence( + torch.zeros(len(hotword_list), int(lengths.max()), 4), + lengths, + batch_first=True, + enforce_sorted=False, + ) + + +@pytest.mark.parametrize("model_cls", [SeacoParaformer, ContextualParaformer]) +def test_hotword_file_is_read_as_utf8_regardless_of_locale(model_cls, tmp_path, monkeypatch): + real_open = builtins.open + + def cp1252_default_open(file, mode="r", *args, **kwargs): + # Simulate a Windows locale: text mode without an explicit encoding. + if "b" not in mode and kwargs.get("encoding") is None and len(args) < 2: + kwargs["encoding"] = "cp1252" + kwargs.setdefault("errors", "replace") + return real_open(file, mode, *args, **kwargs) + + monkeypatch.setattr(builtins, "open", cp1252_default_open) + + hotword_list = _generate(model_cls, tmp_path, "阿里巴巴\n阿里云\n") + + assert hotword_list == [[2, 3, 4, 4], [2, 3, 5], [SOS]]