From c26848f6480fb0b97178e4ef301c5e07a086505e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= <zhifu.gzf@alibaba-inc.com>
Date: Thu, 21 May 2026 13:05:51 +0000
Subject: [PATCH] fix: repair KWS symbol cleanup regex

---
 funasr/utils/kws_utils.py |  2 +-
 tests/run_test.py         |  5 +++++
 tests/test_kws_utils.py   | 23 +++++++++++++++++++++++
 3 files changed, 29 insertions(+), 1 deletion(-)
 create mode 100644 tests/test_kws_utils.py

diff --git a/funasr/utils/kws_utils.py b/funasr/utils/kws_utils.py
index 1086050e4..11de3238b 100644
--- a/funasr/utils/kws_utils.py
+++ b/funasr/utils/kws_utils.py
@@ -7,7 +7,7 @@ from collections import defaultdict
 from typing import List, Optional, Tuple
 
 
-symbol_str = '[’!"#$%&\'()*+,-./:;<>=?@，。?★、…【】《》？“”‘’！[\\]^_`{|}~\s]+'
+symbol_str = r'''[’!"#$%&'()*+,\-./:;<>=?@，。?★、…【】《》？“”‘’！\[\]\\^_`{|}~\s]+'''
 
 
 def split_mixed_label(input_str):
diff --git a/tests/run_test.py b/tests/run_test.py
index 9ff648bc1..e32ff12b2 100644
--- a/tests/run_test.py
+++ b/tests/run_test.py
@@ -7,6 +7,11 @@ import unittest
 from fnmatch import fnmatch
 
 
+PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if PROJECT_ROOT not in sys.path:
+    sys.path.insert(0, PROJECT_ROOT)
+
+
 def gather_test_cases(test_dir, pattern, list_tests):
     case_list = []
     for dirpath, dirnames, filenames in os.walk(test_dir):
diff --git a/tests/test_kws_utils.py b/tests/test_kws_utils.py
new file mode 100644
index 000000000..79fe4cdc8
--- /dev/null
+++ b/tests/test_kws_utils.py
@@ -0,0 +1,23 @@
+import re
+import unittest
+
+from funasr.utils.kws_utils import query_token_set, split_mixed_label, symbol_str
+
+
+class KwsUtilsTest(unittest.TestCase):
+    def test_symbol_regex_strips_punctuation_and_whitespace(self):
+        self.assertEqual(re.sub(symbol_str, "", "abc!@# 中文\t★"), "abc中文")
+
+    def test_split_mixed_label_lowercases_ascii_words(self):
+        self.assertEqual(split_mixed_label("Hello中文"), ["hello", "中", "文"])
+
+    def test_query_token_set_strips_symbols_before_character_lookup(self):
+        symbol_table = {"a": 1, "中": 2, "<unk>": 0}
+        tokens, token_ids = query_token_set("A!中", symbol_table, {})
+
+        self.assertEqual(tokens, ("a", "中"))
+        self.assertEqual(token_ids, (1, 2))
+
+
+if __name__ == "__main__":
+    unittest.main()