fix(nano): preserve semantic symbol timestamp spans

funasr/models/fun_asr_nano/tools/utils.py

@@ -81,13 +81,30 @@ def forced_align(log_probs: torch.Tensor, targets: torch.Tensor, blank: int = 0)
     return items
 
 
-_PUNCTUATION_TOKEN_RE = re.compile(r"^[^\w\s]+$")
 _SPECIAL_TOKEN_RE = re.compile(r"^<[^>]*>$")
 # tiktoken decodes a partial UTF-8 byte sequence (one id of a multi-id
 # character, e.g. SenseVoice ids [10958, 245] for 郗) as U+FFFD. All three
 # Nano timestamp paths decode each token id independently, so such fragments
 # reach classification as "�". They are spoken content, never punctuation.
 _UNDECODABLE_FRAGMENT_MARK = "�"
+# Supported sentence punctuation: the closed set of terminators and clause
+# separators that ASR emits as structural markup with no acoustic
+# realization — the only tokens the anchoring policy is entitled to collapse.
+# Grounded in the official SenseVoice multilingual.tiktoken vocabulary
+# (FunAudioLLM/Fun-ASR-Nano-2512 @272c57b, sha256
+# 74797963e813193436aabcff7c1c235d37de8097b71c563ec8b63b7a515c718), where
+# every character below is a single-id Po token. Symbol tokens are
+# deliberately excluded: they carry real acoustic extent as spoken semantic
+# content, e.g. "++" (id 24754, Unicode Sm) in 我用C++。你好。 and "$" (id 3,
+# Sc) in 价格是$10。你好。 — anchoring them would delete spoken timing (PR
+# #3703 review). Anything outside this set defaults to spoken, so unknown
+# tokens fail safe rather than lose their span.
+_SENTENCE_PUNCTUATION_CHARS = frozenset("。！？，、；：.,!?;:")
+
+
+def _is_sentence_punctuation(token):
+    """True iff every character of ``token`` is supported sentence punctuation."""
+    return bool(token) and all(char in _SENTENCE_PUNCTUATION_CHARS for char in token)
 
 
 def _classify_timestamp_token(token):
@@ -98,8 +115,11 @@ def _classify_timestamp_token(token):
     serve as anchors. Undecodable byte fragments (containing U+FFFD) are
     spoken: they are slices of a multi-id character with real acoustic
     extent, and collapsing their spans would delete spoken timing (issue
-    #3703). Non-string tokens (integer ids, if called pre-decode) fall back
-    to spoken to preserve pass-through behavior.
+    #3703). Punctuation is only the supported sentence-punctuation set
+    (see ``_SENTENCE_PUNCTUATION_CHARS``); every other non-word token —
+    semantic symbols such as ``++`` or ``$`` — is spoken and keeps its span.
+    Non-string tokens (integer ids, if called pre-decode) fall back to
+    spoken to preserve pass-through behavior.
     """
     if not isinstance(token, str) or not token:
         return "spoken"
@@ -107,7 +127,7 @@ def _classify_timestamp_token(token):
         return "special"
     if _UNDECODABLE_FRAGMENT_MARK in token:
         return "spoken"
-    if _PUNCTUATION_TOKEN_RE.match(token):
+    if _is_sentence_punctuation(token):
         return "punctuation"
     return "spoken"