fix(nano): correct punctuation timestamps across VAD merges (#3703)

* fix(nano): correct punctuation timestamps across VAD merges Anchor zero-acoustic-mass punctuation at the preceding spoken token end (FunASR #3702). Reviewed patch only. * fix(nano): keep byte-fragment spoken timestamps * test(nano): cover byte-fragment timestamp preservation * fix(nano): preserve semantic symbol timestamp spans * test(nano): preserve semantic symbol timestamp spans --------- Signed-off-by: Bruce-Yii <[email protected]> Co-authored-by: Bruce-Yii <[email protected]>
2026-10-12 02:19:18 +00:00 · 2026-09-25 10:16:47 +08:00
parent 3ff9259aad
commit 41778c45de
7 changed files with 798 additions and 4 deletions
@@ -694,7 +694,10 @@ class FunASRNanoVLLM:
    @torch.no_grad()
    def _compute_timestamps(self, encoder_out, encoder_out_lens, text):
        """CTC forced alignment for character-level timestamps."""
-        from funasr.models.fun_asr_nano.tools.utils import forced_align
+        from funasr.models.fun_asr_nano.tools.utils import (
+            anchor_punctuation_timestamps,
+            forced_align,
+        )

        decoder_out, decoder_out_lens = self.ctc_decoder(encoder_out, encoder_out_lens)
        ctc_logits = self.ctc.log_softmax(decoder_out)
@@ -709,6 +712,9 @@ class FunASRNanoVLLM:
            ts["token"] = self.ctc_tokenizer.decode([ts["token"]])
            ts["start_time"] = ts["start_time"] * 6 * 10 / 1000
            ts["end_time"] = ts["end_time"] * 6 * 10 / 1000
+        # See model.py: anchor punctuation at the preceding spoken token's end
+        # (issue #3702); punctuation has no acoustic extent to align.
+        anchor_punctuation_timestamps(timestamps)
        return timestamps

    @classmethod
@@ -322,7 +322,10 @@ class FunASRNanoVLLMPipeline:

    def _compute_all_timestamps(self, segment_audios, vad_segments, asr_results):
        """Compute CTC timestamps for all segments with VAD offsets."""
-        from funasr.models.fun_asr_nano.tools.utils import forced_align
+        from funasr.models.fun_asr_nano.tools.utils import (
+            anchor_punctuation_timestamps,
+            forced_align,
+        )

        all_timestamps = []
        for seg_audio, vad_seg, text in zip(segment_audios, vad_segments, asr_results):
@@ -356,6 +359,13 @@ class FunASRNanoVLLMPipeline:
                    ts["token"] = self.asr_engine.ctc_tokenizer.decode([ts["token"]])
                    ts["start_time"] = ts["start_time"] * 6 * 10 / 1000 + vad_offset_ms / 1000
                    ts["end_time"] = ts["end_time"] * 6 * 10 / 1000 + vad_offset_ms / 1000
+                # Same invariant as the offline/vLLM paths (issue #3702), applied
+                # per independently aligned VAD segment: punctuation carries no
+                # acoustic extent, so pin sandwiched punctuation at the preceding
+                # spoken token's end. Segment-local on purpose — speech in a
+                # later VAD segment must not rewrite this segment's trailing
+                # punctuation.
+                anchor_punctuation_timestamps(timestamps)
                all_timestamps.extend(timestamps)
            except Exception as e:
                logger.debug(f"Timestamp failed for segment: {e}")
@@ -25,7 +25,7 @@ from .ctc import CTC
 from .checkpoint_utils import disable_incomplete_ctc, normalize_checkpoint_state
 from .device_utils import resolve_autocast_device_type
 from .llm_forward_opts import configure_llm_forward
-from .tools.utils import forced_align
+from .tools.utils import anchor_punctuation_timestamps, forced_align

 dtype_map = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}

@@ -1075,6 +1075,12 @@ class FunASRNano(nn.Module):
                    timestamp["token"] = self.ctc_tokenizer.decode([timestamp["token"]])
                    timestamp["start_time"] = timestamp["start_time"] * 6 * 10 / 1000
                    timestamp["end_time"] = timestamp["end_time"] * 6 * 10 / 1000
+            # Punctuation tokens have no acoustic realization; without anchoring,
+            # forced alignment places them at the next sentence's onset inside
+            # merged multi-sentence windows (issue #3702). Applied after decoding
+            # (tokens are still integer ids above). ctc_timestamps is left
+            # untouched: greedy CTC text carries no punctuation to anchor.
+            anchor_punctuation_timestamps(result["timestamps"])

        if ibest_writer is not None:
            ibest_writer["text"][key[0]] = response.replace("\n", " ")
@@ -1,4 +1,5 @@
 from itertools import groupby
+import re

 import librosa
 import soundfile as sf
@@ -77,4 +78,97 @@ def forced_align(log_probs: torch.Tensor, targets: torch.Tensor, blank: int = 0)
            )
    except Exception:
        pass
-    return items
+    return items
+
+
+_SPECIAL_TOKEN_RE = re.compile(r"^<[^>]*>$")
+# tiktoken decodes a partial UTF-8 byte sequence (one id of a multi-id
+# character, e.g. SenseVoice ids [10958, 245] for 郗) as U+FFFD. All three
+# Nano timestamp paths decode each token id independently, so such fragments
+# reach classification as "�". They are spoken content, never punctuation.
+_UNDECODABLE_FRAGMENT_MARK = "�"
+# Supported sentence punctuation: the closed set of terminators and clause
+# separators that ASR emits as structural markup with no acoustic
+# realization — the only tokens the anchoring policy is entitled to collapse.
+# Grounded in the official SenseVoice multilingual.tiktoken vocabulary
+# (FunAudioLLM/Fun-ASR-Nano-2512 @272c57b, sha256
+# 74797963e813193436aabcff7c1c235d37de8097b71c563ec8b63b7a515c718), where
+# every character below is a single-id Po token. Symbol tokens are
+# deliberately excluded: they carry real acoustic extent as spoken semantic
+# content, e.g. "++" (id 24754, Unicode Sm) in 我用C++。你好。 and "$" (id 3,
+# Sc) in 价格是$10。你好。 — anchoring them would delete spoken timing (PR
+# #3703 review). Anything outside this set defaults to spoken, so unknown
+# tokens fail safe rather than lose their span.
+_SENTENCE_PUNCTUATION_CHARS = frozenset("。！？，、；：.,!?;:")
+
+
+def _is_sentence_punctuation(token):
+    """True iff every character of ``token`` is supported sentence punctuation."""
+    return bool(token) and all(char in _SENTENCE_PUNCTUATION_CHARS for char in token)
+
+
+def _classify_timestamp_token(token):
+    """Classify one decoded timestamp token into spoken/punctuation/special.
+
+    Special tokens (``<sil>``-shaped) are their own class: they are neither
+    spoken content nor punctuation, so they neither trigger anchoring nor
+    serve as anchors. Undecodable byte fragments (containing U+FFFD) are
+    spoken: they are slices of a multi-id character with real acoustic
+    extent, and collapsing their spans would delete spoken timing (issue
+    #3703). Punctuation is only the supported sentence-punctuation set
+    (see ``_SENTENCE_PUNCTUATION_CHARS``); every other non-word token —
+    semantic symbols such as ``++`` or ``$`` — is spoken and keeps its span.
+    Non-string tokens (integer ids, if called pre-decode) fall back to
+    spoken to preserve pass-through behavior.
+    """
+    if not isinstance(token, str) or not token:
+        return "spoken"
+    if _SPECIAL_TOKEN_RE.match(token):
+        return "special"
+    if _UNDECODABLE_FRAGMENT_MARK in token:
+        return "spoken"
+    if _is_sentence_punctuation(token):
+        return "punctuation"
+    return "spoken"
+
+
+def anchor_punctuation_timestamps(timestamps):
+    """Anchor punctuation-only tokens at the preceding spoken token's end.
+
+    CTC forced alignment must place every target token — including punctuation
+    tokens with no acoustic realization — on at least one frame. Inside a
+    merged multi-sentence window (see FunASR issue #3702) such a token lands
+    at the next sentence's onset frame, so the current sentence's punctuation
+    timestamp reads as the next sentence start. Punctuation carries no acoustic
+    extent, so a punctuation span that is followed by more speech is pinned
+    (zero-width) at the end of the speech it terminates.
+
+    Leading punctuation (no predecessor) and trailing punctuation (no
+    successor, e.g. single-segment output) are left untouched, as are special
+    tokens such as ``<sil>``. This preserves existing single-segment behavior.
+
+    Args:
+        timestamps: List of ``{"token": str, "start_time": float,
+            "end_time": float}`` dicts in time order. Mutated in place.
+
+    Returns:
+        The same list, for convenient chaining.
+    """
+    last_spoken_end = None
+    total = len(timestamps)
+    for index, ts in enumerate(timestamps):
+        kind = _classify_timestamp_token(ts.get("token", ""))
+        if kind != "punctuation":
+            if kind == "spoken":
+                end = ts.get("end_time")
+                if isinstance(end, (int, float)):
+                    last_spoken_end = end
+            continue
+        following_spoken = any(
+            _classify_timestamp_token(later.get("token", "")) == "spoken"
+            for later in timestamps[index + 1 : total]
+        )
+        if last_spoken_end is not None and following_spoken:
+            ts["start_time"] = last_spoken_end
+            ts["end_time"] = last_spoken_end
+    return timestamps
@@ -0,0 +1,205 @@
+"""Byte-fragment spoken tokens must never be treated as punctuation.
+
+Review regression for FunASR PR #3703 (issue #3702 follow-up): Nano's CTC
+tokenizer is byte-based, so one spoken character can span several token ids
+(official SenseVoiceTokenizer: 郗 -> [10958, 245]). All three Nano timestamp
+paths decode each token id independently, and each fragment decodes to U+FFFD
+while the id sequence decodes to real speech. The punctuation classifier must
+not collapse such fragments' valid acoustic spans.
+
+These tests instantiate the repository's real ``SenseVoiceTokenizer``
+production path (``SenseVoiceTokenizer`` factory -> ``get_tokenizer`` ->
+``get_encoding`` -> production ``Tokenizer``) against a deterministic
+temporary offline ``.tiktoken`` vocab, so they run without the Fun-ASR-Nano
+``multilingual.tiktoken`` model artifact (which ships with the model, not
+this repo) and without weights/GPU. The fixture vocab mirrors the official
+vocab shape: ordinary CJK chars and punctuation are single ids, while 郗
+(deliberately given no merged rank) splits into single-byte ids whose
+independent decodes are U+FFFD — the exact byte-fragment mechanism
+(official: 郗 -> [10958, 245]). Official-artifact cross-check
+(SenseVoiceTokenizer from FunAudioLLM/Fun-ASR-Nano-2512
+``multilingual.tiktoken`` @272c57b, sha256 74797963...): 郗 -> [10958, 245],
+per-id decodes U+FFFD, roundtrip intact; helper/vLLM/pipeline probes preserve
+[0.60, 0.72]/[0.84, 0.96]-style spans post-fix (see PR review evidence).
+"""
+
+import base64
+import types
+
+import numpy as np
+import pytest
+import torch
+
+from funasr.models.fun_asr_nano.tools.utils import (
+    _classify_timestamp_token,
+    anchor_punctuation_timestamps,
+    forced_align,
+)
+from funasr.tokenizer.whisper_tokenizer import SenseVoiceTokenizer
+
+TEXT = "我叫郗明。你好。"
+FRAMES_PER_TOKEN = 10
+FRAME_TO_SEC = 6 * 10 / 1000
+
+
+def _write_fragment_vocab(path):
+    """Deterministic offline vocab: byte-level base plus single-id merged
+    ranks for every fixture char except 郗, which stays byte-split. Format
+    matches production ``.tiktoken`` files (``base64(token) rank`` lines)."""
+    lines = []
+    for byte in range(256):
+        lines.append(f"{base64.b64encode(bytes([byte])).decode()} {byte}")
+    next_rank = 256
+    # Prefix pair first so BPE completes each 3-byte merge (tiktoken merges
+    # lowest-rank pairs first); 郗 gets no ranks and stays byte-split.
+    for char in "我叫明你好。，":
+        raw = char.encode("utf-8")
+        assert len(raw) == 3, char
+        lines.append(f"{base64.b64encode(raw[:2]).decode()} {next_rank}")
+        next_rank += 1
+        lines.append(f"{base64.b64encode(raw).decode()} {next_rank}")
+        next_rank += 1
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+@pytest.fixture()
+def sensevoice_tokenizer(tmp_path):
+    """The real production Tokenizer via the SenseVoiceTokenizer factory."""
+    vocab = tmp_path / "byte-fragment.tiktoken"
+    _write_fragment_vocab(vocab)
+    tokenizer = SenseVoiceTokenizer(vocab_path=str(vocab))
+    target_ids = tokenizer.encode(TEXT)
+    assert tokenizer.decode(target_ids) == TEXT
+    return tokenizer
+
+
+def _peaky_log_probs(target_ids, vocab_size, frames_per_token=FRAMES_PER_TOKEN):
+    n_frames = len(target_ids) * frames_per_token
+    log_probs = torch.full((n_frames, vocab_size), -30.0)
+    for i, tid in enumerate(target_ids):
+        log_probs[i * frames_per_token : (i + 1) * frames_per_token, tid] = 0.0
+    return log_probs
+
+
+def _blank_id(tokenizer):
+    return tokenizer.get_vocab_size() - 1
+
+
+def _production_timestamps(tokenizer, target_ids):
+    """Mirror the production decode-per-id + scale path shared by all three
+    Nano timestamp methods, over the repo's real forced_align."""
+    items = forced_align(
+        _peaky_log_probs(target_ids, tokenizer.get_vocab_size()),
+        torch.tensor(target_ids, dtype=torch.int64),
+        _blank_id(tokenizer),
+    )
+    assert [it["token"] for it in items] == list(target_ids)
+    return [
+        {
+            "token": tokenizer.decode([it["token"]]),
+            "start_time": it["start_time"] * FRAME_TO_SEC,
+            "end_time": it["end_time"] * FRAME_TO_SEC,
+        }
+        for it in items
+    ]
+
+
+def test_undecodable_fragment_is_spoken_not_punctuation():
+    assert _classify_timestamp_token("�") == "spoken"
+    assert _classify_timestamp_token("��") == "spoken"
+    # Genuine punctuation still classifies as punctuation.
+    assert _classify_timestamp_token("。") == "punctuation"
+    assert _classify_timestamp_token(",") == "punctuation"
+    # Special tokens keep their own class.
+    assert _classify_timestamp_token("<sil>") == "special"
+
+
+def test_byte_fragment_spans_survive_anchoring(sensevoice_tokenizer):
+    tokenizer = sensevoice_tokenizer
+    target_ids = tokenizer.encode(TEXT)
+    assert len(target_ids) > len(TEXT), "郗 must stay byte-split"
+    assert tokenizer.decode(target_ids) == TEXT
+    assert any(tokenizer.decode([i]) == "�" for i in target_ids)
+
+    timestamps = _production_timestamps(tokenizer, target_ids)
+    before = [(t["token"], t["start_time"], t["end_time"]) for t in timestamps]
+    anchor_punctuation_timestamps(timestamps)
+
+    for (token, start, end), ts in zip(before, timestamps):
+        if token == "�":
+            assert (ts["start_time"], ts["end_time"]) == (start, end)
+            assert ts["end_time"] > ts["start_time"], "fragment keeps acoustic extent"
+    # Genuine 。 between speech still anchors (control).
+    punct = [t for t in timestamps if t["token"] == "。"]
+    assert punct, "expected genuine punctuation in fixtures"
+    assert punct[0]["start_time"] == punct[0]["end_time"]
+
+
+def test_vllm_method_preserves_byte_fragment_spans(sensevoice_tokenizer):
+    from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM
+
+    tokenizer = sensevoice_tokenizer
+    target_ids = tokenizer.encode(TEXT)
+    n_frames = len(target_ids) * FRAMES_PER_TOKEN
+    log_probs = _peaky_log_probs(target_ids, tokenizer.get_vocab_size())
+
+    engine = FunASRNanoVLLM.__new__(FunASRNanoVLLM)
+    engine.ctc_decoder = lambda e, l: (log_probs.unsqueeze(0), torch.tensor([n_frames]))
+    engine.ctc = types.SimpleNamespace(log_softmax=lambda d: d)
+    engine.ctc_tokenizer = tokenizer
+    engine.blank_id = _blank_id(tokenizer)
+
+    result = engine._compute_timestamps(
+        torch.zeros(1, n_frames, 8), torch.tensor([n_frames]), TEXT
+    )
+    fragments = [t for t in result if t["token"] == "�"]
+    assert len(fragments) >= 2
+    for t in fragments:
+        assert t["end_time"] > t["start_time"]
+    assert any(t["token"] == "。" and t["start_time"] == t["end_time"] for t in result)
+
+
+def test_pipeline_method_preserves_byte_fragment_spans(
+    sensevoice_tokenizer, monkeypatch
+):
+    from funasr.models.fun_asr_nano.inference_vllm_pipeline import (
+        FunASRNanoVLLMPipeline,
+    )
+
+    tokenizer = sensevoice_tokenizer
+    target_ids = tokenizer.encode(TEXT)
+    n_frames = len(target_ids) * FRAMES_PER_TOKEN
+    log_probs = _peaky_log_probs(target_ids, tokenizer.get_vocab_size())
+
+    monkeypatch.setattr(
+        "funasr.utils.load_utils.extract_fbank",
+        lambda *a, **k: (torch.zeros(1, 4, 80), torch.tensor([[4]])),
+    )
+    pipeline = FunASRNanoVLLMPipeline.__new__(FunASRNanoVLLMPipeline)
+    pipeline.device = "cpu"
+    pipeline.asr_engine = types.SimpleNamespace(
+        frontend=object(),
+        audio_encoder=lambda s, sl: (
+            torch.zeros(1, n_frames, 8),
+            torch.tensor([n_frames]),
+        ),
+        ctc_decoder=lambda e, l: (log_probs.unsqueeze(0), torch.tensor([n_frames])),
+        ctc=types.SimpleNamespace(log_softmax=lambda d: d),
+        ctc_tokenizer=tokenizer,
+        blank_id=_blank_id(tokenizer),
+    )
+    result = pipeline._compute_all_timestamps(
+        [np.zeros(32000, dtype=np.float32)], [[1000, 9000]], [TEXT]
+    )
+    fragments = [t for t in result if t["token"] == "�"]
+    assert len(fragments) >= 2
+    for t in fragments:
+        assert t["end_time"] > t["start_time"]
+    # +1s VAD offset applied on top of preserved spans: compare against the
+    # un-offset helper path for the identical target sequence.
+    expected = _production_timestamps(tokenizer, target_ids)
+    expected_frags = [t for t in expected if t["token"] == "�"]
+    assert len(fragments) == len(expected_frags)
+    for got, want in zip(fragments, expected_frags):
+        assert got["start_time"] == pytest.approx(want["start_time"] + 1.0)
+        assert got["end_time"] == pytest.approx(want["end_time"] + 1.0)
@@ -0,0 +1,164 @@
+"""Punctuation-timestamp anchoring for Fun-ASR-Nano forced alignment.
+
+Issue #3702: with merge_vad, a sentence-final punctuation token (which has no
+acoustic realization) is placed by CTC forced alignment at the next sentence's
+onset frame, so the current sentence's punctuation timestamp lands on the next
+sentence start. Proven with synthetic peaky emissions through the repo's exact
+``forced_align`` (see run evidence): in a merged [sent1 + silence + sent2]
+window the punctuation lands at the silence end, while the single-sentence
+control keeps it at the window end.
+
+These tests pin the post-processing rule: a punctuation-only token that is
+followed by more speech is anchored (zero-width) at the preceding spoken
+token's end. Punctuation carries no acoustic extent, so anchoring invents no
+duration. Leading punctuation (no predecessor) and trailing punctuation (no
+successor, e.g. single-segment output) are left untouched, preserving
+existing single-segment behavior.
+"""
+
+import copy
+
+import pytest
+
+from funasr.models.fun_asr_nano.tools.utils import anchor_punctuation_timestamps
+
+
+def _ts(token, start, end):
+    return {"token": token, "start_time": start, "end_time": end}
+
+
+def test_punctuation_anchored_at_previous_spoken_end():
+    # Mirrors the proven repro: 好 ends at 19, 。misplaced at [72, 73).
+    timestamps = [_ts("好", 18, 19), _ts("。", 72, 73), _ts("他", 78, 79)]
+    result = anchor_punctuation_timestamps(timestamps)
+    assert result[1] == {"token": "。", "start_time": 19, "end_time": 19}
+    assert result[0] == {"token": "好", "start_time": 18, "end_time": 19}
+    assert result[2] == {"token": "他", "start_time": 78, "end_time": 79}
+
+
+def test_leading_punctuation_without_predecessor_is_kept():
+    timestamps = [_ts("。", 5, 6), _ts("好", 18, 19)]
+    before = copy.deepcopy(timestamps)
+    assert anchor_punctuation_timestamps(timestamps) == before
+
+
+def test_consecutive_punctuation_share_one_anchor():
+    timestamps = [_ts("好", 18, 19), _ts("，", 50, 51), _ts("。", 72, 73), _ts("他", 78, 79)]
+    result = anchor_punctuation_timestamps(timestamps)
+    assert (result[1]["start_time"], result[1]["end_time"]) == (19, 19)
+    assert (result[2]["start_time"], result[2]["end_time"]) == (19, 19)
+    assert (result[3]["start_time"], result[3]["end_time"]) == (78, 79)
+
+
+def test_spoken_tokens_are_never_rewritten():
+    timestamps = [_ts("你", 12, 13), _ts("好", 18, 19), _ts("。", 39, 40)]
+    before = copy.deepcopy(timestamps)
+    result = anchor_punctuation_timestamps(timestamps)
+    assert result[0] == before[0]
+    # Single-sentence control shape: punctuation already at window end is kept.
+    assert result[2] == before[2]
+
+
+def test_special_tokens_are_not_treated_as_punctuation():
+    timestamps = [_ts("<sil>", 0, 5), _ts("好", 18, 19)]
+    before = copy.deepcopy(timestamps)
+    assert anchor_punctuation_timestamps(timestamps) == before
+
+
+def test_punctuation_followed_only_by_special_is_kept():
+    # No later real speech: the trailing <sil> must not make 。 eligible.
+    timestamps = [_ts("好", 18, 19), _ts("。", 72, 73), _ts("<sil>", 80, 85)]
+    before = copy.deepcopy(timestamps)
+    assert anchor_punctuation_timestamps(timestamps) == before
+
+
+def test_special_token_is_not_used_as_predecessor_anchor():
+    # The <sil> between speech and punctuation must not become the anchor.
+    timestamps = [_ts("好", 18, 19), _ts("<sil>", 30, 35), _ts("。", 72, 73), _ts("他", 78, 79)]
+    result = anchor_punctuation_timestamps(timestamps)
+    assert (result[2]["start_time"], result[2]["end_time"]) == (19, 19)
+    assert (result[1]["start_time"], result[1]["end_time"]) == (30, 35)
+    assert (result[3]["start_time"], result[3]["end_time"]) == (78, 79)
+
+
+def test_pipeline_segment_local_anchor_boundary(monkeypatch):
+    """Pipeline segment boundary: anchoring applies per VAD segment, never
+    across segments. segA shows intra-segment anchoring (misplaced 。 before
+    later speech in the SAME aligned segment anchors); segB shows a
+    segment-final 。 staying put even though segC contains later speech;
+    VAD offsets and spoken spans pass through verbatim."""
+    import types
+
+    import numpy as np
+    import torch
+
+    from funasr.models.fun_asr_nano.inference_vllm_pipeline import FunASRNanoVLLMPipeline
+
+    id2char = {2: "好", 3: "。", 4: "他", 5: "在", 6: "是"}
+    spans = iter(
+        [
+            [
+                {"token": 2, "start_time": 18, "end_time": 19},
+                {"token": 3, "start_time": 72, "end_time": 73},
+                {"token": 4, "start_time": 78, "end_time": 79},
+            ],
+            [
+                {"token": 6, "start_time": 10, "end_time": 12},
+                {"token": 3, "start_time": 40, "end_time": 41},
+            ],
+            [{"token": 5, "start_time": 50, "end_time": 56}],
+        ]
+    )
+
+    class StubTok:
+        def encode(self, text):
+            return [1] * max(1, len(text))
+
+        def decode(self, ids):
+            return "".join(id2char.get(i, "?") for i in ids)
+
+    engine = types.SimpleNamespace(
+        frontend=object(),
+        device="cpu",
+        blank_id=0,
+        audio_encoder=lambda s, sl: (torch.zeros(1, 10, 8), torch.tensor([10])),
+        ctc_decoder=lambda e, l: (torch.zeros(1, 10, 8), torch.tensor([10])),
+        ctc=types.SimpleNamespace(log_softmax=lambda d: d),
+        ctc_tokenizer=StubTok(),
+    )
+    monkeypatch.setattr(
+        "funasr.utils.load_utils.extract_fbank",
+        lambda *a, **k: (torch.zeros(1, 4, 80), torch.tensor([[4]])),
+    )
+    monkeypatch.setattr(
+        "funasr.models.fun_asr_nano.tools.utils.forced_align",
+        lambda *a, **k: next(spans),
+    )
+
+    pipeline = FunASRNanoVLLMPipeline.__new__(FunASRNanoVLLMPipeline)
+    pipeline.asr_engine = engine
+    pipeline.device = "cpu"
+    result = pipeline._compute_all_timestamps(
+        [
+            np.zeros(32000, dtype=np.float32),
+            np.zeros(16000, dtype=np.float32),
+            np.zeros(16000, dtype=np.float32),
+        ],
+        [[1000, 3000], [5000, 6000], [9000, 10000]],
+        ["segAtext", "segBtext", "segCtext"],
+    )
+
+    assert [t["token"] for t in result] == ["好", "。", "他", "是", "。", "在"]
+    # segA (offset 1.0s): intra-segment 。 anchors at 好.end bit-identically.
+    assert result[1]["start_time"] == result[0]["end_time"]
+    assert result[1]["end_time"] == result[0]["end_time"]
+    assert result[0]["end_time"] == pytest.approx(2.14)
+    assert result[2]["start_time"] == pytest.approx(5.68)
+    # segB (offset 5.0s): segment-final 。 keeps its aligned span even though
+    # segC contains later speech — cross-segment speech must not trigger it.
+    assert result[4]["start_time"] == pytest.approx(7.4)
+    assert result[4]["end_time"] == pytest.approx(7.46)
+    assert result[3]["end_time"] == pytest.approx(5.72)
+    # segC (offset 9.0s) passes through untouched.
+    assert result[5]["start_time"] == pytest.approx(12.0)
+    assert result[5]["end_time"] == pytest.approx(12.36)
@@ -0,0 +1,309 @@
+"""Semantic-symbol tokens must keep their acoustic spans (PR #3703 rework).
+
+LauraGPT's blocking review on PR #3703 (comment 3996970511): the accepted
+U+FFFD byte-fragment guard fixed undecodable fragments, but the classifier
+still treated every non-word character as punctuation. With the official
+SenseVoice vocabulary, ``我用C++。你好。`` encodes ``++`` as one token (id
+24754, Unicode Sm) and ``价格是$10。你好。`` encodes ``$`` as one token (id 3,
+Sc). Both decode without replacement characters, are spoken semantic
+content, and had their aligned spans [1.8, 2.4] collapsed to [1.8, 1.8] by
+the actual ``FunASRNanoVLLM._compute_timestamps`` (the pipeline path after
+its +1s VAD offset: [2.8, 3.4] -> [2.8, 2.8]).
+
+The classifier now anchors only the supported sentence-punctuation set
+(``_SENTENCE_PUNCTUATION_CHARS``); everything else defaults to spoken.
+
+These tests instantiate the repository's real ``SenseVoiceTokenizer``
+production path (``SenseVoiceTokenizer`` factory -> ``get_tokenizer`` ->
+``get_encoding`` -> production ``Tokenizer``) against a deterministic
+temporary offline ``.tiktoken`` vocab, so they run without the Fun-ASR-Nano
+``multilingual.tiktoken`` model artifact (which ships with the model, not
+this repo) and without weights/GPU. The fixture vocab mirrors the official
+vocab shape for both reviewer cases: CJK chars and ``。`` are single ids,
+``++`` is one merged id (official: id 24754), ``$`` decodes standalone
+(official: id 3, a single-byte rank), ``C`` stays a single-byte id
+(official: id 34), and ``10`` is one merged id (official: id 3254).
+Official-artifact cross-check (SenseVoiceTokenizer from
+FunAudioLLM/Fun-ASR-Nano-2512 ``multilingual.tiktoken`` @272c57b, sha256
+74797963e813193436aabcff7c1c235d37de8097b71c563ec8b63b7a515c718):
+``我用C++。你好。`` -> [51405, 53719, 34, 24754, 1542, 48934, 50371, 1542]
+and ``价格是$10。你好。`` -> [48869, 52258, 51971, 3, 3254, 1542, 48934,
+50371, 1542]; the symbol ids decode standalone to ``++`` / ``$`` while
+``。`` is id 1542.
+"""
+
+import base64
+import types
+
+import numpy as np
+import pytest
+import torch
+
+from funasr.models.fun_asr_nano.tools.utils import (
+    _classify_timestamp_token,
+    _is_sentence_punctuation,
+    anchor_punctuation_timestamps,
+    forced_align,
+)
+from funasr.tokenizer.whisper_tokenizer import SenseVoiceTokenizer
+
+TEXT_PLUS = "我用C++。你好。"
+TEXT_CURRENCY = "价格是$10。你好。"
+FRAMES_PER_TOKEN = 10
+FRAME_TO_SEC = 6 * 10 / 1000
+
+# Representative official-vocab symbol tokens (FunAudioLLM/Fun-ASR-Nano-2512
+# @272c57b): all are single-id spoken semantic content, never sentence
+# punctuation. Categories are Unicode general categories.
+OFFICIAL_SYMBOL_TOKENS = [
+    ("++", "Sm"),
+    ("+", "Sm"),
+    ("=", "Sm"),
+    ("<", "Sm"),
+    (">", "Sm"),
+    ("^", "Sk"),
+    ("~", "Sm"),
+    ("$", "Sc"),
+    ("€", "Sc"),
+    ("℃", "So"),
+    ("°", "So"),
+    ("%", "Po"),
+    ("#", "Po"),
+    ("@", "Po"),
+    ("&", "Po"),
+    ("*", "Po"),
+    ("/", "Po"),
+    ("-", "Pd"),
+    ("—", "Pd"),
+    ("…", "Po"),
+    ('"', "Pe"),
+    ("(", "Ps"),
+    ("「", "Ps"),
+    ("》", "Pe"),
+]
+
+# Sentence punctuation the anchoring policy supports: terminators and clause
+# separators that ASR emits as structural markup with no acoustic extent.
+SUPPORTED_SENTENCE_PUNCTUATION = [
+    "。",
+    "，",
+    "、",
+    "；",
+    "：",
+    "！",
+    "？",
+    ".",
+    ",",
+    "!",
+    "?",
+    ";",
+    ":",
+]
+
+
+def _write_semantic_symbol_vocab(path):
+    """Deterministic offline vocab: byte-level base plus merged ranks that
+    mirror the official reviewer-case tokenization. Format matches production
+    ``.tiktoken`` files (``base64(token) rank`` lines)."""
+    lines = []
+    for byte in range(256):
+        lines.append(f"{base64.b64encode(bytes([byte])).decode()} {byte}")
+    next_rank = 256
+    # CJK chars and 。/，: prefix pair first so BPE completes each 3-byte
+    # merge (tiktoken merges lowest-rank pairs first).
+    for char in "我用你好价格是。，":
+        raw = char.encode("utf-8")
+        assert len(raw) == 3, char
+        lines.append(f"{base64.b64encode(raw[:2]).decode()} {next_rank}")
+        next_rank += 1
+        lines.append(f"{base64.b64encode(raw).decode()} {next_rank}")
+        next_rank += 1
+    # Multi-char symbol tokens, mirroring official ids 24754 ("++") and 3254
+    # ("10"): single merged ranks, so each decodes standalone as one token.
+    for token in ("++", "10"):
+        lines.append(f"{base64.b64encode(token.encode('utf-8')).decode()} {next_rank}")
+        next_rank += 1
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+@pytest.fixture()
+def sensevoice_tokenizer(tmp_path):
+    """The real production Tokenizer via the SenseVoiceTokenizer factory."""
+    vocab = tmp_path / "semantic-symbol.tiktoken"
+    _write_semantic_symbol_vocab(vocab)
+    tokenizer = SenseVoiceTokenizer(vocab_path=str(vocab))
+    for text in (TEXT_PLUS, TEXT_CURRENCY):
+        assert tokenizer.decode(tokenizer.encode(text)) == text
+    return tokenizer
+
+
+def _peaky_log_probs(target_ids, vocab_size, frames_per_token=FRAMES_PER_TOKEN):
+    n_frames = len(target_ids) * frames_per_token
+    log_probs = torch.full((n_frames, vocab_size), -30.0)
+    for i, tid in enumerate(target_ids):
+        log_probs[i * frames_per_token : (i + 1) * frames_per_token, tid] = 0.0
+    return log_probs
+
+
+def _blank_id(tokenizer):
+    return tokenizer.get_vocab_size() - 1
+
+
+def _production_timestamps(tokenizer, target_ids):
+    """Mirror the production decode-per-id + scale path shared by all three
+    Nano timestamp methods, over the repo's real forced_align."""
+    items = forced_align(
+        _peaky_log_probs(target_ids, tokenizer.get_vocab_size()),
+        torch.tensor(target_ids, dtype=torch.int64),
+        _blank_id(tokenizer),
+    )
+    assert [it["token"] for it in items] == list(target_ids)
+    return [
+        {
+            "token": tokenizer.decode([it["token"]]),
+            "start_time": it["start_time"] * FRAME_TO_SEC,
+            "end_time": it["end_time"] * FRAME_TO_SEC,
+        }
+        for it in items
+    ]
+
+
+def test_supported_sentence_punctuation_anchors():
+    for token in SUPPORTED_SENTENCE_PUNCTUATION:
+        assert _is_sentence_punctuation(token), token
+        assert _classify_timestamp_token(token) == "punctuation", token
+    # Combinations of supported punctuation stay punctuation.
+    assert _is_sentence_punctuation("!?")
+    assert _is_sentence_punctuation("!!")
+    assert _classify_timestamp_token("!?") == "punctuation"
+
+
+def test_semantic_symbols_are_spoken_not_punctuation():
+    for token, _category in OFFICIAL_SYMBOL_TOKENS:
+        assert _classify_timestamp_token(token) == "spoken", token
+        assert not _is_sentence_punctuation(token), token
+    # A symbol glued to sentence punctuation must not anchor either.
+    assert _classify_timestamp_token('。"') == "spoken"
+    assert _classify_timestamp_token("%,") == "spoken"
+    # Undecodable byte fragments stay spoken; special tokens keep their class.
+    assert _classify_timestamp_token("�") == "spoken"
+    assert _classify_timestamp_token("<sil>") == "special"
+
+
+def test_symbol_token_ids_decode_standalone(sensevoice_tokenizer):
+    tokenizer = sensevoice_tokenizer
+    plus_ids = tokenizer.encode(TEXT_PLUS)
+    currency_ids = tokenizer.encode(TEXT_CURRENCY)
+    assert tokenizer.decode(plus_ids) == TEXT_PLUS
+    assert tokenizer.decode(currency_ids) == TEXT_CURRENCY
+    # ++ and $ must be single ids that decode standalone (official: one id
+    # each, 24754 and 3), not per-character or byte-split.
+    plus_tokens = [tokenizer.decode([i]) for i in plus_ids]
+    currency_tokens = [tokenizer.decode([i]) for i in currency_ids]
+    assert "++" in plus_tokens, plus_tokens
+    assert "$" in currency_tokens, currency_tokens
+    assert _classify_timestamp_token("++") == "spoken"
+    assert _classify_timestamp_token("$") == "spoken"
+
+
+def test_semantic_symbol_spans_survive_anchoring(sensevoice_tokenizer):
+    tokenizer = sensevoice_tokenizer
+    for text, symbol in ((TEXT_PLUS, "++"), (TEXT_CURRENCY, "$")):
+        target_ids = tokenizer.encode(text)
+        timestamps = _production_timestamps(tokenizer, target_ids)
+        before = [(t["token"], t["start_time"], t["end_time"]) for t in timestamps]
+        anchor_punctuation_timestamps(timestamps)
+
+        symbols = [t for t in timestamps if t["token"] == symbol]
+        assert symbols, f"expected {symbol!r} in {text!r}"
+        for (token, start, end), ts in zip(before, timestamps):
+            if token == symbol:
+                # The acoustic span is preserved verbatim: no collapse.
+                assert (ts["start_time"], ts["end_time"]) == (start, end)
+                assert ts["end_time"] > ts["start_time"], f"{symbol} keeps acoustic extent"
+        # Genuine 。 between speech still anchors (zero-width control): it
+        # pins at the preceding spoken token's end, which for these texts is
+        # the symbol or the token right after it.
+        punct = [t for t in timestamps if t["token"] == "。"]
+        assert punct
+        assert punct[0]["start_time"] == punct[0]["end_time"]
+        punct_index = next(i for i, t in enumerate(before) if t[0] == "。")
+        assert before[punct_index - 1][2] == punct[0]["start_time"]
+
+
+def test_vllm_method_preserves_semantic_symbol_spans(sensevoice_tokenizer):
+    from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM
+
+    tokenizer = sensevoice_tokenizer
+    for text, symbol in ((TEXT_PLUS, "++"), (TEXT_CURRENCY, "$")):
+        target_ids = tokenizer.encode(text)
+        n_frames = len(target_ids) * FRAMES_PER_TOKEN
+        log_probs = _peaky_log_probs(target_ids, tokenizer.get_vocab_size())
+
+        engine = FunASRNanoVLLM.__new__(FunASRNanoVLLM)
+        engine.ctc_decoder = lambda e, l, lp=log_probs, nf=n_frames: (
+            lp.unsqueeze(0),
+            torch.tensor([nf]),
+        )
+        engine.ctc = types.SimpleNamespace(log_softmax=lambda d: d)
+        engine.ctc_tokenizer = tokenizer
+        engine.blank_id = _blank_id(tokenizer)
+
+        result = engine._compute_timestamps(
+            torch.zeros(1, n_frames, 8), torch.tensor([n_frames]), text
+        )
+        symbols = [t for t in result if t["token"] == symbol]
+        assert symbols
+        for t in symbols:
+            assert t["end_time"] > t["start_time"], f"{symbol} span preserved"
+        # Genuine punctuation still anchors between speech.
+        assert any(t["token"] == "。" and t["start_time"] == t["end_time"] for t in result)
+
+
+def test_pipeline_method_preserves_semantic_symbol_spans(sensevoice_tokenizer, monkeypatch):
+    from funasr.models.fun_asr_nano.inference_vllm_pipeline import (
+        FunASRNanoVLLMPipeline,
+    )
+
+    tokenizer = sensevoice_tokenizer
+    for text, symbol in ((TEXT_PLUS, "++"), (TEXT_CURRENCY, "$")):
+        target_ids = tokenizer.encode(text)
+        n_frames = len(target_ids) * FRAMES_PER_TOKEN
+        log_probs = _peaky_log_probs(target_ids, tokenizer.get_vocab_size())
+
+        monkeypatch.setattr(
+            "funasr.utils.load_utils.extract_fbank",
+            lambda *a, **k: (torch.zeros(1, 4, 80), torch.tensor([[4]])),
+        )
+        pipeline = FunASRNanoVLLMPipeline.__new__(FunASRNanoVLLMPipeline)
+        pipeline.device = "cpu"
+        pipeline.asr_engine = types.SimpleNamespace(
+            frontend=object(),
+            audio_encoder=lambda s, sl, nf=n_frames: (
+                torch.zeros(1, nf, 8),
+                torch.tensor([nf]),
+            ),
+            ctc_decoder=lambda e, l, lp=log_probs, nf=n_frames: (
+                lp.unsqueeze(0),
+                torch.tensor([nf]),
+            ),
+            ctc=types.SimpleNamespace(log_softmax=lambda d: d),
+            ctc_tokenizer=tokenizer,
+            blank_id=_blank_id(tokenizer),
+        )
+        result = pipeline._compute_all_timestamps(
+            [np.zeros(32000, dtype=np.float32)], [[1000, 9000]], [text]
+        )
+        symbols = [t for t in result if t["token"] == symbol]
+        assert symbols
+        # +1s VAD offset applied on top of preserved spans: compare against
+        # the un-offset helper path for the identical target sequence.
+        expected = _production_timestamps(tokenizer, target_ids)
+        expected_symbols = [t for t in expected if t["token"] == symbol]
+        assert len(symbols) == len(expected_symbols)
+        for got, want in zip(symbols, expected_symbols):
+            assert got["start_time"] == pytest.approx(want["start_time"] + 1.0)
+            assert got["end_time"] == pytest.approx(want["end_time"] + 1.0)
+            assert got["end_time"] > got["start_time"], f"{symbol} span preserved"
+        # Genuine punctuation still anchors between speech.
+        assert any(t["token"] == "。" and t["start_time"] == t["end_time"] for t in result)