diff --git a/.github/workflows/test-numpy-compatibility.yml b/.github/workflows/test-numpy-compatibility.yml index bdaaf26ec..d23e0c8ff 100644 --- a/.github/workflows/test-numpy-compatibility.yml +++ b/.github/workflows/test-numpy-compatibility.yml @@ -8,10 +8,12 @@ on: - "funasr/frontends/**" - "funasr/models/emotion2vec/**" - "funasr/models/campplus/**" + - "funasr/models/sense_voice/model.py" - "funasr/models/eend/utils/feature.py" - "tests/test_numpy_compatibility.py" - "tests/test_numpy_cluster_compatibility.py" - "tests/test_eend_feature_compatibility.py" + - "tests/test_sensevoice_word_timestamps.py" - "tests/test_pypi_metadata.py" - ".github/workflows/test-numpy-compatibility.yml" push: @@ -22,10 +24,12 @@ on: - "funasr/frontends/**" - "funasr/models/emotion2vec/**" - "funasr/models/campplus/**" + - "funasr/models/sense_voice/model.py" - "funasr/models/eend/utils/feature.py" - "tests/test_numpy_compatibility.py" - "tests/test_numpy_cluster_compatibility.py" - "tests/test_eend_feature_compatibility.py" + - "tests/test_sensevoice_word_timestamps.py" - "tests/test_pypi_metadata.py" - ".github/workflows/test-numpy-compatibility.yml" @@ -71,6 +75,7 @@ jobs: tests/test_numpy_cluster_compatibility.py \ tests/test_eend_feature_compatibility.py \ tests/test_pypi_metadata.py \ + tests/test_sensevoice_word_timestamps.py \ --junitxml=numpy-results.xml - uses: actions/upload-artifact@v4 if: always() diff --git a/funasr/models/sense_voice/model.py b/funasr/models/sense_voice/model.py index c10c663d9..d822939d5 100644 --- a/funasr/models/sense_voice/model.py +++ b/funasr/models/sense_voice/model.py @@ -1093,8 +1093,10 @@ class SenseVoiceSmall(nn.Module): start = int(start * 1000) end = int(end * 1000) if word == "▁": + prev_word = None continue if i == 0: + word = word[1:] if word.startswith("▁") else word # timestamp_new.append([word, start, end]) timestamp_new.append([start, end]) words_new.append(word) diff --git a/tests/test_sensevoice_word_timestamps.py b/tests/test_sensevoice_word_timestamps.py new file mode 100644 index 000000000..c1e539d99 --- /dev/null +++ b/tests/test_sensevoice_word_timestamps.py @@ -0,0 +1,99 @@ +"""Word timestamps use the same SentencePiece boundaries for the first word.""" + +from copy import deepcopy + +import pytest + +from funasr.models.sense_voice.model import SenseVoiceSmall + + +@pytest.mark.parametrize( + ("tokens", "words", "timestamps"), + [ + pytest.param( + ["\u2581Hello", "\u2581world", "."], + ["Hello", "world", "."], + [[0, 125], [125, 250], [250, 375]], + id="first-word-marker", + ), + pytest.param( + ["\u2581Hel", "lo", "\u2581world", "."], + ["Hello", "world", "."], + [[0, 250], [250, 375], [375, 500]], + id="first-word-subpieces", + ), + pytest.param( + ["Hel", "lo", "\u2581wor", "ld", "."], + ["Hello", "world", "."], + [[0, 250], [250, 500], [500, 625]], + id="unmarked-first-word-and-later-boundary", + ), + pytest.param( + ["\u2581", "\u2581Hel", "lo"], + ["Hello"], + [[125, 375]], + id="standalone-leading-marker", + ), + pytest.param( + ["\u2581", "Hel", "lo"], + ["Hello"], + [[125, 375]], + id="standalone-marker-before-unmarked-word", + ), + pytest.param( + ["\u2581\u4f60", "\u597d", "\u3002"], + ["\u4f60", "\u597d", "\u3002"], + [[0, 125], [125, 250], [250, 375]], + id="chinese-first-word", + ), + pytest.param( + ["\u2581Hello", "\u4e16", "\u754c"], + ["Hello", "\u4e16", "\u754c"], + [[0, 125], [125, 250], [250, 375]], + id="mixed-script-boundaries", + ), + pytest.param(["\u2581", "\u2581"], [], [], id="only-markers"), + pytest.param([], [], [], id="empty"), + pytest.param( + ["under\u2581score"], ["under\u2581score"], [[0, 125]], id="nonleading-marker" + ), + pytest.param( + ["\u2581I'm", "\u2581here", "."], + ["I'm", "here", "."], + [[0, 125], [125, 250], [250, 375]], + id="preserve-punctuation", + ), + pytest.param( + ["A", "\u2581B", "C"], ["A", "BC"], [[0, 125], [125, 375]], id="word-boundary" + ), + pytest.param( + ["Hello", "\u2581", "world"], + ["Hello", "world"], + [[0, 125], [250, 375]], + id="standalone-word-boundary", + ), + pytest.param( + ["\u2581Hel", "lo", "\u2581", "wor", "ld"], + ["Hello", "world"], + [[0, 250], [375, 625]], + id="subpieces-around-standalone-boundary", + ), + pytest.param( + ["Hello", "\u2581", "\u2581", "world"], + ["Hello", "world"], + [[0, 125], [375, 500]], + id="repeated-standalone-boundaries", + ), + ], +) +def test_post_preserves_word_boundaries_and_timestamp_alignment(tokens, words, timestamps): + token_timestamps = [[token, index / 8, (index + 1) / 8] for index, token in enumerate(tokens)] + original = deepcopy(token_timestamps) + + # post uses no model state; importing the real class requires no checkpoint. + result_timestamps, result_words = SenseVoiceSmall.post(None, token_timestamps) + + assert result_words == words + assert result_timestamps == timestamps + assert len(result_words) == len(result_timestamps) + assert token_timestamps == original