modelscope/FunASR
1
0
Fork 0
mirror of https://github.com/modelscope/FunASR.git synced 2026-10-12 02:19:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3576 from modelscope/codex/fix-3302-realtime-final-regression

Browse Source 
fix(realtime): preserve long segment tails
This commit is contained in:
zhifu gao
2026-08-30 21:39:07 +08:00
committed by GitHub
parent e5a1f4b115 25b1245e9c
commit 8d65a38a8f
2 changed files with 350 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
funasr/bin/realtime_ws.py
+141 -5
View File
@@ -282,6 +282,22 @@ def _normalize_transcript(text):
return _normalize_transcript_with_positions(text)[0]
def _is_supported_transcript_extension(candidate, reference):
candidate, hallucinated = detect_and_fix_hallucination(candidate)
if hallucinated:
return False
candidate_cmp = _normalize_transcript(candidate)
reference_cmp = _normalize_transcript(reference)
if len(candidate_cmp) - len(reference_cmp) < 4:
return False
common_prefix_len = 0
for candidate_char, reference_char in zip(candidate_cmp, reference_cmp):
if candidate_char != reference_char:
break
common_prefix_len += 1
return common_prefix_len >= max(8, (len(reference_cmp) * 3) // 4)
def _merge_overlapping_transcripts(existing, incoming):
"""Join adjacent rolling-window transcripts when their text overlap is clear."""
existing_norm, existing_positions = _normalize_transcript_with_positions(existing)
@@ -714,6 +730,11 @@ class RealtimeASRSession:
self.segment_partial_start_ms = 0
self.segment_partial_end_ms = 0
self.segment_partial_stable_count = 0
self.segment_partial_observation_count = 0
self.segment_best_partial_text = ""
self.segment_best_partial_start_ms = 0
self.segment_best_partial_end_ms = 0
self.segment_best_partial_observation_count = 0
self.last_decode_samples = 0
self.locked_sentences = []
self.prev_seg_text = ""
@@ -771,11 +792,17 @@ class RealtimeASRSession:
self.audio_buffer = np.array([], dtype=np.float32)
self.audio_buffer_start_sample = self.total_samples
def _reset_partial_history(self):
def _reset_partial_history(self, preserve_best=False):
self.segment_partial_text = ""
self.segment_partial_start_ms = 0
self.segment_partial_end_ms = 0
self.segment_partial_stable_count = 0
self.segment_partial_observation_count = 0
if not preserve_best:
self.segment_best_partial_text = ""
self.segment_best_partial_start_ms = 0
self.segment_best_partial_end_ms = 0
self.segment_best_partial_observation_count = 0
def _record_partial_text(self, text, start_ms, hallucinated=False):
"""Build a confidence-bearing transcript across overlapping partial windows."""
@@ -783,20 +810,33 @@ class RealtimeASRSession:
self.last_partial_end_ms = end_ms
self.last_partial_eligible = bool(text.strip()) and not hallucinated
if not text.strip() or hallucinated:
self._reset_partial_history()
self._reset_partial_history(preserve_best=hallucinated)
return
start_ms = int(start_ms)
incoming_norm = _normalize_transcript(text)
best_norm = _normalize_transcript(self.segment_best_partial_text)
if not self.segment_best_partial_text or start_ms != self.segment_best_partial_start_ms:
self.segment_best_partial_text = text
self.segment_best_partial_start_ms = start_ms
self.segment_best_partial_end_ms = end_ms
self.segment_best_partial_observation_count = 1
else:
self.segment_best_partial_observation_count += 1
if len(incoming_norm) > len(best_norm):
self.segment_best_partial_text = text
self.segment_best_partial_end_ms = end_ms
if not self.segment_partial_text:
self.segment_partial_text = text
self.segment_partial_start_ms = start_ms
self.segment_partial_end_ms = end_ms
self.segment_partial_stable_count = 1
self.segment_partial_observation_count = 1
return
history_norm = _normalize_transcript(self.segment_partial_text)
if start_ms == self.segment_partial_start_ms:
self.segment_partial_observation_count += 1
if incoming_norm == history_norm:
self.segment_partial_text = text
self.segment_partial_end_ms = end_ms
@@ -812,6 +852,7 @@ class RealtimeASRSession:
self.segment_partial_start_ms = start_ms
self.segment_partial_end_ms = end_ms
self.segment_partial_stable_count = 1
self.segment_partial_observation_count = 1
return
if start_ms > self.segment_partial_end_ms:
@@ -819,6 +860,7 @@ class RealtimeASRSession:
self.segment_partial_start_ms = start_ms
self.segment_partial_end_ms = end_ms
self.segment_partial_stable_count = 1
self.segment_partial_observation_count = 1
return
merged = _merge_overlapping_transcripts(self.segment_partial_text, text)
@@ -826,11 +868,13 @@ class RealtimeASRSession:
self.segment_partial_text = merged
self.segment_partial_end_ms = end_ms
self.segment_partial_stable_count = 1
self.segment_partial_observation_count += 1
else:
self.segment_partial_text = text
self.segment_partial_start_ms = start_ms
self.segment_partial_end_ms = end_ms
self.segment_partial_stable_count = 1
self.segment_partial_observation_count = 1
def should_decode(self):
threshold = self.first_chunk_samples if not self.first_decode_done else self.chunk_samples
@@ -1028,6 +1072,64 @@ class RealtimeASRSession:
partial_text = self._partial_fallback_candidate(
seg, require_stable=not final_hallucinated
)
long_segment_reason = ""
if not partial_text:
segment_duration_ms = max(0, int(seg[1]) - int(seg[0]))
decode_chunk_ms = int(self.chunk_samples * 1000 / self.sample_rate)
recent_partial = self.last_partial_text.strip()
recent_start_ms = self.last_partial_start_ms
recent_end_ms = self.last_partial_end_ms
recent_observations = self.segment_partial_observation_count
recent_eligible = self.last_partial_eligible
if len(_normalize_transcript(self.segment_best_partial_text)) > len(
_normalize_transcript(recent_partial)
):
recent_partial = self.segment_best_partial_text.strip()
recent_start_ms = self.segment_best_partial_start_ms
recent_end_ms = self.segment_best_partial_end_ms
recent_observations = self.segment_best_partial_observation_count
recent_eligible = True
tail_gap_ms = int(seg[1]) - int(recent_end_ms)
recent_partial, recent_hallucinated = detect_and_fix_hallucination(
recent_partial
)
if (
segment_duration_ms >= 8000
and recent_eligible
and recent_observations >= 2
and int(recent_start_ms) == int(seg[0])
and 0 <= tail_gap_ms <= max(100, decode_chunk_ms * 2)
and recent_partial
and not recent_hallucinated
):
final_cmp = _normalize_transcript(final_text)
recent_cmp = _normalize_transcript(recent_partial)
common_prefix_len = 0
for final_char, partial_char in zip(final_cmp, recent_cmp):
if final_char != partial_char:
break
common_prefix_len += 1
diverged = common_prefix_len < min(len(final_cmp), len(recent_cmp))
catastrophically_short = (
diverged
and len(final_cmp) >= 2
and len(recent_cmp) >= 24
and len(recent_cmp) >= len(final_cmp) * 3
and common_prefix_len >= max(1, len(final_cmp) // 2)
)
tail_regressed = (
diverged
and len(final_cmp) >= 16
and len(recent_cmp) - len(final_cmp) >= 4
and common_prefix_len >= max(12, (len(final_cmp) * 3) // 4)
)
if catastrophically_short or tail_regressed:
partial_text = recent_partial
long_segment_reason = (
"catastrophically short"
if catastrophically_short
else "tail-regressed"
)
if not partial_text:
return final_text
partial_text, partial_hallucinated = detect_and_fix_hallucination(partial_text)
@@ -1044,12 +1146,15 @@ class RealtimeASRSession:
use_partial = (
(final_hallucinated and not partial_hallucinated)
or truncated_prefix
or bool(long_segment_reason)
)
if use_partial and partial_cmp:
reason = "hallucinated" if final_hallucinated else "truncated"
reason = long_segment_reason or (
"hallucinated" if final_hallucinated else "truncated"
)
logger.warning(
"Completed segment [%d-%dms] decode was %s; "
"keeping the latest aligned partial (%d -> %d chars)",
"keeping an aligned observed partial (%d -> %d chars)",
int(seg[0]), int(seg[1]), reason, len(final_text), len(partial_text),
)
return partial_text
@@ -1109,9 +1214,40 @@ class RealtimeASRSession:
except Exception as e:
logger.error(f"Segment decode error: {e}")
decoded_text = text
text = self._reconcile_completed_segment_text(
text, seg, decode_succeeded=decode_succeeded
decoded_text, seg, decode_succeeded=decode_succeeded
)
retry_end_ms = min(int(self.last_partial_end_ms), int(seg[1]))
if (
decode_succeeded
and text != decoded_text
and int(seg[1]) - int(seg[0]) >= 8000
and int(seg[0]) < retry_end_ms < int(seg[1])
):
retry_end_sample = min(
int(retry_end_ms * self.sample_rate / 1000), self.total_samples
)
retry_audio = self._slice_audio(start_sample, retry_end_sample)
if len(retry_audio) >= 1600:
try:
retry_results = self.vllm_engine.generate(
inputs=[torch.from_numpy(retry_audio).float()],
hotwords=self.asr_kwargs.get("hotwords"),
language=self.asr_kwargs.get("language"),
max_new_tokens=512,
)
retry_text = retry_results[0]["text"] if retry_results else ""
retry_text = _clean_asr_text(retry_text)
if _is_supported_transcript_extension(retry_text, text):
logger.warning(
"Completed segment [%d-%dms] accepted an aligned "
"boundary retry extension (%d -> %d chars)",
int(seg[0]), int(seg[1]), len(text), len(retry_text),
)
text = retry_text
except Exception as error:
logger.warning("Segment boundary retry failed: %s", error)
text = _postprocess_result_text(text, self.asr_kwargs)
self.prev_seg_text = text
self._clear_completed_partial(seg)
tests/test_realtime_ws_service.py
+209
View File
@@ -492,6 +492,215 @@ def test_completed_segment_keeps_aligned_partial_when_final_is_truncated_prefix(
assert session.last_partial_text == ""
def make_reported_long_segment_reconciliation_session(module, partial, partial_end_ms):
session = module.RealtimeASRSession(
FixedTextEngine(""),
{},
ControllableVad(),
sample_rate=16000,
chunk_ms=960,
)
session.last_partial_text = partial
session.last_partial_start_ms = 2920
session.last_partial_end_ms = partial_end_ms
session.last_partial_eligible = True
session.segment_partial_observation_count = 2
session.segment_best_partial_text = partial
session.segment_best_partial_start_ms = 2920
session.segment_best_partial_end_ms = partial_end_ms
session.segment_best_partial_observation_count = 2
return session
def test_long_segment_recovers_when_final_regresses_after_shared_prefix():
module = load_service_module()
partial = (
"哇，不好意思，上午高雄交通比较混乱一点，我们不晓得我们有点。"
"哎，OK OK，好嘞，胡总各位长官。"
)
session = make_reported_long_segment_reconciliation_session(
module, partial, partial_end_ms=13500
)
text = session._reconcile_completed_segment_text(
"哇，不好意思，上午高雄交通比较混乱一点，我们不晓得我们有点哎。好，拜",
[2920, 14420],
decode_succeeded=True,
)
assert text == partial
def test_long_segment_recovers_from_catastrophically_short_final():
module = load_service_module()
partial = (
"哇，不好意思，上午高雄交通比较混乱一点，我们不晓得我们有点。"
"哎，O K O K，好嘞，胡总各位长官。"
)
session = make_reported_long_segment_reconciliation_session(
module, partial, partial_end_ms=13500
)
text = session._reconcile_completed_segment_text(
"哇，嘿",
[2920, 14000],
decode_succeeded=True,
)
assert text == partial
def test_long_segment_keeps_distinct_short_final_without_prefix_alignment():
module = load_service_module()
partial = "会议取消，稍后另行通知，请各位等待后续的完整安排和确认消息。"
session = make_reported_long_segment_reconciliation_session(
module, partial, partial_end_ms=13500
)
text = session._reconcile_completed_segment_text(
"项目批准",
[2920, 14000],
decode_succeeded=True,
)
assert text == "项目批准"
def test_long_segment_does_not_trust_a_single_regressed_partial_observation():
module = load_service_module()
partial = (
"哇，不好意思，上午高雄交通比较混乱一点，我们不晓得我们有点。"
"哎，OK OK，好嘞，胡总各位长官。"
)
session = make_reported_long_segment_reconciliation_session(
module, partial, partial_end_ms=13500
)
session.segment_partial_observation_count = 1
final = "哇，不好意思，上午高雄交通比较混乱一点，我们不晓得我们有点哎。好，拜"
text = session._reconcile_completed_segment_text(
final,
[2920, 14420],
decode_succeeded=True,
)
assert text == final
def test_long_segment_can_recover_the_best_earlier_partial():
module = load_service_module()
best_partial = (
"哇，不好意思，上午高雄交通比较混乱一点，我们不晓得我们有点哎，"
"OK OK OK。"
)
final = "哇，不好意思，上午高雄交通比较混乱一点，我们不晓得我们有点哎。好，拜"
session = make_reported_long_segment_reconciliation_session(
module, final, partial_end_ms=13500
)
session.segment_partial_observation_count = 10
session.segment_best_partial_text = best_partial
session.segment_best_partial_start_ms = 2920
session.segment_best_partial_end_ms = 12500
session.segment_best_partial_observation_count = 10
text = session._reconcile_completed_segment_text(
final,
[2920, 14420],
decode_succeeded=True,
)
assert text == best_partial
def test_long_segment_keeps_clean_history_after_a_hallucinated_partial():
module = load_service_module()
best_partial = (
"哇，不好意思，上午高雄交通比较混乱一点，我们不晓得我们有点。"
"哎，OK OK，好嘞，胡总各位长官。"
)
final = "哇，不好意思，上午高雄交通比较混乱一点，我们不晓得我们有点哎。好，拜"
session = make_reported_long_segment_reconciliation_session(
module, best_partial, partial_end_ms=13500
)
session.segment_best_partial_observation_count = 8
session.last_partial_text = final
session.last_partial_start_ms = 2920
session.last_partial_end_ms = 13500
session.last_partial_eligible = False
session._reset_partial_history(preserve_best=True)
text = session._reconcile_completed_segment_text(
final,
[2920, 14420],
decode_succeeded=True,
)
assert text == best_partial
def test_retry_extension_requires_a_longer_aligned_clean_transcript():
module = load_service_module()
fallback = (
"哇，不好意思，上午高雄交通比较混乱一点，我们不晓得我们有点哎，"
"OK OK OK。"
)
extension = (
"哇，不好意思，上午高雄交通比较混乱一点，我们不晓得我们有点。"
"哎，OK OK，好嘞，胡总各位长官。"
)
assert module._is_supported_transcript_extension(extension, fallback) is True
assert module._is_supported_transcript_extension("完全不同的较长结果文本", fallback) is False
assert module._is_supported_transcript_extension("哇，不好意思。", fallback) is False
def test_completed_segment_retries_only_after_a_supported_regression():
module = load_service_module()
fallback = (
"哇，不好意思，上午高雄交通比较混乱一点，我们不晓得我们有点哎，"
"OK OK OK。"
)
extension = (
"哇，不好意思，上午高雄交通比较混乱一点，我们不晓得我们有点。"
"哎，OK OK，好嘞，胡总各位长官。"
)
engine = ScriptedTextEngine(
[
"哇，不好意思，上午高雄交通比较混乱一点，我们不晓得我们有点哎。好，拜",
extension,
]
)
session = make_reported_long_segment_reconciliation_session(
module, fallback, partial_end_ms=13500
)
session.vllm_engine = engine
session.total_samples = int(14.42 * session.sample_rate)
session.audio_buffer = np.zeros(session.total_samples, dtype=np.float32)
text = session._decode_segment([2920, 14420])
assert text == extension
assert len(engine.generate_kwargs) == 2
def test_completed_segment_does_not_retry_a_normal_final():
module = load_service_module()
final = "感谢各位今天参加会议，后续安排将在明天正式通知。"
engine = ScriptedTextEngine([final])
session = make_reported_long_segment_reconciliation_session(
module, "感谢各位今天参加会以。", partial_end_ms=13500
)
session.vllm_engine = engine
session.total_samples = int(14.42 * session.sample_rate)
session.audio_buffer = np.zeros(session.total_samples, dtype=np.float32)
text = session._decode_segment([2920, 14420])
assert text == final
assert len(engine.generate_kwargs) == 1
def test_completed_segment_keeps_clean_aligned_partial_when_final_hallucinates():
module = load_service_module()
partial = "不好意思，上午杭高架交通比较混乱一点，我们不晓得我们有点耽误。"