diff --git a/.github/workflows/product-site.yml b/.github/workflows/product-site.yml index 5ee0cb5fa..8256de757 100644 --- a/.github/workflows/product-site.yml +++ b/.github/workflows/product-site.yml @@ -89,6 +89,7 @@ jobs: tests/test_vllm_examples_contract.py tests/test_training_docs_contract.py tests/test_python_api_docs_contract.py + tests/test_streaming_vad_docs.py tests/test_service_docs_contract.py tests/test_generated_api_docs.py tests/test_api_signatures.py diff --git a/docs/README.md b/docs/README.md index 5742ce54a..7b0d9ec93 100644 --- a/docs/README.md +++ b/docs/README.md @@ -28,6 +28,7 @@ | Your task | Guide | | --- | --- | | Compare CPU, GPU, realtime and API runtimes | [Deployment matrix](deployment_matrix.md) / [中文](deployment_matrix_zh.md) | +| Handle streaming speech boundaries and finalization | [Streaming VAD](streaming_vad.md) / [中文](streaming_vad_zh.md) | | Transcribe and diarize with third-party MOSS | [MOSS](moss_transcribe_diarize.md) / [中文](moss_transcribe_diarize_zh.md) | | Accelerate with the FunASR vLLM split engine | [vLLM](vllm_guide.md) / [中文](vllm_guide_zh.md) | | Evaluate native vLLM serving | [Validation record](vllm_native_funasr_validation.md) | diff --git a/docs/index.rst b/docs/index.rst index 944875224..e22567dfa 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -40,6 +40,8 @@ entry to these guides. Source Markdown remains in this repository. :maxdepth: 1 :caption: Models and Deployment + streaming_vad + streaming_vad_zh deployment_matrix deployment_matrix_zh ./moss_transcribe_diarize.md diff --git a/docs/streaming_vad.md b/docs/streaming_vad.md new file mode 100644 index 000000000..ab5d84823 --- /dev/null +++ b/docs/streaming_vad.md @@ -0,0 +1,162 @@ +[简体中文](streaming_vad_zh.md) | English + +# Streaming Voice Activity Detection + +Use FSMN VAD to receive speech-start and speech-end events while feeding one +recording in order. VAD does not transcribe speech, identify speakers, or locate +linguistic sentence boundaries. For a first offline VAD call, use the +[SDK tutorial](tutorial/README.md); for streaming ASR, use the separate +[Paraformer cache example](python_api.md#streaming-cache-lifecycle). + +## Prepare the Model and Audio + +Complete the [installation checks](installation/installation.md). Prepare a +complete local snapshot of `iic/speech_fsmn_vad_zh-cn-16k-common-pytorch`, +including configuration, frontend files and weights. Record the resolved model +revision and review its model license. This recipe expects that 16 kHz checkpoint, +not an arbitrary VAD model or a streaming ASR checkpoint. + +Use a nonempty mono 16 kHz WAV. The program below takes the local model directory +and audio path as its two positional command-line arguments. It reads the file +once and simulates ordered streaming chunks; it is not microphone capture or a +WebSocket service. The entire file remains in memory. + +## Interpret Events + +Each returned dictionary has a `value` list. Every pair uses **milliseconds +relative to the beginning of this stream**, not the current input chunk. +Do not add the chunk offset a second time. + +| Event | Meaning | +| --- | --- | +| `[]` | No new boundary; an earlier speech span may still be open. | +| `[[start, -1]]` | A speech span started; retain the start for a later call. | +| `[[-1, end]]` | The previously started span ended. | +| `[[start, end]]` | Both boundaries are available in this call. | + +One call can return several pairs. `-1` is a missing-boundary sentinel, never a +timestamp to pass to a waveform slice. Keep the pending start per stream. +The strict example below reports unexpected event order instead of inventing +a missing start or silently overwriting one. + +## Run Ordered Chunks + +The two small helpers are application-side example code, not new FunASR APIs. +The last **nonempty** chunk carries `is_final=True`, including when the input +length is exactly a multiple of the stride. + +```python +import argparse +from pathlib import Path +import soundfile as sf +from funasr import AutoModel + + +def chunk_ranges(length, stride): + if length <= 0 or stride <= 0: + raise ValueError("Audio length and chunk stride must be positive") + for start in range(0, length, stride): + end = min(start + stride, length) + yield start, end, end == length + + +def consume_events(events, pending_start): + spans = [] + for start_ms, end_ms in events: + if start_ms < 0 and end_ms < 0: + raise ValueError("An event must provide a boundary") + if start_ms >= 0: + if pending_start is not None: + raise ValueError("New start before the previous speech span ended") + pending_start = start_ms + if end_ms >= 0: + if pending_start is None or end_ms < pending_start: + raise ValueError("End without a matching earlier start") + spans.append((pending_start, end_ms)) + pending_start = None + return pending_start, spans + + +parser = argparse.ArgumentParser() +parser.add_argument("model_dir") +parser.add_argument("audio") +args = parser.parse_args() +model_dir = Path(args.model_dir).expanduser().resolve(strict=True) +if not model_dir.is_dir(): + raise ValueError("Expected a complete local FSMN VAD model directory") +speech, sample_rate = sf.read(args.audio, dtype="float32") +if sample_rate != 16000 or speech.ndim != 1 or len(speech) == 0: + raise ValueError("Expected nonempty mono 16 kHz audio") + +model = AutoModel( + model=str(model_dir), device="cpu", ncpu=4, + disable_update=True, trust_remote_code=False, +) +chunk_ms = 200 +stride = sample_rate * chunk_ms // 1000 +cache = {} +pending_start = None +for start, end, final in chunk_ranges(len(speech), stride): + results = model.generate( + input=speech[start:end], fs=sample_rate, cache=cache, + chunk_size=chunk_ms, is_streaming_input=True, is_final=final, + batch_size=1, dynamic_silence=False, + ) + for item in results: + pending_start, spans = consume_events(item.get("value", []), pending_start) + for start_ms, end_ms in spans: + print(start_ms, end_ms) +if pending_start is not None: + print("Unclosed speech start (ms):", pending_start) +# End this session; never share its cache with another recording. +cache = {} +``` + +At 16 kHz, `chunk_size=200` means 200 ms, or 3200 input samples. It is a +scalar duration for FSMN VAD, not Paraformer streaming's three-element +`chunk_size` list. Keep the same cache dictionary and chunk settings during +a stream, use `batch_size=1`, and create a new cache for every other recording, +user or cancelled session. This implementation reinitializes its cache after a +nonempty final call; discard it at the application boundary anyway. + +Do not append an empty final call to flush this example: the current +[inference path](../funasr/models/fsmn_vad_streaming/model.py) returns early for +empty input, before its normal finalization path. For live input whose end is +not known in advance, hold the latest nonempty chunk. When the next chunk +arrives, submit the held chunk with `is_final=False` and hold the new one. +Only when end-of-stream arrives, submit the held chunk with `is_final=True`. +That strategy adds one chunk of input buffering; do not resend an already +consumed chunk. Cancelling a session instead discards its held audio and cache. + +## Tune Only After Measuring + +The example explicitly sets `is_streaming_input=True` to avoid the +implementation's mode default depending on `chunk_size`. It also sets +`dynamic_silence=False` to keep checkpoint/configured end-silence settings +instead of the current dynamic schedule. This is a reproducibility choice, +not a recommended threshold or a latency guarantee. Inspect +`max_end_silence_time`, `max_single_segment_time` and model configuration +before tuning; their durations are milliseconds. Input chunk duration is not +the total detection latency or a promise of phonetic boundary accuracy. + +Keep raw boundary events, model/SDK revisions, configuration and the original +audio when investigating clipping. A missing or late boundary is not fixed by +globally padding every segment. If a final start remains unclosed, report it; +do not fabricate an end at the recording duration. + +## SDK and Deployment Boundaries + +- This recipe emits boundaries, not ASR text or subtitles. Combining VAD, ASR, + punctuation and speakers has a different [SDK pipeline contract](python_api.md). +- Python cache events are not a C++ server wire protocol; use the + [WebSocket protocol](../runtime/docs/websocket_protocol.md) for that service. +- Third-party OpenMOSS [MOSS-Transcribe-Diarize](moss_transcribe_diarize.md) + provides its own joint transcription/diarization path. Do not add this external + VAD stage as an assumed prerequisite. +- Source references: [upstream demo](../examples/industrial_data_pretraining/fsmn_vad_streaming/demo.py), + [model implementation](../funasr/models/fsmn_vad_streaming/model.py), + [buffer tests](../tests/test_fsmn_vad_streaming_buffers.py) and + [dynamic-silence tests](../tests/test_dynamic_streaming_vad.py). + [Recipe tests](../tests/test_streaming_vad_docs.py) execute chunk arithmetic and + event assembly without downloading weights; they are not acoustic accuracy, + live microphone, or deployment benchmarks. diff --git a/docs/streaming_vad_zh.md b/docs/streaming_vad_zh.md new file mode 100644 index 000000000..93a4f13a3 --- /dev/null +++ b/docs/streaming_vad_zh.md @@ -0,0 +1,148 @@ +简体中文 | [English](streaming_vad.md) + +# 流式语音活动检测 + +使用 FSMN VAD,按顺序输入同一条录音的音频块,接收语音起点与终点事件。 +VAD 不负责转写、说话人身份识别或语言学断句。首次离线 VAD 调用见 +[SDK 教程](tutorial/README_zh.md);流式 ASR 请使用独立的 +[Paraformer 缓存示例](python_api_zh.md),不要混用两者的协议。 + +## 准备模型与音频 + +先完成[安装检查](installation/installation_zh.md),准备 +`iic/speech_fsmn_vad_zh-cn-16k-common-pytorch` 的完整本地快照, +包括配置、前端文件与权重,记录实际模型 revision 并阅读模型自己的许可证。 +本例针对该 16 kHz 检查点,不适用于任意 VAD 或流式 ASR 模型。 + +输入必须为非空、单声道、16 kHz WAV。下方完整程序的两个命令行位置参数 +依次是本地模型目录和音频路径。它一次读取文件,再模拟按序到达的音频块; +不是麦克风采集或 WebSocket 服务,整段文件仍会占用内存。 + +## 理解边界事件 + +返回的每个字典包含 `value` 列表。每一对边界的单位都是**毫秒**, +相对于**本条流的起点**,不是当前音频块的起点,不要再叠加块偏移量。 + +| 事件 | 含义 | +| --- | --- | +| `[]` | 没有新边界;之前的语音段仍可能未结束。 | +| `[[start, -1]]` | 检测到起点,保存它,等待后续调用给出终点。 | +| `[[-1, end]]` | 之前开始的语音段结束。 | +| `[[start, end]]` | 本次已经获得完整起止边界。 | + +一次调用可以返回多对边界。`-1` 是缺少边界的标记,不能拿它直接切音频。 +每条流独立保存待配对的起点。下面的严格示例遇到异常事件顺序会报错, +不会凭空补起点或悄悄覆盖尚未结束的语音段。 + +## 按序输入音频块 + +两个辅助函数属于应用示例,不是新增的 FunASR API。将 `is_final=True` +放在最后一个**非空**音频块上;音频长度恰好整除块长度时也遵守这一规则。 + +```python +import argparse +from pathlib import Path +import soundfile as sf +from funasr import AutoModel + + +def chunk_ranges(length, stride): + if length <= 0 or stride <= 0: + raise ValueError("Audio length and chunk stride must be positive") + for start in range(0, length, stride): + end = min(start + stride, length) + yield start, end, end == length + + +def consume_events(events, pending_start): + spans = [] + for start_ms, end_ms in events: + if start_ms < 0 and end_ms < 0: + raise ValueError("An event must provide a boundary") + if start_ms >= 0: + if pending_start is not None: + raise ValueError("New start before the previous speech span ended") + pending_start = start_ms + if end_ms >= 0: + if pending_start is None or end_ms < pending_start: + raise ValueError("End without a matching earlier start") + spans.append((pending_start, end_ms)) + pending_start = None + return pending_start, spans + + +parser = argparse.ArgumentParser() +parser.add_argument("model_dir") +parser.add_argument("audio") +args = parser.parse_args() +model_dir = Path(args.model_dir).expanduser().resolve(strict=True) +if not model_dir.is_dir(): + raise ValueError("Expected a complete local FSMN VAD model directory") +speech, sample_rate = sf.read(args.audio, dtype="float32") +if sample_rate != 16000 or speech.ndim != 1 or len(speech) == 0: + raise ValueError("Expected nonempty mono 16 kHz audio") + +model = AutoModel( + model=str(model_dir), device="cpu", ncpu=4, + disable_update=True, trust_remote_code=False, +) +chunk_ms = 200 +stride = sample_rate * chunk_ms // 1000 +cache = {} +pending_start = None +for start, end, final in chunk_ranges(len(speech), stride): + results = model.generate( + input=speech[start:end], fs=sample_rate, cache=cache, + chunk_size=chunk_ms, is_streaming_input=True, is_final=final, + batch_size=1, dynamic_silence=False, + ) + for item in results: + pending_start, spans = consume_events(item.get("value", []), pending_start) + for start_ms, end_ms in spans: + print(start_ms, end_ms) +if pending_start is not None: + print("Unclosed speech start (ms):", pending_start) +# End this session; never share its cache with another recording. +cache = {} +``` + +16 kHz 下,`chunk_size=200` 代表 200 毫秒,即 3200 个输入采样点。 +FSMN VAD 的这个参数是单个时长数值,不是 Paraformer 流式模型的三元素列表。 +同一条流保持同一个缓存字典和固定块设置,使用 `batch_size=1`。 +不同录音、用户或取消后的新会话必须使用新缓存。当前实现会在非空最终调用后 +重新初始化缓存;应用结束会话时仍应丢弃旧缓存。 + +不要另加一次空输入来刷新本例的尾部:[当前推理实现](../funasr/models/fsmn_vad_streaming/model.py) +会在空输入时提前返回,尚未走到正常的收尾路径。实时输入若事先不知道何时结束, +可保留最新一个非空块。下一个块到达时,以 `is_final=False` 发送保留块, +再保留新到达的块;只有收到结束信号时,才以 `is_final=True` 发送保留块。 +这种方式增加一个输入块的缓冲延迟,不能重复发送已经消费过的音频块。 +如果是取消会话,则丢弃保留的音频和缓存。 + +## 测量之后再调参 + +本例显式设置 `is_streaming_input=True`,避免流式模式默认值随 +`chunk_size` 改变。同时设置 `dynamic_silence=False`,使用检查点或配置的 +结束静音设置,而不是当前的动态策略。这是便于复现的选择,不是推荐阈值, +也不保证检测延迟。调参前检查 `max_end_silence_time`、 +`max_single_segment_time` 和模型配置,它们的时长单位都是毫秒。 +音频块时长不等于整体检测延迟,也不代表音素边界精度。 + +排查截断问题时保留原始边界事件、模型与 SDK revision、配置及原音频。 +边界缺失或偏晚,不代表给所有段全局增加 padding 就能解决。 +最终仍有未闭合起点时应报告这一状态,不要擅自把录音长度当作终点。 + +## SDK 与部署边界 + +- 本例只产生语音边界,不输出转写或字幕。组合 VAD、ASR、标点和说话人时, + 应遵循另外的 [SDK 流水线契约](python_api_zh.md)。 +- Python 缓存事件不是 C++ 服务网络协议;连接对应服务请看 + [WebSocket 协议](../runtime/docs/websocket_protocol_zh.md)。 +- 第三方 OpenMOSS 的 [MOSS-Transcribe-Diarize](moss_transcribe_diarize_zh.md) + 有自己的联合转写与说话人分离路径,不要假定它必须增加本例的外部 VAD。 +- 实现依据:[原始示例](../examples/industrial_data_pretraining/fsmn_vad_streaming/demo.py)、 + [模型源码](../funasr/models/fsmn_vad_streaming/model.py)、 + [缓冲测试](../tests/test_fsmn_vad_streaming_buffers.py)、 + [动态静音测试](../tests/test_dynamic_streaming_vad.py)。 + [本文示例测试](../tests/test_streaming_vad_docs.py) 无需下载权重,只执行分块计算 + 和事件配对;不构成声学准确率、实时麦克风或部署性能测试。 diff --git a/tests/test_streaming_vad_docs.py b/tests/test_streaming_vad_docs.py new file mode 100644 index 000000000..108c70a69 --- /dev/null +++ b/tests/test_streaming_vad_docs.py @@ -0,0 +1,67 @@ +"""Execute the documented event/chunk handling without loading model weights.""" + +import ast +from pathlib import Path + +import pytest +from bs4 import BeautifulSoup +from markdown import markdown + +ROOT = Path(__file__).resolve().parents[1] + + +@pytest.fixture(params=["streaming_vad.md", "streaming_vad_zh.md"]) +def recipe(request): + return lambda: load_recipe(ROOT / "docs" / request.param) + + +def load_recipe(path): + assert path.is_file(), f"Missing runnable guide: {path}" + soup = BeautifulSoup(markdown(path.read_text(), extensions=["fenced_code"]), "html.parser") + blocks = soup.select("code.language-python") + assert len(blocks) == 1 + tree = ast.parse(blocks[0].get_text()) + functions = [node for node in tree.body if isinstance(node, ast.FunctionDef)] + scope = {} + exec(compile(ast.Module(body=functions, type_ignores=[]), str(path), "exec"), scope) + return scope + + +@pytest.mark.parametrize("length,want", [ + (1, [(0, 1, True)]), + (3200, [(0, 3200, True)]), + (3201, [(0, 3200, False), (3200, 3201, True)]), + (6400, [(0, 3200, False), (3200, 6400, True)]), +]) +def test_last_nonempty_chunk_is_final(recipe, length, want): + recipe = recipe() + assert list(recipe["chunk_ranges"](length, 3200)) == want + + +@pytest.mark.parametrize("length,stride", [(0, 3200), (-1, 3200), (10, 0), (10, -1)]) +def test_invalid_audio_chunk_dimensions_are_rejected(recipe, length, stride): + recipe = recipe() + with pytest.raises(ValueError): + list(recipe["chunk_ranges"](length, stride)) + + +def test_partial_events_pair_across_calls_without_chunk_offsets(recipe): + recipe = recipe() + consume = recipe["consume_events"] + pending, spans = consume([[100, -1]], None) + assert (pending, spans) == (100, []) + pending, spans = consume([], pending) + assert (pending, spans) == (100, []) + pending, spans = consume([[-1, 600], [800, 1000], [1200, -1]], pending) + assert (pending, spans) == (1200, [(100, 600), (800, 1000)]) + assert consume([[-1, 1500]], pending) == (None, [(1200, 1500)]) + + +@pytest.mark.parametrize("events,pending", [ + ([[-1, 600]], None), ([[100, 50]], None), + ([[200, -1]], 100), ([[-1, -1]], None), +]) +def test_invalid_event_order_does_not_silently_invent_spans(recipe, events, pending): + recipe = recipe() + with pytest.raises(ValueError): + recipe["consume_events"](events, pending) diff --git a/web-pages/product-site/data/documentation.json b/web-pages/product-site/data/documentation.json index 1992b2a20..cee4f7abe 100644 --- a/web-pages/product-site/data/documentation.json +++ b/web-pages/product-site/data/documentation.json @@ -17,6 +17,7 @@ {"slug": "migration", "group": "start", "zh": "从 Whisper 与云 API 迁移", "en": "Migrate from Whisper & cloud APIs", "source_zh": "docs/migration_from_whisper_zh.md", "source_en": "docs/migration_from_whisper.md"}, {"slug": "moss-transcribe-diarize", "group": "models", "zh": "MOSS 转写与说话人分离", "en": "MOSS transcription & diarization", "source_zh": "docs/moss_transcribe_diarize_zh.md", "source_en": "docs/moss_transcribe_diarize.md"}, {"slug": "model-zoo", "group": "models", "zh": "Model Zoo", "en": "Model Zoo", "source_zh": "model_zoo/readme_zh.md", "source_en": "model_zoo/readme.md"}, + {"slug": "streaming-vad", "group": "models", "zh": "流式语音活动检测", "en": "Streaming voice activity detection", "source_zh": "docs/streaming_vad_zh.md", "source_en": "docs/streaming_vad.md"}, {"slug": "training", "group": "train", "zh": "微调与评测", "en": "Fine-tuning & evaluation", "source_zh": "docs/training_zh.md", "source_en": "docs/training.md"}, {"slug": "model-registration", "group": "train", "zh": "自定义模型注册", "en": "Custom model registration", "source_zh": "docs/model_registration_zh.md", "source_en": "docs/model_registration.md"}, {"slug": "docker", "group": "deploy", "zh": "容器环境与镜像选择", "en": "Containers & image selection", "source_zh": "docs/installation/docker_zh.md", "source_en": "docs/installation/docker.md"},