From d6ee34ea92e1fd89a1410883140640f349ad24e8 Mon Sep 17 00:00:00 2001 From: LauraGPT <18321252+LauraGPT@users.noreply.github.com> Date: Sat, 29 Aug 2026 22:10:12 +0000 Subject: [PATCH] feat(moss): support long vLLM transcriptions Signed-off-by: LauraGPT <18321252+LauraGPT@users.noreply.github.com> --- docs/moss_transcribe_diarize.md | 48 ++++++++++++++++++- docs/moss_transcribe_diarize_zh.md | 38 ++++++++++++++- .../models/moss_transcribe_diarize/model.py | 14 ++++-- tests/test_moss_transcribe_diarize_docs.py | 4 ++ tests/test_moss_transcribe_diarize_model.py | 42 ++++++++++++++++ 5 files changed, 138 insertions(+), 8 deletions(-) diff --git a/docs/moss_transcribe_diarize.md b/docs/moss_transcribe_diarize.md index 02ed5d460..abbb2b4cf 100644 --- a/docs/moss_transcribe_diarize.md +++ b/docs/moss_transcribe_diarize.md @@ -81,7 +81,7 @@ model = AutoModel( vllm_response_format="diarized_json", disable_update=True, ) -result = model.generate("audio.wav")[0] +result = model.generate("audio.wav", max_completion_tokens=8192)[0] ``` The vLLM adapter sends the documented OpenAI-compatible multipart request and @@ -102,6 +102,7 @@ metadata is in `sentence_info`. | vLLM | CUDA 12 (`cu129`) or CUDA 13 (`cu130`) | `response_format=diarized_json` returns OpenAI-compatible speaker segments; `response_format=json` keeps raw `[start][Sxx]text[end]` text | You already operate vLLM or need its scheduler | | SGLang Omni | CUDA 13 in the current upstream guide | `response_format=verbose_json` returns parsed segments | You need structured speaker segments directly from the API | | Transformers | PyTorch process | Python objects and raw tagged text | Evaluation, debugging, or custom preprocessing | +| moss-transcribe.cpp / LocalAI | C++17, ggml, and GGUF on CPU or ggml GPU backends | LocalAI's OpenAI-compatible transcription endpoint | You need quantized GGUF or edge deployment without Python/PyTorch on the inference host | The two HTTP backends share `/v1/audio/transcriptions`, but their documented response formats are not interchangeable. The pinned vLLM revision supports @@ -145,6 +146,7 @@ curl -fsS http://127.0.0.1:8898/v1/audio/transcriptions \ -F file=@audio.wav \ -F model=moss-transcribe-diarize \ -F response_format=diarized_json \ + -F max_completion_tokens=8192 \ -F temperature=0 \ | tee moss-transcription.json @@ -171,7 +173,10 @@ For an audit of the model's compact tagged generation, repeat the request with `response_format=json` and verify `[S01]` appears in the returned `text`. For long recordings, test `max_completion_tokens` against the longest expected -meeting. A larger limit can increase memory use and tail latency. +meeting. The FunASR vLLM adapter accepts both the OpenAI-compatible +`max_completion_tokens` name and the compatibility alias `max_new_tokens`; the +native name wins when both are present. A larger limit can increase memory use +and tail latency. ### Reproduced vLLM contract @@ -182,6 +187,26 @@ returned two `diarized_json` segments labelled `S01` and `S02`; the same request through the FunASR adapter returned two monotonic `sentence_info` entries with the same speakers. +The same environment also processed the two real Chinese long recordings from +FunASR #3539: + +- the 309.600-second sample + (`6561ee553c8f762aac4ebd65439d3414820761b547fa3a2edcea43b86a2abc02`) + returned 158 segments through 309.41 seconds with the default 5120-token + limit in 7.459 seconds; +- the 379.664-second sample + (`779899a3ce937dd7352b4db1ea53e3f6aa2cfef7109de0249082223c936f9372`) + truncated at 354.98 seconds with the default 5120-token limit, causing an + HTTP 400 for incomplete `diarized_json` tags. With + `max_completion_tokens=8192`, it returned 224 segments through 378.55 + seconds in 10.516 seconds. + +MOSS corrected several proper-noun and near-homophone errors named by the +reporter and preserved the semantic continuity of the foundation-statement +phrase. It still emitted some short standalone segments, so this does not prove +the subtitle-grouping issue is resolved. These measurements establish an exact +input completeness boundary, not accuracy, throughput, or production capacity. + Earlier raw-response validation on vLLM `0.23.1rc1.dev949+g68b4a1d58`, Torch `2.11.0+cu129`, and one H100 80GB found: @@ -210,6 +235,25 @@ returned two monotonic segments labelled `S01` and `S02` through the common `AutoModel` result contract. The temporary vLLM worker was stopped after the test; this is a contract smoke test, not an accuracy benchmark. +## moss-transcribe.cpp and LocalAI + +[`localai-org/moss-transcribe.cpp`](https://github.com/localai-org/moss-transcribe.cpp) +is a third-party C++17/ggml reimplementation maintained by the LocalAI team and +licensed under MIT. The original OpenMOSS model weights remain Apache-2.0. It is +not a FunASR `AutoModel` backend and is not interchangeable with the Python +adapter above. + +Choose this path for GGUF quantization, CPU inference, or ggml +CUDA/Metal/Vulkan/HIP backends. LocalAI +`master@a7cc5873ef5b7c909fc9ff7d349d51738ba9bb05` includes a +`moss-transcribe-cpp` backend and Hugging Face importer. The backend pins +`moss-transcribe.cpp@190a569c13b4b247450f2fb3b2a431244e84833e`, while the +importer recognizes `huggingface://mudler/moss-transcribe.cpp-gguf` and selects +Q5_K by default. Before production, follow LocalAI's documentation and verify +the backend package, GGUF SHA-256, actual hardware backend, and +`/v1/audio/transcriptions` response. Do not treat upstream README performance +claims as measurements of your host. + ## SGLang Omni Follow the pinned upstream SGLang Omni installation guide for CUDA 13, then diff --git a/docs/moss_transcribe_diarize_zh.md b/docs/moss_transcribe_diarize_zh.md index 324d8d68e..87e8b390c 100644 --- a/docs/moss_transcribe_diarize_zh.md +++ b/docs/moss_transcribe_diarize_zh.md @@ -73,7 +73,7 @@ model = AutoModel( vllm_response_format="diarized_json", disable_update=True, ) -result = model.generate("audio.wav")[0] +result = model.generate("audio.wav", max_completion_tokens=8192)[0] ``` vLLM 适配器发送官方文档中的 OpenAI-compatible multipart 请求，并把官方返回的 @@ -92,6 +92,7 @@ speaker `segments` 直接映射到 `sentence_info`。认证可通过 `vllm_api_k | vLLM | CUDA 12（`cu129`）或 CUDA 13（`cu130`） | `response_format=diarized_json` 返回 OpenAI-compatible 说话人 segments；`response_format=json` 保留原始 `[start][Sxx]text[end]` 文本 | 已运行 vLLM，或需要其调度能力 | | SGLang Omni | 当前上游指南为 CUDA 13 | `response_format=verbose_json` 返回解析后的 segments | API 需要直接返回结构化说话人分段 | | Transformers | PyTorch 进程 | Python 对象和原始标签文本 | 评测、排障或自定义预处理 | +| moss-transcribe.cpp / LocalAI | C++17、ggml、GGUF；CPU 或 ggml GPU 后端 | LocalAI 的 OpenAI-compatible 转写接口 | 不希望在推理机安装 Python/PyTorch，或需要量化 GGUF 和边缘部署 | 两个 HTTP 后端都使用 `/v1/audio/transcriptions`，但官方文档中的响应格式不能混为 一谈。固定的 vLLM revision 支持 `diarized_json`；承诺 `segments` 前仍要验证实际 @@ -134,6 +135,7 @@ curl -fsS http://127.0.0.1:8898/v1/audio/transcriptions \ -F file=@audio.wav \ -F model=moss-transcribe-diarize \ -F response_format=diarized_json \ + -F max_completion_tokens=8192 \ -F temperature=0 \ | tee moss-transcription.json @@ -159,7 +161,9 @@ PY 如需审计模型原始紧凑标签生成，请把请求改为 `response_format=json`，并验证返回 `text` 中包含 `[S01]`。 -长音频需要按业务最长会议验证 `max_completion_tokens`。增大上限也会影响显存和尾延迟。 +长音频需要按业务最长会议验证 `max_completion_tokens`。FunASR vLLM 适配器同时接受 +OpenAI-compatible 名称 `max_completion_tokens` 和兼容名称 `max_new_tokens`；两者同时 +出现时以前者为准。增大上限也会影响显存和尾延迟。 ### 已复现的 vLLM 契约 @@ -169,6 +173,21 @@ PY 返回两个 `diarized_json` segments，标签为 `S01`、`S02`；同一请求通过 FunASR 适配器后，返回两个时间单调且说话人一致的 `sentence_info`。 +同一环境还验证了 FunASR #3539 的两段真实中文长音频： + +- 309.600 秒样例 + （`6561ee553c8f762aac4ebd65439d3414820761b547fa3a2edcea43b86a2abc02`） + 使用默认 5120 上限返回 158 个 segments，最后时间戳为 309.41 秒，7.459 秒完成； +- 379.664 秒样例 + （`779899a3ce937dd7352b4db1ea53e3f6aa2cfef7109de0249082223c936f9372`） + 使用默认 5120 上限时生成在 354.98 秒处截断，`diarized_json` 因标签不完整返回 + HTTP 400；设置 `max_completion_tokens=8192` 后返回 224 个 segments，最后时间戳 + 覆盖到 378.55 秒，10.516 秒完成。 + +这两段样例中，MOSS 修正了报告者指出的多处专有词和近音词，也正确保持了“转述基金会 +声明”的语义连续性；但模型输出仍包含较短的独立分段，不能据此宣称字幕切分问题已解决。 +这些数字只证明 exact input 的完整性边界，不是精度、吞吐或生产容量基准。 + 较早的 raw-response 验证使用 vLLM `0.23.1rc1.dev949+g68b4a1d58`、Torch `2.11.0+cu129` 和一张 H100 80GB，结果如下： @@ -193,6 +212,21 @@ FunASR 适配器还分别使用 `backend="hf"` 和 `backend="vllm"`，按模型 统一 `AutoModel` 结果契约返回两个单调时间分段和 `S01`、`S02`。测试完成后已停止 临时 vLLM worker；这只是契约冒烟测试，不是精度基准。 +## moss-transcribe.cpp 与 LocalAI + +[`localai-org/moss-transcribe.cpp`](https://github.com/localai-org/moss-transcribe.cpp) +是 LocalAI 团队维护的第三方 C++17/ggml 重写，许可证为 MIT；原始 OpenMOSS 模型 +权重仍保持 Apache-2.0。它不是 FunASR `AutoModel` 后端，也不能与上面的 Python +适配器混用。 + +在需要 GGUF 量化、CPU 或 ggml CUDA/Metal/Vulkan/HIP 后端时，可以选择这条路径。 +LocalAI `master@a7cc5873ef5b7c909fc9ff7d349d51738ba9bb05` 已包含 +`moss-transcribe-cpp` 后端及 Hugging Face importer；后端固定 +`moss-transcribe.cpp@190a569c13b4b247450f2fb3b2a431244e84833e`，importer 可识别 +`huggingface://mudler/moss-transcribe.cpp-gguf` 并默认选择 Q5_K。部署前应按 LocalAI +文档验证所选 backend package、GGUF SHA-256、实际硬件后端及 +`/v1/audio/transcriptions` 响应，不要把上游 README 的性能数字当作本机基准。 + ## SGLang Omni 按固定上游 SGLang Omni 安装指南准备 CUDA 13 环境，然后下载不可变模型快照并从本地目录启动： diff --git a/funasr/models/moss_transcribe_diarize/model.py b/funasr/models/moss_transcribe_diarize/model.py index 0f00dfb1a..64d476071 100644 --- a/funasr/models/moss_transcribe_diarize/model.py +++ b/funasr/models/moss_transcribe_diarize/model.py @@ -210,7 +210,10 @@ class MossTranscribeDiarize(nn.Module): ) self.device_name = kwargs.get("device", "cuda:0") self.dtype_name = kwargs.get("dtype", "bf16") - self.max_new_tokens = int(kwargs.get("max_new_tokens", 5120)) + max_new_tokens = kwargs.get("max_new_tokens", 5120) + if self.backend == "vllm": + max_new_tokens = kwargs.get("max_completion_tokens", max_new_tokens) + self.max_new_tokens = int(max_new_tokens) self.max_length = int(kwargs.get("max_length", 131072)) self._placeholder = nn.Parameter(torch.empty(0), requires_grad=False) @@ -402,9 +405,12 @@ class MossTranscribeDiarize(nn.Module): prompt = kwargs.get("prompt") if prompt: data["prompt"] = prompt - max_new_tokens = kwargs.get("max_new_tokens", self.max_new_tokens) - if max_new_tokens is not None: - data["max_completion_tokens"] = str(max_new_tokens) + max_completion_tokens = kwargs.get( + "max_completion_tokens", + kwargs.get("max_new_tokens", self.max_new_tokens), + ) + if max_completion_tokens is not None: + data["max_completion_tokens"] = str(max_completion_tokens) headers = {} if self.vllm_api_key and self.vllm_api_key != "EMPTY": headers["Authorization"] = "Bearer " + self.vllm_api_key diff --git a/tests/test_moss_transcribe_diarize_docs.py b/tests/test_moss_transcribe_diarize_docs.py index 219d663c2..f74da8196 100644 --- a/tests/test_moss_transcribe_diarize_docs.py +++ b/tests/test_moss_transcribe_diarize_docs.py @@ -43,6 +43,10 @@ def test_moss_guides_pin_upstream_and_separate_serving_contracts(guide: Path) -> "sentence_info", "raw_text", "vad_model", + "max_completion_tokens=8192", + "6561ee553c8f762aac4ebd65439d3414820761b547fa3a2edcea43b86a2abc02", + "779899a3ce937dd7352b4db1ea53e3f6aa2cfef7109de0249082223c936f9372", + "localai-org/moss-transcribe.cpp", ): assert marker in text, f"{guide.name} is missing {marker}" diff --git a/tests/test_moss_transcribe_diarize_model.py b/tests/test_moss_transcribe_diarize_model.py index 55176dae3..a73452837 100644 --- a/tests/test_moss_transcribe_diarize_model.py +++ b/tests/test_moss_transcribe_diarize_model.py @@ -138,6 +138,48 @@ class MossVllmBackendTest(unittest.TestCase): self.assertEqual(content_type, "audio/wav") response.raise_for_status.assert_called_once_with() + def test_prefers_native_vllm_completion_limit_for_long_audio(self): + session = MagicMock() + response = MagicMock() + response.json.return_value = {"text": "[0][S01]hello[1.5]"} + session.post.return_value = response + model = MossTranscribeDiarize( + backend="vllm", + vllm_base_url="http://vllm.test:8000/v1", + http_session=session, + ) + + model.inference( + [np.zeros(1600, dtype=np.float32)], + max_new_tokens=4096, + max_completion_tokens=8192, + ) + + self.assertEqual( + session.post.call_args.kwargs["data"]["max_completion_tokens"], + "8192", + ) + + def test_accepts_native_vllm_completion_limit_at_construction(self): + session = MagicMock() + response = MagicMock() + response.json.return_value = {"text": "[0][S01]hello[1.5]"} + session.post.return_value = response + model = MossTranscribeDiarize( + backend="vllm", + vllm_base_url="http://vllm.test:8000/v1", + max_new_tokens=4096, + max_completion_tokens=8192, + http_session=session, + ) + + model.inference([np.zeros(1600, dtype=np.float32)]) + + self.assertEqual( + session.post.call_args.kwargs["data"]["max_completion_tokens"], + "8192", + ) + def test_normalizes_official_vllm_diarized_json_response(self): session = MagicMock() response = MagicMock()