diff --git a/.github/workflows/test-moss-adapter.yml b/.github/workflows/test-moss-adapter.yml new file mode 100644 index 000000000..53a97c82e --- /dev/null +++ b/.github/workflows/test-moss-adapter.yml @@ -0,0 +1,53 @@ +name: Validate MOSS adapter + +on: + pull_request: + paths: + - "funasr/models/moss_transcribe_diarize/**" + - "funasr/auto/auto_model.py" + - "docs/moss_transcribe_diarize*.md" + - "tests/test_moss_transcribe_diarize_*.py" + - ".github/workflows/test-moss-adapter.yml" + push: + branches: + - main + paths: + - "funasr/models/moss_transcribe_diarize/**" + - "funasr/auto/auto_model.py" + - "docs/moss_transcribe_diarize*.md" + - "tests/test_moss_transcribe_diarize_*.py" + - ".github/workflows/test-moss-adapter.yml" + +permissions: + contents: read + +jobs: + contract: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + cache: pip + - name: Install contract-test dependencies + run: | + python -m pip install --upgrade pip + python -m pip install \ + torch --index-url https://download.pytorch.org/whl/cpu + python -m pip install \ + pytest black==25.1.0 numpy soundfile requests packaging \ + omegaconf hydra-core kaldiio librosa tqdm pyyaml + - name: Check formatting and syntax + run: | + python -m black --check \ + funasr/models/moss_transcribe_diarize/model.py \ + tests/test_moss_transcribe_diarize_model.py \ + tests/test_moss_transcribe_diarize_docs.py + python -m py_compile funasr/models/moss_transcribe_diarize/model.py + - name: Run adapter and documentation contracts + run: | + python -m pytest -q \ + tests/test_moss_transcribe_diarize_model.py \ + tests/test_moss_transcribe_diarize_docs.py diff --git a/docs/moss_transcribe_diarize.md b/docs/moss_transcribe_diarize.md index ae0f5f810..6467de591 100644 --- a/docs/moss_transcribe_diarize.md +++ b/docs/moss_transcribe_diarize.md @@ -5,8 +5,9 @@ This guide connects the third-party [OpenMOSS/MOSS-Transcribe-Diarize](https://github.com/OpenMOSS/MOSS-Transcribe-Diarize) model to the FunASR deployment ecosystem. The model is published by OpenMOSS -under Apache-2.0; it is not a FunASR model and is not registered in FunASR -`AutoModel`. +under Apache-2.0; it is not a FunASR model. FunASR provides an adapter for its +public Transformers and vLLM interfaces while retaining the OpenMOSS model +name, license, and upstream revision. MOSS-Transcribe-Diarize jointly generates transcription, timestamps, and speaker labels such as `[S01]`. An application therefore does not need to @@ -25,6 +26,65 @@ This guide was verified against: Pin all three revisions. The model uses `trust_remote_code`; do not execute a floating model revision in a production service. +## FunASR AutoModel contract + +Use an isolated Python 3.10+ environment with Transformers 5.6 or newer for +the local backend. MOSS performs long-form transcription and speaker +diarization in one generation, so do **not** pass `vad_model` or `spk_model`. +External VAD segmentation would break the model's global speaker identity +across chunks. + +```python +from funasr import AutoModel + +model = AutoModel( + model="OpenMOSS-Team/MOSS-Transcribe-Diarize", + model_revision="e8681d68e7042738ffca8ac8212bc8fcb1131ab8", + backend="hf", + device="cuda:0", + dtype="bf16", + attn_implementation="sdpa", + disable_update=True, +) + +result = model.generate("audio.wav", max_new_tokens=5120)[0] +print(result["text"]) +for segment in result["sentence_info"]: + print(segment["start"], segment["end"], segment["spk"], segment["text"]) +``` + +The adapter preserves the raw tagged generation in `raw_text` and returns the +common FunASR fields: + +- `text`: readable transcript without MOSS control tags; +- `timestamp`: segment-level `[start_ms, end_ms]` pairs; +- `sentence_info`: `start`, `end`, `text`, `sentence`, `spk`, and `timestamp`; +- `raw_text`: exact `[start][Sxx]text[end]` generation for auditing. + +If the parser cannot prove the tagged structure, it leaves the model text +visible in `text` and `raw_text` and returns empty timestamp/segment arrays +instead of silently inventing speaker metadata. + +The same result contract can wrap an already running vLLM service without +downloading local weights: + +```python +from funasr import AutoModel + +model = AutoModel( + model="OpenMOSS-Team/MOSS-Transcribe-Diarize", + backend="vllm", + vllm_base_url="http://127.0.0.1:8898/v1", + vllm_model="moss-transcribe-diarize", + disable_update=True, +) +result = model.generate("audio.wav")[0] +``` + +The vLLM adapter sends the documented OpenAI-compatible multipart request and +normalizes the returned raw text. Authentication can be supplied with +`vllm_api_key`; keep it in a secret store rather than source code. + ## Choose a serving path | Path | Environment | Response contract | Use when | @@ -105,6 +165,14 @@ This proves the pinned API and speaker-return contract on those inputs. It is not a diarization accuracy result, overlap test, throughput benchmark, or production capacity claim. +The FunASR adapter was also tested on both `backend="hf"` and +`backend="vllm"` with model revision +`e8681d68e7042738ffca8ac8212bc8fcb1131ab8`. The 15.1685-second two-speaker +probe (`43dccc068506439cb633b382b6b98185baa837363d08cc5f7152ca89b0fdc3c8`) +returned two monotonic segments labelled `S01` and `S02` through the common +`AutoModel` result contract. The temporary vLLM worker was stopped after the +test; this is a contract smoke test, not an accuracy benchmark. + ## SGLang Omni Follow the pinned upstream SGLang Omni installation guide for CUDA 13, then diff --git a/docs/moss_transcribe_diarize_zh.md b/docs/moss_transcribe_diarize_zh.md index 0ded1b05a..c0409a759 100644 --- a/docs/moss_transcribe_diarize_zh.md +++ b/docs/moss_transcribe_diarize_zh.md @@ -5,7 +5,8 @@ 本文把第三方 [OpenMOSS/MOSS-Transcribe-Diarize](https://github.com/OpenMOSS/MOSS-Transcribe-Diarize) 模型接入 FunASR 部署生态。模型由 OpenMOSS 以 Apache-2.0 发布，不是 FunASR -自有模型，也没有注册到 FunASR `AutoModel`。 +自有模型。FunASR 只为其公开的 Transformers 与 vLLM 接口提供适配器，并保留 +OpenMOSS 模型名称、许可证和上游 revision。 MOSS-Transcribe-Diarize 会联合生成转写、时间戳和 `[S01]` 等说话人标签，应用侧 不必再拼接外部 VAD、ASR 和 diarization 管线。这里描述的是部署形态，不表示模型 @@ -22,6 +23,59 @@ MOSS-Transcribe-Diarize 会联合生成转写、时间戳和 `[S01]` 等说话 三者都要固定。模型依赖 `trust_remote_code`，生产服务不要执行浮动的模型 revision。 +## FunASR AutoModel 契约 + +本地后端应使用隔离的 Python 3.10+ 环境，并安装 Transformers 5.6 或更新版本。 +MOSS 在一次生成中完成长音频转写与说话人识别，因此不要传 `vad_model` 或 +`spk_model`。外部 VAD 会把长音频切开，并破坏跨分块的全局说话人身份。 + +```python +from funasr import AutoModel + +model = AutoModel( + model="OpenMOSS-Team/MOSS-Transcribe-Diarize", + model_revision="e8681d68e7042738ffca8ac8212bc8fcb1131ab8", + backend="hf", + device="cuda:0", + dtype="bf16", + attn_implementation="sdpa", + disable_update=True, +) + +result = model.generate("audio.wav", max_new_tokens=5120)[0] +print(result["text"]) +for segment in result["sentence_info"]: + print(segment["start"], segment["end"], segment["spk"], segment["text"]) +``` + +适配器在 `raw_text` 中保留原始标签生成，并返回 FunASR 通用字段： + +- `text`：移除 MOSS 控制标签后的可读转写； +- `timestamp`：segment 级 `[start_ms, end_ms]`； +- `sentence_info`：包含 `start`、`end`、`text`、`sentence`、`spk` 和 `timestamp`； +- `raw_text`：用于审计的原始 `[start][Sxx]text[end]` 生成。 + +当解析器无法确认标签结构时，会保留 `text` 与 `raw_text`，并返回空时间戳和分段， +不会静默编造说话人信息。 + +同一结果契约也可以包装已经启动的 vLLM 服务，并且不会下载本地权重： + +```python +from funasr import AutoModel + +model = AutoModel( + model="OpenMOSS-Team/MOSS-Transcribe-Diarize", + backend="vllm", + vllm_base_url="http://127.0.0.1:8898/v1", + vllm_model="moss-transcribe-diarize", + disable_update=True, +) +result = model.generate("audio.wav")[0] +``` + +vLLM 适配器发送官方文档中的 OpenAI-compatible multipart 请求，并统一解析返回文本。 +认证可通过 `vllm_api_key` 提供；请放在 secret store，不要写入源码。 + ## 选择服务后端 | 路径 | 环境 | 响应契约 | 适用场景 | @@ -98,6 +152,12 @@ FunASR 生态验证使用 vLLM `0.23.1rc1.dev949+g68b4a1d58`、Torch 这证明固定版本对这些输入满足 API 和说话人返回契约，不是 diarization 精度、重叠语音、 吞吐或生产容量结论。 +FunASR 适配器还分别使用 `backend="hf"` 和 `backend="vllm"`，按模型 revision +`e8681d68e7042738ffca8ac8212bc8fcb1131ab8` 做了真实测试。15.1685 秒的双说话人 +样例（`43dccc068506439cb633b382b6b98185baa837363d08cc5f7152ca89b0fdc3c8`）通过 +统一 `AutoModel` 结果契约返回两个单调时间分段和 `S01`、`S02`。测试完成后已停止 +临时 vLLM worker；这只是契约冒烟测试，不是精度基准。 + ## SGLang Omni 按固定上游 SGLang Omni 安装指南准备 CUDA 13 环境，然后下载不可变模型快照并从本地目录启动： diff --git a/funasr/auto/auto_model.py b/funasr/auto/auto_model.py index 3dbb7a4f2..d38174663 100644 --- a/funasr/auto/auto_model.py +++ b/funasr/auto/auto_model.py @@ -542,6 +542,12 @@ class AutoModel: if kwargs["model"] in {"silero-vad", "silero_vad"}: kwargs.setdefault("model_conf", {}) kwargs["model"] = "SileroVad" + if kwargs["model"] in { + "MOSS-Transcribe-Diarize", + "OpenMOSS-Team/MOSS-Transcribe-Diarize", + }: + kwargs.setdefault("model_conf", {}) + kwargs.setdefault("model_path", kwargs["model"]) if "model_conf" not in kwargs: logging.info("download models from model hub: {}".format(kwargs.get("hub", "ms"))) kwargs = download_model(**kwargs) diff --git a/funasr/models/moss_transcribe_diarize/__init__.py b/funasr/models/moss_transcribe_diarize/__init__.py new file mode 100644 index 000000000..0c65ebc45 --- /dev/null +++ b/funasr/models/moss_transcribe_diarize/__init__.py @@ -0,0 +1,5 @@ +"""OpenMOSS MOSS-Transcribe-Diarize integration.""" + +from .model import MossTranscribeDiarize + +__all__ = ["MossTranscribeDiarize"] diff --git a/funasr/models/moss_transcribe_diarize/model.py b/funasr/models/moss_transcribe_diarize/model.py new file mode 100644 index 000000000..fcc302be5 --- /dev/null +++ b/funasr/models/moss_transcribe_diarize/model.py @@ -0,0 +1,391 @@ +"""FunASR adapter for OpenMOSS MOSS-Transcribe-Diarize. + +The model and its reference implementation are maintained by the OpenMOSS +Team under Apache-2.0: +https://github.com/OpenMOSS/MOSS-Transcribe-Diarize +""" + +import copy +import io +import mimetypes +import os +import re +import time + +import numpy as np +import torch +import torch.nn as nn + +from funasr.register import tables + + +DEFAULT_MODEL = "OpenMOSS-Team/MOSS-Transcribe-Diarize" +DEFAULT_PROMPT = ( + "请将音频转写为文本，每一段需以起始时间戳和说话人编号" + "（[S01]、[S02]、[S03]…）开头，正文为对应的语音内容，" + "并在段末标注结束时间戳，以清晰标明该段语音范围。" +) +_NUMBER = r"(?:\d+(?:\.\d+)?|\.\d+)" +_SEGMENT_START = re.compile(r"\[(%s)\]\[(S\d+)\]" % _NUMBER) +_TIMESTAMP = re.compile(r"\[(%s)\]" % _NUMBER) + + +def _parse_transcript(transcript): + """Parse the model's ``[start][Sxx]text[end]`` output.""" + segments = [] + cursor = 0 + while True: + start_match = _SEGMENT_START.search(transcript, cursor) + if start_match is None: + break + + text_start = start_match.end() + candidate_cursor = text_start + end_match = None + while True: + candidate = _TIMESTAMP.search(transcript, candidate_cursor) + if candidate is None: + break + tail = candidate.end() + next_start = _SEGMENT_START.match(transcript, tail) + if next_start is None: + whitespace_tail = tail + while ( + whitespace_tail < len(transcript) + and transcript[whitespace_tail].isspace() + ): + whitespace_tail += 1 + next_start = _SEGMENT_START.match(transcript, whitespace_tail) + at_end = whitespace_tail == len(transcript) + else: + at_end = False + if next_start is not None or at_end: + end_match = candidate + break + candidate_cursor = candidate.end() + + if end_match is None: + cursor = start_match.end() + continue + + start = float(start_match.group(1)) + end = float(end_match.group(1)) + if end >= start: + text = transcript[text_start : end_match.start()].strip() + if text: + segments.append((start, end, start_match.group(2), text)) + cursor = end_match.end() + return segments + + +def _join_texts(texts): + if not texts: + return "" + joined = texts[0] + for text in texts[1:]: + previous = joined[-1] if joined else "" + current = text[0] if text else "" + cjk = "\u3400" <= previous <= "\u9fff" and "\u3400" <= current <= "\u9fff" + joined += ("" if cjk else " ") + text + return joined + + +def _result_from_transcript(key, transcript): + segments = _parse_transcript(transcript) + if not segments: + return { + "key": key, + "text": transcript.strip(), + "raw_text": transcript, + "timestamp": [], + "sentence_info": [], + } + + sentence_info = [] + timestamps = [] + texts = [] + for start_s, end_s, speaker, text in segments: + timestamp = [int(round(start_s * 1000)), int(round(end_s * 1000))] + timestamps.append(timestamp) + texts.append(text) + sentence_info.append( + { + "start": timestamp[0], + "end": timestamp[1], + "text": text, + "sentence": text, + "spk": speaker, + "timestamp": [timestamp], + } + ) + return { + "key": key, + "text": _join_texts(texts), + "raw_text": transcript, + "timestamp": timestamps, + "sentence_info": sentence_info, + } + + +@tables.register("model_classes", "MOSS-Transcribe-Diarize") +@tables.register("model_classes", DEFAULT_MODEL) +class MossTranscribeDiarize(nn.Module): + """End-to-end ASR, diarization, timestamps, and acoustic-event inference.""" + + def __init__(self, **kwargs): + super().__init__() + if kwargs.get("vad_model") is not None or kwargs.get("spk_model") is not None: + raise ValueError( + "MOSS-Transcribe-Diarize performs long-form segmentation and speaker " + "diarization end to end; omit vad_model and spk_model to preserve global " + "speaker identity." + ) + + self.backend = kwargs.get("backend", "hf").lower() + if self.backend not in {"hf", "vllm"}: + raise ValueError("backend must be 'hf' or 'vllm'") + self.model_path = ( + kwargs.get("model_path") or kwargs.get("model") or DEFAULT_MODEL + ) + self.device_name = kwargs.get("device", "cuda:0") + self.dtype_name = kwargs.get("dtype", "bf16") + self.max_new_tokens = int(kwargs.get("max_new_tokens", 5120)) + self.max_length = int(kwargs.get("max_length", 131072)) + self._placeholder = nn.Parameter(torch.empty(0), requires_grad=False) + + self.vllm_base_url = kwargs.get("vllm_base_url") + self.vllm_model = kwargs.get("vllm_model", DEFAULT_MODEL) + self.vllm_api_key = kwargs.get("vllm_api_key", "EMPTY") + self.vllm_timeout = float(kwargs.get("vllm_timeout", 600.0)) + self.http_session = kwargs.get("http_session") + + self.hf_model = None + self.processor = None + if self.backend == "vllm": + if not self.vllm_base_url: + raise ValueError("vllm_base_url is required when backend='vllm'") + else: + self._load_hf_backend(kwargs) + + def _load_hf_backend(self, kwargs): + try: + import transformers + from packaging.version import Version + from transformers import AutoModelForCausalLM, AutoProcessor + except ImportError as exc: + raise ImportError( + "MOSS-Transcribe-Diarize requires Transformers 5.6 or newer. " + "Install it in an isolated environment with `pip install 'transformers>=5.6,<6'`." + ) from exc + + if Version(transformers.__version__) < Version("5.6.0"): + raise ImportError( + "MOSS-Transcribe-Diarize requires Transformers 5.6 or newer; found %s. " + "Use an isolated environment so other FunASR model constraints remain unchanged." + % transformers.__version__ + ) + + self.processor = AutoProcessor.from_pretrained( + self.model_path, + trust_remote_code=True, + revision=kwargs.get("model_revision"), + ) + attention = kwargs.get("attn_implementation", "sdpa") + attempts = [attention] + if attention == "flash_attention_2": + attempts.append("sdpa") + if "eager" not in attempts: + attempts.append("eager") + last_error = None + for implementation in attempts: + try: + self.hf_model = AutoModelForCausalLM.from_pretrained( + self.model_path, + trust_remote_code=True, + revision=kwargs.get("model_revision"), + dtype="auto", + attn_implementation=implementation, + ) + break + except RuntimeError as exc: + last_error = exc + if self.hf_model is None: + raise last_error + + dtype = { + "bf16": torch.bfloat16, + "bfloat16": torch.bfloat16, + "fp16": torch.float16, + "float16": torch.float16, + "fp32": torch.float32, + "float32": torch.float32, + }.get(self.dtype_name, torch.bfloat16) + device = torch.device(self.device_name) + if device.type == "cuda" and not torch.cuda.is_available(): + device = torch.device("cpu") + dtype = torch.float32 + self.hf_model.to(dtype=dtype).to(device).eval() + self._hf_device = device + self._hf_dtype = dtype + + def forward(self, **kwargs): + raise NotImplementedError("MOSS-Transcribe-Diarize supports inference only") + + def inference(self, data_in, data_lengths=None, key=None, **kwargs): + del data_lengths + started = time.perf_counter() + inputs = list(data_in) if isinstance(data_in, (list, tuple)) else [data_in] + keys = key or ["sample_%d" % index for index in range(len(inputs))] + audio_duration = sum(self._audio_duration(audio) for audio in inputs) + results = [] + for index, audio in enumerate(inputs): + if self.backend == "vllm": + transcript = self._transcribe_vllm(audio, **kwargs) + else: + transcript = self._transcribe_hf(audio, **kwargs) + results.append(_result_from_transcript(keys[index], transcript)) + return results, { + "batch_data_time": max(audio_duration, 1e-9), + "forward": time.perf_counter() - started, + } + + def _transcribe_hf(self, audio, **kwargs): + from transformers.audio_utils import load_audio + + sampling_rate = self.processor.feature_extractor.sampling_rate + waveform = ( + audio + if isinstance(audio, np.ndarray) + else load_audio(audio, sampling_rate=sampling_rate) + ) + prompt = kwargs.get("prompt", DEFAULT_PROMPT) + messages = [ + { + "role": "user", + "content": [ + {"type": "audio", "audio": waveform}, + {"type": "text", "text": prompt.strip() or DEFAULT_PROMPT}, + ], + } + ] + text = self.processor.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True + ) + audio_kwargs = ( + {"device": str(self._hf_device)} if self._hf_device.type == "cuda" else {} + ) + inputs = self.processor( + text=text, + audio=[waveform], + max_length=int(kwargs.get("max_length", self.max_length)), + audio_kwargs=audio_kwargs, + return_tensors="pt", + ).to(self._hf_device) + prompt_len = int(inputs["attention_mask"][0].sum().item()) + generation_config = copy.deepcopy(self.hf_model.generation_config) + generation_config.max_new_tokens = int( + kwargs.get("max_new_tokens", self.max_new_tokens) + ) + generation_config.do_sample = bool(kwargs.get("do_sample", False)) + if generation_config.do_sample: + generation_config.temperature = float(kwargs.get("temperature", 0.8)) + + with torch.inference_mode(), ( + torch.amp.autocast("cuda", dtype=self._hf_dtype) + if self._hf_device.type == "cuda" + and self._hf_dtype in (torch.float16, torch.bfloat16) + else torch.no_grad() + ): + outputs = self.hf_model.generate( + input_ids=inputs["input_ids"], + attention_mask=inputs["attention_mask"], + input_features=inputs["input_features"], + audio_feature_lengths=inputs["audio_feature_lengths"], + audio_chunk_mapping=inputs["audio_chunk_mapping"], + generation_config=generation_config, + ) + generated = outputs[0][prompt_len:] + return self.processor.tokenizer.decode( + generated, skip_special_tokens=True + ).strip() + + def _transcriptions_url(self): + base = self.vllm_base_url.rstrip("/") + if base.endswith("/v1/audio/transcriptions"): + return base + if not base.endswith("/v1"): + base += "/v1" + return base + "/audio/transcriptions" + + def _transcribe_vllm(self, audio, **kwargs): + import requests + + session = self.http_session or requests.Session() + filename, payload, content_type = self._multipart_audio(audio) + data = { + "model": self.vllm_model, + "response_format": "json", + "temperature": str(kwargs.get("temperature", 0)), + } + prompt = kwargs.get("prompt") + if prompt: + data["prompt"] = prompt + max_new_tokens = kwargs.get("max_new_tokens", self.max_new_tokens) + if max_new_tokens is not None: + data["max_completion_tokens"] = str(max_new_tokens) + headers = {} + if self.vllm_api_key and self.vllm_api_key != "EMPTY": + headers["Authorization"] = "Bearer " + self.vllm_api_key + response = session.post( + self._transcriptions_url(), + data=data, + files={"file": (filename, payload, content_type)}, + headers=headers, + timeout=self.vllm_timeout, + ) + response.raise_for_status() + result = response.json() + if not isinstance(result, dict) or not isinstance(result.get("text"), str): + raise RuntimeError( + "vLLM transcription response did not contain a text field" + ) + return result["text"] + + @staticmethod + def _multipart_audio(audio): + if isinstance(audio, np.ndarray): + import soundfile as sf + + payload = io.BytesIO() + sf.write(payload, audio, 16000, format="WAV") + payload.seek(0) + return "audio.wav", payload, "audio/wav" + if isinstance(audio, bytes): + return "audio.wav", io.BytesIO(audio), "audio/wav" + if isinstance(audio, os.PathLike): + audio = os.fspath(audio) + if isinstance(audio, str) and os.path.isfile(audio): + with open(audio, "rb") as source: + payload = io.BytesIO(source.read()) + content_type = mimetypes.guess_type(audio)[0] or "application/octet-stream" + return os.path.basename(audio), payload, content_type + raise TypeError( + "vLLM backend accepts a local audio path, bytes, or a 16 kHz numpy array" + ) + + @staticmethod + def _audio_duration(audio): + if isinstance(audio, np.ndarray): + return float(audio.size) / 16000.0 + try: + import soundfile as sf + + if isinstance(audio, bytes): + return float(sf.info(io.BytesIO(audio)).duration) + if isinstance(audio, os.PathLike): + audio = os.fspath(audio) + if isinstance(audio, str) and os.path.isfile(audio): + return float(sf.info(audio).duration) + except (RuntimeError, TypeError, ValueError): + pass + return 0.0 diff --git a/tests/test_moss_transcribe_diarize_docs.py b/tests/test_moss_transcribe_diarize_docs.py index 95581ee1f..91ee0ff49 100644 --- a/tests/test_moss_transcribe_diarize_docs.py +++ b/tests/test_moss_transcribe_diarize_docs.py @@ -29,12 +29,16 @@ def test_moss_guides_pin_upstream_and_separate_serving_contracts(guide: Path) -> "vllm[audio]", "0.23.1rc1.dev949+g68b4a1d58", "dbb32bcfed2e8226bedf64248a9f4a44685b293a4696d18fb4cfa701b04db912", + "43dccc068506439cb633b382b6b98185baa837363d08cc5f7152ca89b0fdc3c8", "S01 -> S02 -> S01", "/v1/audio/transcriptions", "response_format=json", "response_format=verbose_json", "[S01]", "Apache-2.0", + "sentence_info", + "raw_text", + "vad_model", ): assert marker in text, f"{guide.name} is missing {marker}" diff --git a/tests/test_moss_transcribe_diarize_model.py b/tests/test_moss_transcribe_diarize_model.py new file mode 100644 index 000000000..938a6fa93 --- /dev/null +++ b/tests/test_moss_transcribe_diarize_model.py @@ -0,0 +1,143 @@ +import io +import unittest +from unittest.mock import MagicMock, patch + +import numpy as np + +from funasr.models.moss_transcribe_diarize.model import ( + MossTranscribeDiarize, + _parse_transcript, + _result_from_transcript, +) + + +class MossTranscriptParserTest(unittest.TestCase): + def test_parses_speaker_segments_and_preserves_numeric_brackets_in_text(self): + transcript = "noise [0.48][S01]Welcome [2026][1.66]" "[12.26][S02]Ready[13.81]" + + self.assertEqual( + _parse_transcript(transcript), + [ + (0.48, 1.66, "S01", "Welcome [2026]"), + (12.26, 13.81, "S02", "Ready"), + ], + ) + + def test_normalizes_moss_output_to_funasr_result_contract(self): + transcript = "[0.48][S01]Welcome[1.66][12.26][S02]Ready[13.81]" + + result = _result_from_transcript("meeting", transcript) + + self.assertEqual(result["key"], "meeting") + self.assertEqual(result["text"], "Welcome Ready") + self.assertEqual(result["raw_text"], transcript) + self.assertEqual(result["timestamp"], [[480, 1660], [12260, 13810]]) + self.assertEqual( + result["sentence_info"], + [ + { + "start": 480, + "end": 1660, + "text": "Welcome", + "sentence": "Welcome", + "spk": "S01", + "timestamp": [[480, 1660]], + }, + { + "start": 12260, + "end": 13810, + "text": "Ready", + "sentence": "Ready", + "spk": "S02", + "timestamp": [[12260, 13810]], + }, + ], + ) + + def test_keeps_unparsed_model_text_visible(self): + result = _result_from_transcript("sample", "plain transcript") + + self.assertEqual(result["text"], "plain transcript") + self.assertEqual(result["raw_text"], "plain transcript") + self.assertEqual(result["timestamp"], []) + self.assertEqual(result["sentence_info"], []) + + +class MossVllmBackendTest(unittest.TestCase): + def test_auto_model_vllm_path_bypasses_weight_download(self): + from funasr.auto.auto_model import AutoModel + + with patch( + "funasr.auto.auto_model.download_model", + side_effect=AssertionError( + "vLLM client mode must not download model weights" + ), + ): + auto_model = AutoModel( + model="OpenMOSS-Team/MOSS-Transcribe-Diarize", + backend="vllm", + vllm_base_url="http://vllm.test:8000/v1", + device="cpu", + disable_update=True, + ) + + self.assertIsInstance(auto_model.model, MossTranscribeDiarize) + + def test_rejects_external_vad_that_would_break_global_speaker_identity(self): + with self.assertRaisesRegex(ValueError, "omit vad_model and spk_model"): + MossTranscribeDiarize( + backend="vllm", + vllm_base_url="http://vllm.test:8000/v1", + vad_model="fsmn-vad", + ) + + def test_posts_openai_compatible_transcription_request_without_loading_hf_model( + self, + ): + session = MagicMock() + response = MagicMock() + response.json.return_value = {"text": "[0][S01]hello[1.5]"} + session.post.return_value = response + + model = MossTranscribeDiarize( + backend="vllm", + vllm_base_url="http://vllm.test:8000/v1", + vllm_model="OpenMOSS-Team/MOSS-Transcribe-Diarize", + vllm_api_key="secret", + http_session=session, + ) + results, metadata = model.inference( + [np.zeros(1600, dtype=np.float32)], + key=["sample"], + prompt="transcribe and diarize", + max_new_tokens=128, + ) + + self.assertEqual(results[0]["text"], "hello") + self.assertEqual(results[0]["sentence_info"][0]["spk"], "S01") + self.assertAlmostEqual(metadata["batch_data_time"], 0.1) + request = session.post.call_args + self.assertEqual( + request.args[0], + "http://vllm.test:8000/v1/audio/transcriptions", + ) + self.assertEqual( + request.kwargs["data"], + { + "model": "OpenMOSS-Team/MOSS-Transcribe-Diarize", + "response_format": "json", + "temperature": "0", + "prompt": "transcribe and diarize", + "max_completion_tokens": "128", + }, + ) + self.assertEqual(request.kwargs["headers"], {"Authorization": "Bearer secret"}) + filename, payload, content_type = request.kwargs["files"]["file"] + self.assertEqual(filename, "audio.wav") + self.assertIsInstance(payload, io.BytesIO) + self.assertEqual(content_type, "audio/wav") + response.raise_for_status.assert_called_once_with() + + +if __name__ == "__main__": + unittest.main()