diff --git a/docs/vllm_guide.md b/docs/vllm_guide.md index 334870d83..ff850a088 100644 --- a/docs/vllm_guide.md +++ b/docs/vllm_guide.md @@ -226,8 +226,6 @@ const result = await resp.json(); 兼容 OpenAI Whisper API 标准,可直接用 OpenAI SDK 接入。 -> 注意:OpenAI 标准不支持 SPK 参数。如需说话人分离,请用 HTTP REST 接口。 - **请求**:`multipart/form-data` | 参数 | 类型 | 默认值 | 说明 | @@ -237,6 +235,7 @@ const result = await resp.json(); | `language` | string | None | 语种 | | `response_format` | string | "json" | "json" / "text" / "verbose_json" | | `timestamp_granularities` | string | "word" | "word" / "segment" | +| `spk` | bool | false | 说话人分离(FunASR 扩展字段) | **响应**(`verbose_json`): @@ -278,9 +277,8 @@ curl -X POST http://localhost:8899/v1/audio/transcriptions \ ### 3.5 协议三:WebSocket — `ws://host:port/ws` -> 注意:WebSocket 接口当前不支持 SPK。如需说话人分离,请用 HTTP REST 接口。 -离线服务的 WebSocket 接口,发送完整音频后获取结果(非流式)。 +离线服务的 WebSocket 接口,发送完整音频后获取结果。STOP 时自动进行说话人聚类,结果中包含 `spk` 字段。 **客户端 → 服务端**: @@ -451,28 +449,18 @@ asyncio.run(stream("audio.wav")) fsmn-vad 默认启用动态静音阈值。离线和流式使用不同配置。 -### 离线配置(保留长段 ≤60s) +| 累积时长 | 离线(保留长段 ≤60s) | 流式(平衡延迟) | +|---------|-------------------|----------------| +| ≤ 5s | 2000ms | 2000ms | +| 5-10s | 2000ms | 1500ms | +| 10-15s | 1000ms | 1000ms | +| 15-20s | 1000ms | 800ms | +| 20-30s | 800ms | 800ms | +| 30-45s | 600ms | 400ms | +| 45-60s | 200-400ms | 100ms | +| > 60s | 100ms | 100ms | -| 累积时长 | 静音阈值 | -|---------|---------| -| ≤ 10s | 2000ms | -| 10-20s | 1000ms | -| 20-30s | 800ms | -| 30-40s | 600ms | -| 40-50s | 400ms | -| 50-60s | 200ms | -| > 60s | 100ms | - -### 流式配置(平衡延迟) - -| 累积时长 | 静音阈值 | -|---------|---------| -| ≤ 5s | 2000ms | -| 5-10s | 1500ms | -| 10-15s | 1000ms | -| 15-30s | 800ms | -| 30-45s | 400ms | -| > 45s | 100ms | +离线倾向保留长段减少边界损失;流式更快收紧以降低延迟。 ### 自定义 diff --git a/examples/industrial_data_pretraining/fun_asr_nano/serve_vllm.py b/examples/industrial_data_pretraining/fun_asr_nano/serve_vllm.py index ef54c69eb..0d9e9a033 100644 --- a/examples/industrial_data_pretraining/fun_asr_nano/serve_vllm.py +++ b/examples/industrial_data_pretraining/fun_asr_nano/serve_vllm.py @@ -210,13 +210,14 @@ async def openai_transcriptions( language: str = Form(default=None), response_format: str = Form(default="json"), timestamp_granularities: str = Form(default="word"), + spk: bool = Form(default=False), ): - """OpenAI Whisper-compatible transcription API.""" + """OpenAI Whisper-compatible transcription API (extended with spk support).""" content = await file.read() audio_data, sr = sf.read(io.BytesIO(content)) use_ts = "word" in timestamp_granularities or "segment" in timestamp_granularities - result = process_audio(audio_data, sr=sr, language=language, use_timestamp=use_ts) + result = process_audio(audio_data, sr=sr, language=language, use_spk=spk, use_timestamp=use_ts) if response_format == "text": return JSONResponse(content=result["text"]) @@ -304,6 +305,34 @@ async def websocket_endpoint(websocket: WebSocket): "text": res[0]["text"], "start": start_ms, "end": end_ms }) + # SPK: run full clustering on all sentences + if locked_sentences and _spk_model is not None: + try: + from funasr.models.campplus.utils import sv_chunk, postprocess, distribute_spk + from funasr.models.campplus.cluster_backend import ClusterBackend + vad_segs = [[s["start"]/1000, s["end"]/1000, + audio_buffer[int(s["start"]*16):int(s["end"]*16)]] + for s in locked_sentences] + chunks = sv_chunk(vad_segs) + if chunks: + speech_list = [ch[2] for ch in chunks] + spk_res = _spk_model.generate(input=speech_list, cache={}, is_final=True) + import torch as _torch + embs = _torch.cat([r["spk_embedding"] for r in spk_res], dim=0) + cluster = ClusterBackend(merge_thr=0.78).to(_args.device) + labels = cluster(embs.cpu(), oracle_num=None) + if not isinstance(labels, np.ndarray): + labels = np.array(labels) + all_sorted = sorted(chunks, key=lambda x: x[0]) + sv_output = postprocess(all_sorted, None, labels, embs.cpu()) + spk_sents = [{"text": s["text"], "start": int(s["start"]), "end": int(s["end"])} + for s in locked_sentences] + distribute_spk(spk_sents, sv_output) + for i, ss in enumerate(spk_sents): + locked_sentences[i]["spk"] = ss.get("spk", 0) + except Exception as e: + logger.warning(f"SPK failed: {e}") + await websocket.send_json({ "sentences": locked_sentences, "is_final": True,