mirror of
https://github.com/modelscope/FunASR.git
synced 2026-10-12 11:20:03 +00:00
feat: add SPK support to all protocols (OpenAI API + WebSocket)
- OpenAI API: added spk param (FunASR extension) - WebSocket: SPK clustering on STOP (full re-cluster all sentences) - All 3 protocols tested: HTTP REST ✓, OpenAI API ✓, WebSocket ✓ - docs: merge VAD tables into one, remove "no SPK" notes
This commit is contained in:
+13
-25
@@ -226,8 +226,6 @@ const result = await resp.json();
|
||||
|
||||
兼容 OpenAI Whisper API 标准,可直接用 OpenAI SDK 接入。
|
||||
|
||||
> 注意:OpenAI 标准不支持 SPK 参数。如需说话人分离,请用 HTTP REST 接口。
|
||||
|
||||
**请求**:`multipart/form-data`
|
||||
|
||||
| 参数 | 类型 | 默认值 | 说明 |
|
||||
@@ -237,6 +235,7 @@ const result = await resp.json();
|
||||
| `language` | string | None | 语种 |
|
||||
| `response_format` | string | "json" | "json" / "text" / "verbose_json" |
|
||||
| `timestamp_granularities` | string | "word" | "word" / "segment" |
|
||||
| `spk` | bool | false | 说话人分离(FunASR 扩展字段) |
|
||||
|
||||
**响应**(`verbose_json`):
|
||||
|
||||
@@ -278,9 +277,8 @@ curl -X POST http://localhost:8899/v1/audio/transcriptions \
|
||||
|
||||
### 3.5 协议三:WebSocket — `ws://host:port/ws`
|
||||
|
||||
> 注意:WebSocket 接口当前不支持 SPK。如需说话人分离,请用 HTTP REST 接口。
|
||||
|
||||
离线服务的 WebSocket 接口,发送完整音频后获取结果(非流式)。
|
||||
离线服务的 WebSocket 接口,发送完整音频后获取结果。STOP 时自动进行说话人聚类,结果中包含 `spk` 字段。
|
||||
|
||||
**客户端 → 服务端**:
|
||||
|
||||
@@ -451,28 +449,18 @@ asyncio.run(stream("audio.wav"))
|
||||
|
||||
fsmn-vad 默认启用动态静音阈值。离线和流式使用不同配置。
|
||||
|
||||
### 离线配置(保留长段 ≤60s)
|
||||
| 累积时长 | 离线(保留长段 ≤60s) | 流式(平衡延迟) |
|
||||
|---------|-------------------|----------------|
|
||||
| ≤ 5s | 2000ms | 2000ms |
|
||||
| 5-10s | 2000ms | 1500ms |
|
||||
| 10-15s | 1000ms | 1000ms |
|
||||
| 15-20s | 1000ms | 800ms |
|
||||
| 20-30s | 800ms | 800ms |
|
||||
| 30-45s | 600ms | 400ms |
|
||||
| 45-60s | 200-400ms | 100ms |
|
||||
| > 60s | 100ms | 100ms |
|
||||
|
||||
| 累积时长 | 静音阈值 |
|
||||
|---------|---------|
|
||||
| ≤ 10s | 2000ms |
|
||||
| 10-20s | 1000ms |
|
||||
| 20-30s | 800ms |
|
||||
| 30-40s | 600ms |
|
||||
| 40-50s | 400ms |
|
||||
| 50-60s | 200ms |
|
||||
| > 60s | 100ms |
|
||||
|
||||
### 流式配置(平衡延迟)
|
||||
|
||||
| 累积时长 | 静音阈值 |
|
||||
|---------|---------|
|
||||
| ≤ 5s | 2000ms |
|
||||
| 5-10s | 1500ms |
|
||||
| 10-15s | 1000ms |
|
||||
| 15-30s | 800ms |
|
||||
| 30-45s | 400ms |
|
||||
| > 45s | 100ms |
|
||||
离线倾向保留长段减少边界损失;流式更快收紧以降低延迟。
|
||||
|
||||
### 自定义
|
||||
|
||||
|
||||
@@ -210,13 +210,14 @@ async def openai_transcriptions(
|
||||
language: str = Form(default=None),
|
||||
response_format: str = Form(default="json"),
|
||||
timestamp_granularities: str = Form(default="word"),
|
||||
spk: bool = Form(default=False),
|
||||
):
|
||||
"""OpenAI Whisper-compatible transcription API."""
|
||||
"""OpenAI Whisper-compatible transcription API (extended with spk support)."""
|
||||
content = await file.read()
|
||||
audio_data, sr = sf.read(io.BytesIO(content))
|
||||
|
||||
use_ts = "word" in timestamp_granularities or "segment" in timestamp_granularities
|
||||
result = process_audio(audio_data, sr=sr, language=language, use_timestamp=use_ts)
|
||||
result = process_audio(audio_data, sr=sr, language=language, use_spk=spk, use_timestamp=use_ts)
|
||||
|
||||
if response_format == "text":
|
||||
return JSONResponse(content=result["text"])
|
||||
@@ -304,6 +305,34 @@ async def websocket_endpoint(websocket: WebSocket):
|
||||
"text": res[0]["text"], "start": start_ms, "end": end_ms
|
||||
})
|
||||
|
||||
# SPK: run full clustering on all sentences
|
||||
if locked_sentences and _spk_model is not None:
|
||||
try:
|
||||
from funasr.models.campplus.utils import sv_chunk, postprocess, distribute_spk
|
||||
from funasr.models.campplus.cluster_backend import ClusterBackend
|
||||
vad_segs = [[s["start"]/1000, s["end"]/1000,
|
||||
audio_buffer[int(s["start"]*16):int(s["end"]*16)]]
|
||||
for s in locked_sentences]
|
||||
chunks = sv_chunk(vad_segs)
|
||||
if chunks:
|
||||
speech_list = [ch[2] for ch in chunks]
|
||||
spk_res = _spk_model.generate(input=speech_list, cache={}, is_final=True)
|
||||
import torch as _torch
|
||||
embs = _torch.cat([r["spk_embedding"] for r in spk_res], dim=0)
|
||||
cluster = ClusterBackend(merge_thr=0.78).to(_args.device)
|
||||
labels = cluster(embs.cpu(), oracle_num=None)
|
||||
if not isinstance(labels, np.ndarray):
|
||||
labels = np.array(labels)
|
||||
all_sorted = sorted(chunks, key=lambda x: x[0])
|
||||
sv_output = postprocess(all_sorted, None, labels, embs.cpu())
|
||||
spk_sents = [{"text": s["text"], "start": int(s["start"]), "end": int(s["end"])}
|
||||
for s in locked_sentences]
|
||||
distribute_spk(spk_sents, sv_output)
|
||||
for i, ss in enumerate(spk_sents):
|
||||
locked_sentences[i]["spk"] = ss.get("spk", 0)
|
||||
except Exception as e:
|
||||
logger.warning(f"SPK failed: {e}")
|
||||
|
||||
await websocket.send_json({
|
||||
"sentences": locked_sentences,
|
||||
"is_final": True,
|
||||
|
||||
Reference in New Issue
Block a user