Files
FunASR/funasr/bin/_server_app.py
T
2026-09-02 03:20:13 +00:00

585 lines
22 KiB
Python

"""FunASR Server — unified vLLM-based inference service.
Provides OpenAI-compatible API (/v1/audio/transcriptions) and REST API (/asr).
Uses vLLM for Fun-ASR-Nano (GPU) or AutoModel for supported models, including
MOSS-Transcribe-Diarize's joint transcription and anonymous speaker labels.
"""
import io
import os
import re
import time
import logging
import tempfile
from pathlib import Path
from typing import Iterable, Optional
import numpy as np
import soundfile as sf
try:
from fastapi import FastAPI, UploadFile, File, Form, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse
except ImportError:
raise ImportError(
"funasr-server requires additional packages. Install with: pip install vllm fastapi uvicorn python-multipart"
)
logger = logging.getLogger("funasr.server")
PACKAGE_VERSION = (Path(__file__).resolve().parents[1] / "version.txt").read_text().strip()
_LANGUAGE_TAG_RE = re.compile(r"<\|(zh|en|yue|ja|ko)\|>")
MOSS_MODEL_REVISION = "e8681d68e7042738ffca8ac8212bc8fcb1131ab8"
NATIVE_DIARIZATION_MODELS = {"moss-transcribe-diarize"}
N8N_OPENAI_MODEL_ALIAS = "whisper-1"
def extract_language_from_asr_text(text):
"""Extract a SenseVoice language code before special tokens are removed."""
if not isinstance(text, str):
return None
match = _LANGUAGE_TAG_RE.search(text)
return match.group(1) if match else None
def resolve_transcription_language(requested_language, result):
"""Prefer the caller's language hint, then backend detection, else unknown."""
if requested_language and requested_language.strip().lower() != "auto":
return requested_language
detected_language = result.get("language")
if isinstance(detected_language, str) and detected_language:
return detected_language
return "unknown"
def resolve_openai_transcription_model(requested_model, default_model):
"""Map n8n's fixed OpenAI transcription model to the server default."""
if requested_model == N8N_OPENAI_MODEL_ALIAS:
return default_model
return requested_model
def _split_text_for_openai_segments(text: str, max_chars: int = 80):
"""Split unsegmented ASR text into readable OpenAI-compatible cues."""
text = text.strip()
if not text:
return []
words = text.split()
if not words:
return [text[i : i + max_chars] for i in range(0, len(text), max_chars)]
parts = []
current = []
current_len = 0
for word in words:
next_len = len(word) if not current else current_len + 1 + len(word)
if current and next_len > max_chars:
parts.append(" ".join(current))
current = []
current_len = 0
current.append(word)
current_len = len(word) if current_len == 0 else current_len + 1 + len(word)
if word[-1:] in ".!?;:" and current_len >= max_chars // 2:
parts.append(" ".join(current))
current = []
current_len = 0
if current:
parts.append(" ".join(current))
if any(len(part) > max_chars for part in parts):
return [text[i : i + max_chars] for i in range(0, len(text), max_chars)]
return parts
def build_openai_fallback_segments(text: str, duration: float, max_chars: int = 80):
"""Build coarse timestamped segments when a backend returns text only."""
parts = _split_text_for_openai_segments(text, max_chars=max_chars)
if not parts:
return []
if len(parts) == 1 or duration <= 0:
return [{"start": 0.0, "end": max(float(duration), 0.0), "text": parts[0]}]
total_chars = sum(len(part) for part in parts)
if total_chars <= 0:
return [{"start": 0.0, "end": float(duration), "text": text.strip()}]
segments = []
consumed = 0
previous_end = 0.0
for i, part in enumerate(parts):
consumed += len(part)
end = float(duration) if i == len(parts) - 1 else float(duration) * consumed / total_chars
end = max(end, previous_end)
segments.append({"start": round(previous_end, 3), "end": round(end, 3), "text": part})
previous_end = end
return segments
def prepare_audio_for_inference(audio_data, sr, target_sr=16000):
"""Return mono float32 audio at target_sr for ASR inference."""
audio_data = np.asarray(audio_data)
if audio_data.ndim > 1:
channel_axis = -1 if audio_data.shape[-1] <= audio_data.shape[0] else 0
audio_data = audio_data.mean(axis=channel_axis)
if sr != target_sr:
import librosa
audio_data = librosa.resample(audio_data, orig_sr=sr, target_sr=target_sr)
sr = target_sr
return audio_data.astype(np.float32), sr
def attach_speaker_labels(audio_data, sr, segments, speaker_model, device):
"""Run speaker diarization once and attach labels to timestamped segments."""
if not segments:
return segments
import torch
from funasr.models.campplus.cluster_backend import ClusterBackend
from funasr.models.campplus.utils import distribute_spk, postprocess, sv_chunk
audio_data, sr = prepare_audio_for_inference(audio_data, sr)
duration = len(audio_data) / sr
diarization_inputs = []
segment_indexes = []
for index, segment in enumerate(segments):
start = max(float(segment.get("start", 0.0)), 0.0)
end = min(max(float(segment.get("end", start)), start), duration)
start_sample = int(start * sr)
end_sample = int(end * sr)
if end_sample <= start_sample:
continue
diarization_inputs.append([start, end, audio_data[start_sample:end_sample]])
segment_indexes.append(index)
chunks = sv_chunk(diarization_inputs, fs=sr)
if not chunks:
return segments
speaker_results = speaker_model.generate(
input=[chunk[2] for chunk in chunks], cache={}, is_final=True
)
embeddings = torch.cat(
[speaker_result["spk_embedding"] for speaker_result in speaker_results], dim=0
)
labels = ClusterBackend(merge_thr=0.78).to(device)(embeddings.cpu(), oracle_num=None)
if not isinstance(labels, np.ndarray):
labels = np.asarray(labels)
speaker_timeline = postprocess(
sorted(chunks, key=lambda chunk: chunk[0]),
None,
labels,
embeddings.detach().cpu().numpy(),
)
sentences = [
{
"text": segments[index]["text"],
"start": int(float(segments[index]["start"]) * 1000),
"end": int(float(segments[index]["end"]) * 1000),
}
for index in segment_indexes
]
distribute_spk(sentences, speaker_timeline)
for index, sentence in zip(segment_indexes, sentences):
speaker = sentence.get("spk")
if speaker is not None:
segments[index]["speaker"] = f"SPK{speaker}"
return segments
def build_openai_verbose_json(result, requested_language=None):
"""Build OpenAI-compatible verbose JSON while preserving FunASR extensions."""
segments = []
for index, segment in enumerate(result.get("segments", [])):
item = {
"id": index,
"start": segment["start"],
"end": segment["end"],
"text": segment["text"],
"words": segment.get("words", []),
}
if segment.get("speaker") is not None:
item["speaker"] = segment["speaker"]
segments.append(item)
return {
"task": "transcribe",
"language": resolve_transcription_language(requested_language, result),
"duration": result.get("duration", 0),
"text": result["text"],
"segments": segments,
}
def create_app(
device: str = "cuda",
preload_model: str = "auto",
model_path: str = None,
hub: str = "ms",
spk_model: str = "cam++",
cors_origins: Optional[Iterable[str]] = None,
) -> FastAPI:
if preload_model == "auto":
preload_model = "fun-asr-nano" if device.startswith("cuda") else "sensevoice"
app = FastAPI(title="FunASR Server", version=PACKAGE_VERSION)
app.state.device = device
app.state.engine = None
app.state.vad_model = None
app.state.spk_model = None
app.state.spk_model_name = spk_model
app.state.fallback_models = {}
app.state.model_path = model_path
app.state.hub = hub
app.state.openai_transcription_model = "custom" if model_path else preload_model
normalized_origins = []
for origin in cors_origins or []:
origin = origin.strip()
if origin and origin not in normalized_origins:
normalized_origins.append(origin)
if normalized_origins:
app.add_middleware(
CORSMiddleware,
allow_origins=normalized_origins,
allow_credentials=False,
allow_methods=["GET", "POST", "OPTIONS"],
allow_headers=["Authorization", "Content-Type"],
)
# Non-LLM model configs (use AutoModel, no vLLM)
FALLBACK_CONFIGS = {
"sensevoice": {
"model": "iic/SenseVoiceSmall",
"vad_model": "fsmn-vad",
"vad_kwargs": {"max_single_segment_time": 30000},
},
"paraformer": {
"model": "paraformer-zh",
"vad_model": "fsmn-vad",
"punc_model": "ct-punc",
},
"moss-transcribe-diarize": {
"model": "OpenMOSS-Team/MOSS-Transcribe-Diarize",
"model_revision": MOSS_MODEL_REVISION,
"hub": "hf",
"backend": "hf",
"trust_remote_code": True,
},
}
def _load_spk_model():
"""Lazily load diarization only when a request opts in with spk=true."""
if app.state.spk_model is not None:
return app.state.spk_model
if not app.state.spk_model_name:
raise HTTPException(400, "Speaker diarization is disabled; configure --spk-model")
from funasr import AutoModel
logger.info(f"Loading speaker model: {app.state.spk_model_name}")
app.state.spk_model = AutoModel(
model=app.state.spk_model_name,
device=device,
disable_update=True,
)
logger.info("Speaker model ready.")
return app.state.spk_model
def _load_vllm_engine():
"""Load Fun-ASR-Nano vLLM engine. Falls back to AutoModel if vLLM unavailable."""
if app.state.engine is not None or "fun-asr-nano" in app.state.fallback_models:
return
try:
from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM
from funasr import AutoModel as _AutoModel
logger.info("Loading Fun-ASR-Nano vLLM engine...")
t0 = time.time()
# Use custom model_path if provided, otherwise default. In both
# cases, honor the server-level hub selection.
vllm_model = app.state.model_path if app.state.model_path else "FunAudioLLM/Fun-ASR-Nano-2512"
vllm_hub = app.state.hub
engine = FunASRNanoVLLM.from_pretrained(
model=vllm_model,
hub=vllm_hub,
device=device,
dtype="bf16",
max_model_len=4096,
gpu_memory_utilization=0.5,
)
logger.info(f"vLLM engine ready in {time.time()-t0:.1f}s")
logger.info("Loading VAD model...")
vad_model = _AutoModel(model="fsmn-vad", device=device, disable_update=True)
app.state.engine = engine
app.state.vad_model = vad_model
app.state.use_vllm = True
logger.info("VAD ready.")
except Exception as e:
logger.warning(f"vLLM failed ({e}), falling back to AutoModel for fun-asr-nano")
app.state.use_vllm = False
from funasr import AutoModel
cfg = {
"model": app.state.model_path if app.state.model_path else "FunAudioLLM/Fun-ASR-Nano-2512",
"hub": app.state.hub,
"trust_remote_code": True,
"vad_model": "fsmn-vad",
"vad_kwargs": {"max_single_segment_time": 30000},
"device": device,
"disable_update": True,
}
app.state.fallback_models["fun-asr-nano"] = AutoModel(**cfg)
logger.info(f"Fallback AutoModel loaded for fun-asr-nano with model={cfg['model']}, hub={cfg['hub']}.")
def _load_fallback(name: str):
"""Load non-LLM model via AutoModel."""
if name in app.state.fallback_models:
return app.state.fallback_models[name]
if name not in FALLBACK_CONFIGS and not app.state.model_path:
return None
from funasr import AutoModel
cfg = FALLBACK_CONFIGS.get(name, {}).copy()
# Override with custom model_path and hub if provided
if app.state.model_path:
cfg["model"] = app.state.model_path
cfg["hub"] = app.state.hub
elif app.state.hub and "hub" not in cfg:
cfg["hub"] = app.state.hub
cfg["device"] = device
cfg["disable_update"] = True
logger.info(f"Loading fallback model '{name}' with model={cfg['model']}, hub={cfg['hub']}...")
model = AutoModel(**cfg)
app.state.fallback_models[name] = model
return model
def _process_vllm(audio_data, sr, language=None, hotwords=None, use_spk=False):
"""Process audio with vLLM engine (Fun-ASR-Nano)."""
audio_data, sr = prepare_audio_for_inference(audio_data, sr)
# VAD
vad_res = app.state.vad_model.generate(input=audio_data, fs=sr)
segments = vad_res[0]["value"] if vad_res and vad_res[0].get("value") else [[0, int(len(audio_data)*1000/sr)]]
seg_audios = []
seg_times = []
for seg in segments:
s0 = int(seg[0] * sr / 1000)
s1 = int(seg[1] * sr / 1000)
seg_audio = audio_data[s0:s1]
if len(seg_audio) > sr * 0.3:
seg_audios.append(seg_audio)
seg_times.append((seg[0], seg[1]))
if not seg_audios:
return {"text": "", "segments": [], "duration": len(audio_data)/sr}
# repetition_penalty is left at the neutral 1.0: the Fun-ASR-Nano vLLM
# engine runs in prompt-embeds mode, where any other value crashes the
# CUDA kernel (see issue #2948 and fun_asr_nano.vllm_utils).
gen_kwargs = {"max_new_tokens": 500, "repetition_penalty": 1.0}
if language:
gen_kwargs["language"] = language
if hotwords:
gen_kwargs["hotwords"] = hotwords
results = app.state.engine.generate(inputs=seg_audios, **gen_kwargs)
output_segments = []
full_text_parts = []
for r, (start_ms, end_ms) in zip(results, seg_times):
text = r["text"]
seg_info = {"text": text, "start": start_ms/1000, "end": end_ms/1000}
if "timestamps" in r:
offset = start_ms / 1000
seg_info["words"] = [
{"word": ts["token"], "start": ts["start_time"]+offset, "end": ts["end_time"]+offset}
for ts in r["timestamps"]
]
output_segments.append(seg_info)
full_text_parts.append(text)
if use_spk:
attach_speaker_labels(
audio_data,
sr,
output_segments,
_load_spk_model(),
device,
)
return {
"text": "".join(full_text_parts),
"segments": output_segments,
"duration": len(audio_data) / sr,
}
def _process_fallback(model_name, audio_path, language=None, use_spk=False):
"""Process with non-LLM model (SenseVoice/Paraformer)."""
model = _load_fallback(model_name)
try:
duration = float(sf.info(audio_path).duration)
except Exception:
duration = 0.0
kwargs = {"input": audio_path, "batch_size": 1}
if language:
kwargs["language"] = language
result = model.generate(**kwargs)
raw_text = result[0]["text"]
detected_language = extract_language_from_asr_text(raw_text)
text = re.sub(r'<\|[^|]*\|>', '', raw_text).strip()
segments = []
if "sentence_info" in result[0]:
for s in result[0]["sentence_info"]:
segment = {
"start": s.get("start", 0)/1000,
"end": s.get("end", 0)/1000,
"text": re.sub(
r'<\|[^|]*\|>', '', s.get("text") or s.get("sentence", "")
).strip(),
}
if s.get("spk") is not None:
segment["speaker"] = s["spk"]
segments.append(segment)
if not segments and text:
segments = build_openai_fallback_segments(text, duration)
if use_spk and model_name not in NATIVE_DIARIZATION_MODELS and segments:
audio_data, sr = sf.read(audio_path)
attach_speaker_labels(
audio_data,
sr,
segments,
_load_spk_model(),
device,
)
return {
"text": text,
"segments": segments,
"duration": duration,
"language": detected_language,
}
# Pre-load
if app.state.model_path:
# When custom model_path is provided, use it as the model name for loading
logger.info(f"Loading custom model: {app.state.model_path} (hub: {app.state.hub})")
_load_fallback("custom")
elif preload_model == "fun-asr-nano":
_load_vllm_engine()
else:
_load_fallback(preload_model)
@app.post("/v1/audio/transcriptions")
async def transcribe(
file: UploadFile = File(...),
model: str = Form(default="fun-asr-nano"),
language: Optional[str] = Form(default=None),
response_format: Optional[str] = Form(default="json"),
spk: bool = Form(default=False),
):
content = await file.read()
t0 = time.perf_counter()
model = resolve_openai_transcription_model(
model, app.state.openai_transcription_model
)
if model == "fun-asr-nano":
_load_vllm_engine()
if app.state.use_vllm:
audio_data, sr = sf.read(io.BytesIO(content))
result = _process_vllm(audio_data, sr, language=language, use_spk=spk)
else:
suffix = os.path.splitext(file.filename)[1] if file.filename else ".wav"
with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
tmp.write(content)
tmp_path = tmp.name
try:
result = _process_fallback(
"fun-asr-nano", tmp_path, language=language, use_spk=spk
)
finally:
os.unlink(tmp_path)
elif model in FALLBACK_CONFIGS or model == "custom":
suffix = os.path.splitext(file.filename)[1] if file.filename else ".wav"
with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
tmp.write(content)
tmp_path = tmp.name
try:
result = _process_fallback(
model, tmp_path, language=language, use_spk=spk
)
finally:
os.unlink(tmp_path)
else:
available = ["fun-asr-nano", "custom"] + list(FALLBACK_CONFIGS.keys())
raise HTTPException(400, f"Unknown model '{model}'. Available: {', '.join(available)}")
t1 = time.perf_counter()
if response_format == "verbose_json":
return JSONResponse(build_openai_verbose_json(result, requested_language=language))
elif response_format == "text":
return JSONResponse(result["text"])
else:
return JSONResponse({"text": result["text"]})
@app.post("/asr")
async def asr_endpoint(
file: UploadFile = File(...),
language: Optional[str] = Form(default=None),
hotwords: str = Form(default=""),
spk: bool = Form(default=False),
):
"""Full-featured ASR endpoint with timestamps and speaker diarization."""
content = await file.read()
_load_vllm_engine()
hw_list = [w.strip() for w in hotwords.split(",") if w.strip()] if hotwords else None
t0 = time.perf_counter()
if app.state.use_vllm:
audio_data, sr = sf.read(io.BytesIO(content))
result = _process_vllm(audio_data, sr, language=language, hotwords=hw_list, use_spk=spk)
else:
suffix = ".wav"
with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:
tmp.write(content)
tmp_path = tmp.name
try:
result = _process_fallback(
"fun-asr-nano", tmp_path, language=language, use_spk=spk
)
finally:
os.unlink(tmp_path)
t1 = time.perf_counter()
result["processing_time"] = round(t1 - t0, 3)
result["rtf"] = round((t1 - t0) / result["duration"], 4) if result.get("duration", 0) > 0 else 0
return JSONResponse(result)
@app.get("/v1/models")
async def list_models():
all_models = ["fun-asr-nano"] + list(FALLBACK_CONFIGS.keys())
if app.state.model_path:
all_models.append("custom")
return JSONResponse({"object": "list", "data": [{"id": n, "object": "model"} for n in all_models]})
@app.get("/health")
async def health():
loaded = []
if app.state.engine is not None:
loaded.append("fun-asr-nano (vLLM)")
loaded.extend(app.state.fallback_models.keys())
return {"status": "ok", "device": device, "models_loaded": loaded}
return app