From b798e2679f9b801fd0175a44a95ad12c0e7bc491 Mon Sep 17 00:00:00 2001 From: zhifu gao Date: Wed, 22 Jul 2026 18:16:22 +0800 Subject: [PATCH] feat(runtime): add GGUF HTTP transcription wrapper Expose the llama.cpp/GGUF command-line binaries through a lightweight OpenAI-compatible transcription endpoint. --- runtime/llama.cpp/README.md | 37 +++ .../llama.cpp/server/funasr_gguf_server.py | 230 ++++++++++++++++++ runtime/llama.cpp/tests/test_gguf_server.py | 100 ++++++++ .../llama.cpp/tests/test_release_workflow.py | 9 + 4 files changed, 376 insertions(+) create mode 100644 runtime/llama.cpp/server/funasr_gguf_server.py create mode 100644 runtime/llama.cpp/tests/test_gguf_server.py diff --git a/runtime/llama.cpp/README.md b/runtime/llama.cpp/README.md index f38460507..6f7cd62e2 100644 --- a/runtime/llama.cpp/README.md +++ b/runtime/llama.cpp/README.md @@ -107,6 +107,43 @@ The shared **FSMN-VAD** front end builds the same way (`funasr-vad/` + `funasr-c target `llama-funasr-vad`); export weights with `export_vad_gguf.py`. Pass `--vad fsmn-vad.gguf` to any of the three tools for built-in long-audio segmentation. +## Lightweight HTTP server + +The GGUF binaries are command-line tools first. For local apps that expect an +HTTP transcription endpoint, `server/funasr_gguf_server.py` wraps an existing +binary and exposes an OpenAI-compatible `POST /v1/audio/transcriptions` route. +It uses only the Python standard library and still runs inference in the C++ +binary: + +```bash +python server/funasr_gguf_server.py \ + --host 127.0.0.1 --port 8000 \ + --binary ./build/bin/llama-funasr-sensevoice \ + --model ./gguf/sensevoice-small-q8.gguf \ + --vad ./gguf/fsmn-vad.gguf +``` + +Then send audio with the same shape used by OpenAI-compatible clients: + +```bash +curl http://127.0.0.1:8000/v1/audio/transcriptions \ + -F file=@sample.wav \ + -F model=funasr-gguf +``` + +Response: + +```json +{"text": "transcribed text"} +``` + +CUDA-enabled SenseVoice builds can be selected with `--backend cuda`. Extra +binary flags can be forwarded with repeated `--extra-arg`, for example +`--extra-arg --keep-tags`. This wrapper starts one subprocess per request, so it +is best for local tools, demos, and integration tests. For sustained production +traffic, use the Python `funasr-server` OpenAI-compatible service or build a +dedicated native server around the C++ runtime. + ## Validation Each model was validated against the FunASR PyTorch reference (encoder cosine ≈ 1.0; diff --git a/runtime/llama.cpp/server/funasr_gguf_server.py b/runtime/llama.cpp/server/funasr_gguf_server.py new file mode 100644 index 000000000..6c6463cf1 --- /dev/null +++ b/runtime/llama.cpp/server/funasr_gguf_server.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +"""OpenAI-compatible HTTP wrapper for FunASR llama.cpp / GGUF binaries. + +This server intentionally keeps inference in the existing C++ command-line +tools. It accepts a multipart audio upload, runs the configured GGUF binary, and +returns the transcript as a small OpenAI-compatible JSON response. +""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass, field +from email import policy +from email.parser import BytesParser +import json +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import os +from pathlib import Path +import subprocess +import tempfile +from typing import Iterable, Optional +from urllib.parse import urlparse + + +@dataclass(frozen=True) +class ServerConfig: + binary: str + model: str + vad: Optional[str] = None + backend: Optional[str] = None + extra_args: list[str] = field(default_factory=list) + work_dir: str = field(default_factory=tempfile.gettempdir) + timeout: float = 600.0 + + +def build_command(config: ServerConfig, audio_path: str) -> list[str]: + command = [config.binary, "-m", config.model, "-a", audio_path] + if config.vad: + command.extend(["--vad", config.vad]) + if config.backend: + command.extend(["--backend", config.backend]) + command.extend(config.extra_args) + return command + + +def extract_transcript(stdout: str) -> str: + return stdout.strip() + + +def transcribe_file(config: ServerConfig, audio_path: str) -> str: + completed = subprocess.run( + build_command(config, audio_path), + check=True, + capture_output=True, + text=True, + timeout=config.timeout, + ) + return extract_transcript(completed.stdout) + + +def _json_bytes(payload: dict) -> bytes: + return json.dumps(payload, ensure_ascii=False).encode("utf-8") + + +def _error_payload(message: str) -> bytes: + return _json_bytes({"error": {"message": message}}) + + +def _field_content_disposition(part) -> str: + return part.get("Content-Disposition", "") + + +def _is_file_field(part) -> bool: + disposition = _field_content_disposition(part) + return "form-data" in disposition and 'name="file"' in disposition + + +def parse_multipart_file(content_type: str, body: bytes) -> tuple[bytes, str]: + message = BytesParser(policy=policy.default).parsebytes( + ( + f"Content-Type: {content_type}\r\n" + "MIME-Version: 1.0\r\n" + "\r\n" + ).encode("utf-8") + + body + ) + if not message.is_multipart(): + raise ValueError("expected multipart/form-data") + + for part in message.iter_parts(): + if not _is_file_field(part): + continue + filename = part.get_filename() or "audio.wav" + payload = part.get_payload(decode=True) or b"" + if not payload: + raise ValueError("uploaded file is empty") + return payload, filename + raise ValueError("missing multipart field: file") + + +def _suffix_from_filename(filename: str) -> str: + suffix = Path(filename).suffix + if not suffix or len(suffix) > 16: + return ".wav" + return suffix + + +class FunASRGGUFHandler(BaseHTTPRequestHandler): + server_version = "FunASRGGUFServer/0.1" + + def log_message(self, fmt: str, *args) -> None: + print("%s - - [%s] %s" % (self.address_string(), self.log_date_time_string(), fmt % args)) + + @property + def config(self) -> ServerConfig: + return self.server.config # type: ignore[attr-defined] + + def _send_json(self, status: HTTPStatus, payload: bytes) -> None: + self.send_response(status) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def do_GET(self) -> None: + if urlparse(self.path).path == "/health": + self._send_json(HTTPStatus.OK, _json_bytes({"status": "ok"})) + return + self._send_json(HTTPStatus.NOT_FOUND, _error_payload("not found")) + + def do_POST(self) -> None: + if urlparse(self.path).path != "/v1/audio/transcriptions": + self._send_json(HTTPStatus.NOT_FOUND, _error_payload("not found")) + return + + try: + length = int(self.headers.get("Content-Length", "0")) + content_type = self.headers.get("Content-Type", "") + audio_bytes, filename = parse_multipart_file(content_type, self.rfile.read(length)) + except Exception as exc: + self._send_json(HTTPStatus.BAD_REQUEST, _error_payload(str(exc))) + return + + tmp_path = None + try: + os.makedirs(self.config.work_dir, exist_ok=True) + with tempfile.NamedTemporaryFile( + "wb", + suffix=_suffix_from_filename(filename), + dir=self.config.work_dir, + delete=False, + ) as tmp: + tmp.write(audio_bytes) + tmp_path = tmp.name + transcript = transcribe_file(self.config, tmp_path) + except subprocess.CalledProcessError as exc: + message = exc.stderr.strip() or exc.stdout.strip() or str(exc) + self._send_json(HTTPStatus.INTERNAL_SERVER_ERROR, _error_payload(message)) + return + except subprocess.TimeoutExpired: + self._send_json(HTTPStatus.GATEWAY_TIMEOUT, _error_payload("transcription timed out")) + return + except Exception as exc: + self._send_json(HTTPStatus.INTERNAL_SERVER_ERROR, _error_payload(str(exc))) + return + finally: + if tmp_path: + try: + os.unlink(tmp_path) + except FileNotFoundError: + pass + + self._send_json(HTTPStatus.OK, _json_bytes({"text": transcript})) + + +class FunASRHTTPServer(ThreadingHTTPServer): + def __init__(self, server_address, handler_class, config: ServerConfig): + super().__init__(server_address, handler_class) + self.config = config + + +def create_server(host: str, port: int, config: ServerConfig) -> FunASRHTTPServer: + return FunASRHTTPServer((host, port), FunASRGGUFHandler, config) + + +def _parse_extra_args(values: Optional[Iterable[str]]) -> list[str]: + args: list[str] = [] + for value in values or []: + args.extend(value.split()) + return args + + +def build_arg_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Serve a FunASR llama.cpp / GGUF binary over HTTP.") + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--port", type=int, default=8000) + parser.add_argument("--binary", required=True, help="Path to llama-funasr-sensevoice or llama-funasr-paraformer.") + parser.add_argument("--model", required=True, help="Path to the model GGUF passed as -m.") + parser.add_argument("--vad", help="Optional FSMN-VAD GGUF passed as --vad.") + parser.add_argument("--backend", choices=["cpu", "cuda"], help="Optional backend passed as --backend.") + parser.add_argument("--work-dir", default=tempfile.gettempdir(), help="Directory for temporary uploaded audio files.") + parser.add_argument("--timeout", type=float, default=600.0, help="Per-request subprocess timeout in seconds.") + parser.add_argument( + "--extra-arg", + action="append", + default=[], + help="Additional argument(s) forwarded to the GGUF binary. May be repeated.", + ) + return parser + + +def main() -> None: + args = build_arg_parser().parse_args() + config = ServerConfig( + binary=args.binary, + model=args.model, + vad=args.vad, + backend=args.backend, + extra_args=_parse_extra_args(args.extra_arg), + work_dir=args.work_dir, + timeout=args.timeout, + ) + httpd = create_server(args.host, args.port, config) + print(f"Serving FunASR GGUF transcription on http://{args.host}:{httpd.server_port}") + httpd.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/runtime/llama.cpp/tests/test_gguf_server.py b/runtime/llama.cpp/tests/test_gguf_server.py new file mode 100644 index 000000000..f24a5cf12 --- /dev/null +++ b/runtime/llama.cpp/tests/test_gguf_server.py @@ -0,0 +1,100 @@ +import importlib.util +import json +import os +import sys +import threading +import urllib.request +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SERVER = ROOT / "server" / "funasr_gguf_server.py" + + +def load_server_module(): + spec = importlib.util.spec_from_file_location("funasr_gguf_server", SERVER) + module = importlib.util.module_from_spec(spec) + sys.modules["funasr_gguf_server"] = module + spec.loader.exec_module(module) + return module + + +def test_build_command_adds_model_audio_vad_backend_and_extra_args(tmp_path): + server = load_server_module() + cfg = server.ServerConfig( + binary="/opt/funasr/llama-funasr-sensevoice", + model="/models/sensevoice.gguf", + vad="/models/fsmn-vad.gguf", + backend="cuda", + extra_args=["--keep-tags"], + work_dir=str(tmp_path), + ) + + command = server.build_command(cfg, "/tmp/request.wav") + + assert command == [ + "/opt/funasr/llama-funasr-sensevoice", + "-m", + "/models/sensevoice.gguf", + "-a", + "/tmp/request.wav", + "--vad", + "/models/fsmn-vad.gguf", + "--backend", + "cuda", + "--keep-tags", + ] + + +def test_transcription_endpoint_runs_binary_and_returns_openai_json(tmp_path): + server = load_server_module() + fake_binary = tmp_path / "fake_funasr.py" + captured_args = tmp_path / "args.json" + fake_binary.write_text( + "#!/usr/bin/env python3\n" + "import json, pathlib, sys\n" + f"pathlib.Path({str(captured_args)!r}).write_text(json.dumps(sys.argv[1:]))\n" + "print('hello from gguf')\n", + encoding="utf-8", + ) + fake_binary.chmod(0o755) + + cfg = server.ServerConfig( + binary=str(fake_binary), + model=str(tmp_path / "sensevoice.gguf"), + vad=str(tmp_path / "fsmn-vad.gguf"), + backend=None, + extra_args=["--ids"], + work_dir=str(tmp_path), + ) + httpd = server.create_server("127.0.0.1", 0, cfg) + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + try: + boundary = "----funasr-test-boundary" + body = ( + f"--{boundary}\r\n" + 'Content-Disposition: form-data; name="file"; filename="sample.wav"\r\n' + "Content-Type: audio/wav\r\n\r\n" + "RIFFfake-audio\r\n" + f"--{boundary}--\r\n" + ).encode() + request = urllib.request.Request( + f"http://127.0.0.1:{httpd.server_port}/v1/audio/transcriptions", + data=body, + headers={"Content-Type": f"multipart/form-data; boundary={boundary}"}, + method="POST", + ) + + with urllib.request.urlopen(request, timeout=10) as response: + payload = json.loads(response.read().decode("utf-8")) + + assert response.status == 200 + assert payload == {"text": "hello from gguf"} + args = json.loads(captured_args.read_text(encoding="utf-8")) + assert args[:4] == ["-m", str(tmp_path / "sensevoice.gguf"), "-a", args[3]] + assert os.path.exists(args[3]) is False + assert args[4:] == ["--vad", str(tmp_path / "fsmn-vad.gguf"), "--ids"] + finally: + httpd.shutdown() + httpd.server_close() diff --git a/runtime/llama.cpp/tests/test_release_workflow.py b/runtime/llama.cpp/tests/test_release_workflow.py index 201c97698..bddd44587 100644 --- a/runtime/llama.cpp/tests/test_release_workflow.py +++ b/runtime/llama.cpp/tests/test_release_workflow.py @@ -41,3 +41,12 @@ def test_release_notes_explain_cpu_and_cuda_windows_assets(): assert "other GPU architectures" in readme assert "CMAKE_CUDA_ARCHITECTURES=120" in readme assert "sm_120" in readme + + +def test_readme_documents_lightweight_http_server(): + readme = (ROOT / "runtime" / "llama.cpp" / "README.md").read_text(encoding="utf-8") + + assert "server/funasr_gguf_server.py" in readme + assert "/v1/audio/transcriptions" in readme + assert "--binary ./build/bin/llama-funasr-sensevoice" in readme + assert "one subprocess per request" in readme