From 8a0c727051a892bb0b067f025229aea45d34bb87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=B8=E9=9B=81?= Date: Wed, 20 May 2026 04:15:11 +0000 Subject: [PATCH] feat: add GLM-ASR-Nano model support Add GLMASR wrapper class integrating GLM-ASR-Nano-2512 (1.5B) into FunASR. Uses HuggingFace transformers AutoModel + AutoProcessor with trust_remote_code. Supports 17 languages with dialect and low-volume speech optimization. Usage: from funasr import AutoModel model = AutoModel(model="zai-org/GLM-ASR-Nano-2512", hub="hf", device="cuda:0") res = model.generate(input="audio.wav") # ModelScope (Chinese users) model = AutoModel(model="ZhipuAI/GLM-ASR-Nano-2512", hub="ms", device="cuda:0") Tested end-to-end on CPU with Chinese and English audio. New files: - funasr/models/glm_asr/model.py - funasr/models/glm_asr/__init__.py - tests_models/test_glm_asr.py - examples/industrial_data_pretraining/glm_asr/demo.py - examples/industrial_data_pretraining/glm_asr/demo_ms.py Note: requires transformers>=5.0.0 (pip install git+https://github.com/huggingface/transformers) --- README.md | 2 + README_zh.md | 2 + .../glm_asr/demo.py | 29 ++++ .../glm_asr/demo_ms.py | 22 +++ funasr/models/glm_asr/__init__.py | 0 funasr/models/glm_asr/model.py | 127 ++++++++++++++++++ tests_models/test_glm_asr.py | 56 ++++++++ 7 files changed, 238 insertions(+) create mode 100644 examples/industrial_data_pretraining/glm_asr/demo.py create mode 100644 examples/industrial_data_pretraining/glm_asr/demo_ms.py create mode 100644 funasr/models/glm_asr/__init__.py create mode 100644 funasr/models/glm_asr/model.py create mode 100644 tests_models/test_glm_asr.py diff --git a/README.md b/README.md index 8e1c37825..fe73cb013 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ ## What's new: - 2026/05/20: Added Qwen3-ASR (0.6B/1.7B) multi-language speech recognition models, supporting 52 languages with auto language detection. [usage](examples/industrial_data_pretraining/qwen3_asr). +- 2026/05/20: Added GLM-ASR-Nano (1.5B) robust speech recognition model, supporting 17 languages with dialect and low-volume speech optimization. [usage](examples/industrial_data_pretraining/glm_asr). - 2026/05/19: Fun-ASR-Nano and SenseVoice now support speaker diarization. Use with `vad_model` + `spk_model` + `punc_model` to get per-sentence speaker labels. See [Fun-ASR-Nano demo](examples/industrial_data_pretraining/fun_asr_nano/demo_spk.py), [SenseVoice demo](examples/industrial_data_pretraining/sense_voice/demo_spk.py). - 2025/12/15: [Fun-ASR-Nano-2512](https://github.com/FunAudioLLM/Fun-ASR) is an end-to-end speech recognition large model trained on tens of millions of hours real speech data. It supports low-latency real-time transcription and covers 31 languages. - 2024/10/29: Real-time Transcription Service 1.12 released, The 2pass-offline mode supports the SensevoiceSmal model；([docs](runtime/readme.md)); @@ -128,6 +129,7 @@ FunASR has open-sourced a large number of pre-trained models on industrial data. | Qwen-Audio-Chat
([⭐](examples/industrial_data_pretraining/qwen_audio/demo_chat.py) [🤗](https://huggingface.co/Qwen/Qwen-Audio-Chat) ) | audio-text multimodal models (chat) | multilingual | 8B | | Qwen3-ASR-1.7B
([⭐](examples/industrial_data_pretraining/qwen3_asr/demo.py) [🤗](https://huggingface.co/Qwen/Qwen3-ASR-1.7B) [🤖](https://modelscope.cn/models/Qwen/Qwen3-ASR-1.7B) ) | speech recognition, 52 languages, auto language detection | multilingual | 1.7B | | Qwen3-ASR-0.6B
([⭐](examples/industrial_data_pretraining/qwen3_asr/demo.py) [🤗](https://huggingface.co/Qwen/Qwen3-ASR-0.6B) [🤖](https://modelscope.cn/models/Qwen/Qwen3-ASR-0.6B) ) | speech recognition, 52 languages, auto language detection | multilingual | 0.6B | +| GLM-ASR-Nano-2512
([⭐](examples/industrial_data_pretraining/glm_asr/demo.py) [🤗](https://huggingface.co/zai-org/GLM-ASR-Nano-2512) [🤖](https://modelscope.cn/models/ZhipuAI/GLM-ASR-Nano-2512) ) | speech recognition, 17 languages, dialect and low-volume speech support | multilingual | 1.5B | | emotion2vec+large
([⭐](https://modelscope.cn/models/iic/emotion2vec_plus_large/summary) [🤗](https://huggingface.co/emotion2vec/emotion2vec_plus_large) ) | speech emotion recongintion | 40000 hours | 300M | [//]: # diff --git a/README_zh.md b/README_zh.md index e6c5aaff6..33dc13f0e 100644 --- a/README_zh.md +++ b/README_zh.md @@ -35,6 +35,7 @@ FunASR 希望在语音识别的学术研究和工业应用之间架起一座桥 ## 最新动态 - 2026/05/20: 新增 Qwen3-ASR（0.6B/1.7B）多语言语音识别模型，支持52种语言自动语言检测。[使用示例](examples/industrial_data_pretraining/qwen3_asr)。 +- 2026/05/20: 新增 GLM-ASR-Nano（1.5B）语音识别模型，支持17种语言，方言及低音量语音优化。[使用示例](examples/industrial_data_pretraining/glm_asr)。 - 2026/05/19: Fun-ASR-Nano 和 SenseVoice 现已支持说话人分离。配合 `vad_model` + `spk_model` + `punc_model` 使用，可获得带说话人标签的逐句结果。参见 [Fun-ASR-Nano demo](examples/industrial_data_pretraining/fun_asr_nano/demo_spk.py)、[SenseVoice demo](examples/industrial_data_pretraining/sense_voice/demo_spk.py)。 - 2025/12/15: [Fun-ASR-Nano-2512](https://github.com/FunAudioLLM/Fun-ASR) 是一款基于数千万小时真实语音数据训练的端到端语音识别大模型。它支持低延迟实时转写，并涵盖 31 种语言识别功能。 - 2024/10/29: 中文实时语音听写服务 1.12 发布，2pass-offline 模式支持 SensevoiceSmall 模型；详细信息参阅([部署文档](runtime/readme_cn.md)) @@ -128,6 +129,7 @@ FunASR 开源了大量在工业数据上预训练模型，您可以在[模型许 | Qwen-Audio-Chat
([⭐](examples/industrial_data_pretraining/qwen_audio/demo_chat.py) [🤗](https://huggingface.co/Qwen/Qwen-Audio-Chat) ) | 音频文本多模态大模型（chat 版本） | 多语言 | 8B | | Qwen3-ASR-1.7B
([⭐](examples/industrial_data_pretraining/qwen3_asr/demo.py) [🤗](https://huggingface.co/Qwen/Qwen3-ASR-1.7B) [🤖](https://modelscope.cn/models/Qwen/Qwen3-ASR-1.7B) ) | 语音识别，支持52种语言，自动语言检测 | 多语言 | 1.7B | | Qwen3-ASR-0.6B
([⭐](examples/industrial_data_pretraining/qwen3_asr/demo.py) [🤗](https://huggingface.co/Qwen/Qwen3-ASR-0.6B) [🤖](https://modelscope.cn/models/Qwen/Qwen3-ASR-0.6B) ) | 语音识别，支持52种语言，自动语言检测 | 多语言 | 0.6B | +| GLM-ASR-Nano-2512
([⭐](examples/industrial_data_pretraining/glm_asr/demo.py) [🤗](https://huggingface.co/zai-org/GLM-ASR-Nano-2512) [🤖](https://modelscope.cn/models/ZhipuAI/GLM-ASR-Nano-2512) ) | 语音识别，支持17种语言，方言及低音量语音优化 | 多语言 | 1.5B | | emotion2vec+large
([⭐](https://modelscope.cn/models/iic/emotion2vec_plus_large/summary) [🤗](https://huggingface.co/emotion2vec/emotion2vec_plus_large) ) | 情感识别模型 | 40000 小时，4 种情感类别 | 300M | diff --git a/examples/industrial_data_pretraining/glm_asr/demo.py b/examples/industrial_data_pretraining/glm_asr/demo.py new file mode 100644 index 000000000..5fc90d394 --- /dev/null +++ b/examples/industrial_data_pretraining/glm_asr/demo.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# -*- encoding: utf-8 -*- +# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. +# MIT License (https://opensource.org/licenses/MIT) + +# Requirements: pip install transformers>=5.0.0 torchaudio + +from funasr import AutoModel + +# Initialize GLM-ASR model +# hub: "ms" for ModelScope (default), "hf" for HuggingFace +model = AutoModel( + model="zai-org/GLM-ASR-Nano-2512", + hub="hf", + device="cuda:0", + dtype="bf16", +) + +# Chinese speech recognition +res = model.generate( + input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav", +) +print("Chinese:", res[0]["text"]) + +# English speech recognition +res = model.generate( + input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_en.wav", +) +print("English:", res[0]["text"]) diff --git a/examples/industrial_data_pretraining/glm_asr/demo_ms.py b/examples/industrial_data_pretraining/glm_asr/demo_ms.py new file mode 100644 index 000000000..661357499 --- /dev/null +++ b/examples/industrial_data_pretraining/glm_asr/demo_ms.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +# -*- encoding: utf-8 -*- +# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved. +# MIT License (https://opensource.org/licenses/MIT) + +# Requirements: pip install transformers>=5.0.0 torchaudio + +from funasr import AutoModel + +# Download from ModelScope (recommended for Chinese users) +# ModelScope model id: ZhipuAI/GLM-ASR-Nano-2512 +model = AutoModel( + model="ZhipuAI/GLM-ASR-Nano-2512", + hub="ms", + device="cuda:0", + dtype="bf16", +) + +res = model.generate( + input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav", +) +print(res[0]["text"]) diff --git a/funasr/models/glm_asr/__init__.py b/funasr/models/glm_asr/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/funasr/models/glm_asr/model.py b/funasr/models/glm_asr/model.py new file mode 100644 index 000000000..bc5673232 --- /dev/null +++ b/funasr/models/glm_asr/model.py @@ -0,0 +1,127 @@ +import logging +import os +import time + +import torch +import torch.nn as nn + +from funasr.register import tables + + +@tables.register("model_classes", "GLMASR") +@tables.register("model_classes", "zai-org/GLM-ASR-Nano-2512") +@tables.register("model_classes", "ZhipuAI/GLM-ASR-Nano-2512") +class GLMASR(nn.Module): + + def __init__(self, **kwargs): + super().__init__() + model_path = kwargs.get("model_path", kwargs.get("model", "zai-org/GLM-ASR-Nano-2512")) + device = kwargs.get("device", "cuda:0") + dtype = kwargs.get("dtype", "bf16") + hub = kwargs.get("hub", "ms") + self._max_new_tokens = kwargs.get("max_new_tokens", 512) + + self._dtype_map = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32} + self._device = device + self._torch_dtype = self._dtype_map.get(dtype, torch.bfloat16) + self._placeholder = nn.Parameter(torch.empty(0)) + + model_path = self._resolve_model_path(model_path, hub, kwargs) + self.model_path = model_path + + from transformers import AutoModel as HFAutoModel + from transformers import AutoProcessor + + self.processor = AutoProcessor.from_pretrained(model_path, trust_remote_code=True) + self.glm_model = HFAutoModel.from_pretrained( + model_path, + dtype=self._torch_dtype, + device_map=device, + trust_remote_code=True, + ) + self.glm_model.eval() + logging.info(f"GLM-ASR model loaded from {model_path}") + + def _resolve_model_path(self, model_path, hub, kwargs): + if os.path.exists(model_path): + return model_path + + if hub in ("ms", "modelscope"): + try: + from modelscope.hub.snapshot_download import snapshot_download + model_revision = kwargs.get("model_revision", "master") + local_path = snapshot_download(model_path, revision=model_revision) + logging.info(f"Downloaded from ModelScope: {model_path} -> {local_path}") + return local_path + except Exception as e: + logging.warning(f"ModelScope download failed: {e}, falling back to HuggingFace path") + + return model_path + + def forward(self, **kwargs): + raise NotImplementedError("GLMASR only supports inference mode") + + def inference( + self, + data_in, + data_lengths=None, + key: list = None, + tokenizer=None, + frontend=None, + **kwargs, + ): + meta_data = {} + time1 = time.perf_counter() + + prompt = kwargs.get("prompt", "Please transcribe this audio into text") + + if isinstance(data_in, (list, tuple)): + audio_list = list(data_in) + elif isinstance(data_in, str): + audio_list = [data_in] + else: + audio_list = [data_in] + + time2 = time.perf_counter() + meta_data["load_data"] = f"{time2 - time1:0.3f}" + + output = [] + for i, audio_input in enumerate(audio_list): + messages = [ + { + "role": "user", + "content": [ + {"type": "audio", "url": audio_input}, + {"type": "text", "text": prompt}, + ], + } + ] + + inputs = self.processor.apply_chat_template( + messages, + tokenize=True, + add_generation_prompt=True, + return_dict=True, + return_tensors="pt", + ) + inputs = inputs.to(self._device, dtype=self._torch_dtype) + + with torch.inference_mode(): + generated = self.glm_model.generate( + **inputs, + max_new_tokens=self._max_new_tokens, + do_sample=False, + ) + + text = self.processor.batch_decode( + generated[:, inputs["input_ids"].shape[1]:], + skip_special_tokens=True, + )[0].strip() + + k = key[i] if key and i < len(key) else f"sample_{i}" + output.append({"key": k, "text": text}) + + time3 = time.perf_counter() + meta_data["batch_data_time"] = time3 - time2 + + return output, meta_data diff --git a/tests_models/test_glm_asr.py b/tests_models/test_glm_asr.py new file mode 100644 index 000000000..579869ed9 --- /dev/null +++ b/tests_models/test_glm_asr.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""Test GLM-ASR: robust multi-language speech recognition""" +import sys +import time + + +def main(): + from funasr import AutoModel + + url_zh = "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav" + url_en = "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_en.wav" + + # Standard FunASR usage + # Default hub="ms" (ModelScope), use hub="hf" for HuggingFace + print("[GLM-ASR] Loading model...") + t0 = time.time() + model = AutoModel( + model="zai-org/GLM-ASR-Nano-2512", + hub="hf", + device="cuda:0", + disable_update=True, + ) + print("[GLM-ASR] Model loaded in %.1fs" % (time.time() - t0)) + + # Test 1: Chinese audio + print("[GLM-ASR] Test 1: Chinese inference...") + t0 = time.time() + res = model.generate(input=url_zh) + print("[GLM-ASR] Inference done in %.1fs" % (time.time() - t0)) + + if res and len(res) > 0 and "text" in res[0]: + print("[GLM-ASR] Result (zh): %s" % res[0]["text"]) + print("[GLM-ASR] Test 1 PASSED") + else: + print("[GLM-ASR] Test 1 FAILED - no text in result") + return 1 + + # Test 2: English audio + print("[GLM-ASR] Test 2: English inference...") + t0 = time.time() + res = model.generate(input=url_en) + print("[GLM-ASR] Inference done in %.1fs" % (time.time() - t0)) + + if res and len(res) > 0 and "text" in res[0]: + print("[GLM-ASR] Result (en): %s" % res[0]["text"]) + print("[GLM-ASR] Test 2 PASSED") + else: + print("[GLM-ASR] Test 2 FAILED - no text in result") + return 1 + + print("[GLM-ASR] All tests PASSED") + return 0 + + +if __name__ == "__main__": + sys.exit(main())