modelscope/FunASR
1
0
Fork 0
mirror of https://github.com/modelscope/FunASR.git synced 2026-10-11 22:39:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

docs: finish reference navigation and correct model and vLLM contracts

Browse Source 
Signed-off-by: zhifu gao <[email protected]>
This commit is contained in:
zhifu gao
2026-09-06 17:43:37 +00:00
parent 0bc3425f99
commit 2dfe391936
16 changed files with 885 additions and 458 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/product-site.yml
+6
View File
@@ -6,8 +6,10 @@ on:
- 'README*.md'
- 'docs/**/*.md'
- 'docs/conf.py'
- 'docs/index.rst'
- 'docs/_ext/**'
- 'tests/test_sphinx_links.py'
- 'tests/test_vllm_examples_contract.py'
- 'model_zoo/**/*.md'
- 'runtime/**/*.md'
- 'examples/openai_api/**/*.md'
@@ -24,8 +26,10 @@ on:
- 'README*.md'
- 'docs/**/*.md'
- 'docs/conf.py'
- 'docs/index.rst'
- 'docs/_ext/**'
- 'tests/test_sphinx_links.py'
- 'tests/test_vllm_examples_contract.py'
- 'model_zoo/**/*.md'
- 'runtime/**/*.md'
- 'examples/openai_api/**/*.md'
@@ -79,6 +83,8 @@ jobs:
run: >-
python -m pytest
tests/test_learning_docs_contract.py
tests/test_reference_docs_contract.py
tests/test_vllm_examples_contract.py
tests/test_training_docs_contract.py
tests/test_python_api_docs_contract.py
tests/test_service_docs_contract.py
docs/README.md
+12 -12
View File
@@ -75,19 +75,19 @@ python -m http.server --directory /tmp/funasr-docs 8000
## Build the Sphinx Reference
For convenience, we provide users with the ability to generate local HTML manually.
First, you should install the following packages, which is required for building HTML:
The legacy Sphinx reference is a separate local build. Use an isolated environment
with the dependency versions in the `legacy-docs-links` job of
[product-site.yml](../.github/workflows/product-site.yml); installing only the
FunASR package does not reproduce that builder. From the repository root:
```sh
pip3 install -U "funasr[doc]"
python -m pytest tests/test_sphinx_links.py tests/test_reference_docs_contract.py -q
python -m sphinx -b html -n -E -d /tmp/funasr-sphinx-doctrees docs /tmp/funasr-sphinx-html
```
Then you can generate HTML manually.
```sh
cd docs
make html
```
The generated files are all contained in the "FunASR/docs/_build" directory. You can access the FunASR documentation by simply opening the "html/index.html" file in your browser from this directory.
Open `/tmp/funasr-sphinx-html/index.html` after the build. Markdown heading
fragments remain usable alongside existing Sphinx anchors, including numbered
and Chinese headings. Missing destinations still produce warnings; inspect the
build log rather than treating an exit code alone as proof of correct links.
The GitHub Pages publishing workflow uses the product documentation exporter
and generated API reference, not this standalone Sphinx output.
docs/_ext/repository_links.py
+63 -6
View File
@@ -3,6 +3,7 @@
from html import escape
from html.parser import HTMLParser
from pathlib import Path
import re
from urllib.parse import quote, unquote, urlsplit, urlunsplit
from docutils import nodes
@@ -18,7 +19,7 @@ def resolve_uri(app, source, uri, *, raw=False, validate=False):
link = urlsplit(uri)
if link.scheme or link.netloc or link.path.startswith('/'):
return uri
if not link.path and not (raw or link.query):
if not link.path and not (raw or link.query or link.fragment):
return None
root = Path(app.srcdir).resolve().parent
target = (source.parent / unquote(link.path)).resolve() if link.path else source.resolve()
@@ -29,12 +30,19 @@ def resolve_uri(app, source, uri, *, raw=False, validate=False):
return None
target_doc = app.env.path2doc(str(target))
if target_doc in app.env.found_docs:
if not raw and not link.query:
# Preserve Sphinx's own document/fragment resolution and warnings.
if app.builder.format != 'html':
return None
if not raw and not (link.query or link.fragment):
# Plain document links retain Sphinx's own resolution.
return None
current_doc = app.env.path2doc(str(source))
destination = app.builder.get_relative_uri(current_doc, target_doc)
if validate and link.fragment:
fragment = unquote(link.fragment)
target_tree = app.env.get_doctree(target_doc)
mapped = target_tree.get('repository_heading_targets', {}).get(fragment)
if mapped:
link = link._replace(fragment=quote(mapped, safe='-_'))
validate_fragment(app, source, target_doc, uri, unquote(link.fragment))
else:
kind = 'tree' if target.is_dir() else 'blob'
@@ -62,7 +70,13 @@ class RepositoryMarkdownParser(CommonMarkParser):
if uri is None:
return super().visit_link(mdnode)
# Resolve before recommonmark strips .md or treats //host as a doc ref.
reference = nodes.reference(refuri=uri)
parsed = urlsplit(uri)
if not (parsed.scheme or parsed.netloc or parsed.path or parsed.query) and parsed.fragment:
# A fragment-only refuri is rewritten to a Sphinx label before our
# heading aliases exist. A refid stays an HTML anchor reference.
reference = nodes.reference(refid=unquote(parsed.fragment))
else:
reference = nodes.reference(refuri=uri)
reference['repository_links_uri'] = mdnode.destination
reference.line = self._get_line(mdnode)
if mdnode.title:
@@ -120,6 +134,42 @@ class RawLinks(HTMLParser):
return result
def add_markdown_heading_aliases(app, doctree):
"""Keep Sphinx IDs while accepting fragments used by repository Markdown."""
if app.builder.format != 'html' or Path(doctree['source']).suffix != '.md':
return
used = set()
raw_ids = set()
targets = {}
for node in doctree.findall(nodes.raw):
if 'html' in node.get('format', '').split():
raw_ids.update(RawLinks(node.astext(), lambda uri: None).ids)
for section in doctree.findall(nodes.section):
title = section.next_node(nodes.title, descend=True, siblings=False)
if title is None:
continue
base = re.sub(r'[^\w\- ]', '', title.astext().lower()).replace(' ', '-')
if not base:
continue
alias = base
suffix = 0
while alias in used:
suffix += 1
alias = f'{base}-{suffix}'
used.add(alias)
if alias in raw_ids:
LOGGER.warning('Markdown heading fragment conflicts with raw HTML anchor: %s',
alias, location=section, type='repository_links', subtype='collision')
elif alias not in doctree.ids:
section['ids'].append(alias)
doctree.ids[alias] = section
elif doctree.ids[alias] is not section:
# An existing public Sphinx ID belongs to a different heading.
# Keep that URL and route Markdown links to this heading's own ID.
targets[alias] = section['ids'][0]
doctree['repository_heading_targets'] = targets
def resolve_links(app, doctree, docname):
if app.builder.format != 'html':
return
@@ -127,7 +177,13 @@ def resolve_links(app, doctree, docname):
for node in doctree.findall(nodes.reference):
original = node.attributes.pop('repository_links_uri', None)
if original is not None:
resolve_uri(app, source, original, validate=True)
destination = resolve_uri(app, source, original, validate=True)
if destination is not None:
parsed = urlsplit(destination)
if 'refid' in node:
node['refid'] = unquote(parsed.fragment)
else:
node['refuri'] = destination
for node in doctree.findall(nodes.raw):
if 'html' not in node.get('format', '').split():
continue
@@ -142,5 +198,6 @@ def resolve_links(app, doctree, docname):
def setup(app):
app.add_source_parser(RepositoryMarkdownParser, override=True)
app.connect('doctree-read', add_markdown_heading_aliases)
app.connect('doctree-resolved', resolve_links)
return {'version': '1.0', 'parallel_read_safe': True}
return {'version': '1.1', 'parallel_read_safe': True}
docs/index.rst
+10 -1
View File
@@ -76,8 +76,17 @@ entry to these guides. Source Markdown remains in this repository.
reference/application
reference/FQA
.. toctree::
:maxdepth: 1
:caption: Japanese and Korean Guides
model_selection_ja
deployment_matrix_ja
model_selection_ko
deployment_matrix_ko
Model weights and runtime choices
--------------------------------
---------------------------------
- `Model Zoo <https://github.com/modelscope/FunASR/tree/main/model_zoo>`_
- `Production deployment manuals <https://www.funasr.com/en/deploy/>`_
docs/reference/FQA.md
+72 -182
View File
@@ -1,232 +1,122 @@
# FAQ and Troubleshooting
This page collects the questions that most often block new FunASR users. Start here before opening an issue.
This compatibility page keeps earlier FAQ links usable. Detailed diagnosis and
current commands live in [troubleshooting](../troubleshooting.md) /
[中文](../troubleshooting_zh.md). Choose the model and runtime before applying a
workaround: Python SDK, Python HTTP, native vLLM and C++ WebSocket are distinct.
## Which install command should I use?
For most users:
```bash
pip install -U funasr
```
For the newest examples, server CLI, or unreleased fixes:
```bash
git clone https://github.com/modelscope/FunASR.git
cd FunASR
pip install -e ./
```
For the OpenAI-compatible API server, install the web runtime dependencies too:
```bash
pip install funasr fastapi uvicorn python-multipart
```
Follow [installation](../installation/installation.md). Choose a released PyPI
package or a recorded source checkout, install matching PyTorch/torchaudio builds,
and verify imports in the same environment used for inference. Repository
examples are not all installed as package data by PyPI.
## Which Python and PyTorch versions are recommended?
Use Python 3.8 or later and install a PyTorch/torchaudio pair that matches your CUDA runtime. If CUDA is not configured, start with CPU to verify the workflow first:
```bash
funasr-server --model sensevoice --device cpu
```
After the CPU smoke test works, switch to CUDA:
```bash
funasr-server --model sensevoice --device cuda
```
Use the selected model/backend's requirements, not a universal version promise.
The installation guide uses Python 3.11 as an environment example and explains
the difference between package metadata and resolved dependency requirements.
MOSS and vLLM have their own dependencies; keep conflicting stacks isolated.
## Model download is slow or fails. What should I check?
FunASR models are available from ModelScope and Hugging Face. Choose the hub that is fastest in your network environment, and make sure the machine can reach it before debugging model code.
Common checks:
```bash
python -c "import torch; print(torch.__version__, torch.cuda.is_available())"
python -c "import funasr; print(funasr.__version__)"
```
If a model is already downloaded on another machine, use the local model path instead of the remote model name.
Check the chosen hub, full model ID, revision, cache permissions and available
disk space. See [models, cache and offline use](../installation/installation.md#4-models-cache-and-offline-use).
For offline inference, prepare every required model and dependency; disabling
FunASR's update check does not disable all network access.
## `funasr-server` says FastAPI or multipart packages are missing
Install the server dependencies:
```bash
pip install fastapi uvicorn python-multipart
```
Then start again:
```bash
funasr-server --model sensevoice --device cuda
```
Install the dependencies documented in the
[Python HTTP service guide](../../examples/openai_api/README.md) using the same
interpreter that launches the service. An SDK-only installation is not evidence
that an HTTP server or a model-specific backend is ready.
## Port 8000 is already in use
Start the service on another port:
```bash
funasr-server --model sensevoice --device cuda --port 9000
```
Then point clients to the new base URL:
```bash
curl http://localhost:9000/health
```
Choose another port with the service's `--port` option and update the client base
URL accordingly. For Compose, use its documented host-port setting. Verify the
health endpoint of the intended process before submitting audio.
## How do I verify the OpenAI-compatible API quickly?
Start the server:
```bash
funasr-server --model sensevoice --device cuda
```
In another terminal:
```bash
curl -L https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/BAC009S0764W0121.wav -o sample.wav
curl http://localhost:8000/v1/audio/transcriptions \
-F file=@sample.wav \
-F model=sensevoice \
-F response_format=verbose_json
```
The response should include `text`. With `verbose_json`, supported models may also return segment-level information.
Use the startup and transcription checks in the
[service guide](../../examples/openai_api/README.md). Compare the response with
the [HTTP schema](../../examples/openai_api/OPENAPI.md). A health response verifies
service readiness, not transcription accuracy or every optional output field.
## How do I run the OpenAI-compatible API with Docker Compose?
Use the example Compose setup when you want a reproducible local smoke test before wiring the API into a product or agent workflow:
```bash
cd examples/openai_api
cp .env.example .env
docker compose up --build
```
Then verify it from another terminal:
```bash
BASE_URL=http://localhost:8000 bash smoke_test.sh
```
The example container defaults to `FUNASR_DEVICE=cpu` so it can start on machines without NVIDIA Container Toolkit. For CUDA, first adapt the image to use CUDA-capable PyTorch/FunASR dependencies, then set `FUNASR_DEVICE=cuda`.
See also:
- [OpenAI API example](../../examples/openai_api/)
- [Client recipes](../../examples/openai_api/CLIENTS.md)
- [Deployment matrix](../deployment_matrix.md)
Follow the [Compose instructions](../../examples/openai_api/README.md) and
[container selection guide](../installation/docker.md). Verify image dependencies,
device, cache and ports together. Changing a device environment variable does
not add CUDA support to an image.
## Docker starts but `/health` or transcription fails
Check the container logs first:
```bash
docker compose logs -f funasr-api
```
Common causes:
- The first startup is still downloading or loading the model.
- Port 8000 is already in use; set `FUNASR_HOST_PORT=9000` in `.env` and use `BASE_URL=http://localhost:9000`.
- The container is running in CPU mode but `.env` or the command expects CUDA.
- CUDA is requested but the image does not include CUDA-capable PyTorch/FunASR dependencies.
- The model cache volume is empty or corrupted; retry after removing the `funasr-cache` Docker volume.
- The uploaded audio file is too large for the machine; verify with the public sample before testing long recordings.
When opening a Deployment Help issue, include your `.env` values without secrets, the `docker compose` command, container logs, model alias, device, and audio duration.
Inspect startup logs, model-loading state, dependencies, host-port mapping and
device availability. Preserve the failing configuration and relevant logs,
redacting secrets and private audio. Do not delete a shared model-cache volume
as a first diagnostic step; isolate an incomplete download only after identifying
it and retaining any needed local artifacts. See [troubleshooting](../troubleshooting.md).
## Long audio is slow, split incorrectly, or runs out of memory
Use VAD segmentation for long audio and tune segment length for your hardware:
Choose the workflow in the [Python SDK guide](../python_api.md). Paraformer-style
pipelines can use VAD and batch-size controls; shorter chunks can also introduce
boundary errors. Separate recognition, punctuation and timestamp errors before
changing segmentation. No single segment length is a universal accuracy fix.
```python
from funasr import AutoModel
model = AutoModel(
model="paraformer-zh",
vad_model="fsmn-vad",
punc_model="ct-punc",
vad_kwargs={"max_single_segment_time": 30000},
device="cuda",
)
result = model.generate(input="long_meeting.wav", batch_size_s=300)
```
If memory is limited, reduce `batch_size_s`, use CPU for verification, or split very long recordings before batch processing.
[MOSS](../moss_transcribe_diarize.md) jointly transcribes and diarizes long-form
audio. Do not add external `vad_model` or `spk_model` to its adapter: independent
chunk processing can break recording-level speaker consistency. Check its own
context, output-token and device requirements instead.
## Speaker diarization has no speaker labels
Use a model pipeline that includes both VAD and speaker models:
There are two different routes:
```python
from funasr import AutoModel
- A Paraformer-style VAD/ASR/punctuation/speaker-embedding pipeline: inspect
`sentence_info` and the [SDK speaker contract](../python_api.md#vad-timestamps-and-speakers).
- [MOSS-Transcribe-Diarize](../moss_transcribe_diarize.md): use its native labels,
without a second VAD or speaker model. The adapter normalizes structured
output into `sentence_info`; malformed tagged output must not invent labels.
model = AutoModel(
model="paraformer-zh",
vad_model="fsmn-vad",
punc_model="ct-punc",
spk_model="cam++",
device="cuda",
)
result = model.generate(input="meeting.wav")
```
Then inspect `result[0]["sentence_info"]`. Each sentence should include fields such as `text`, `start`, `end`, and `spk` when diarization is available.
Speaker labels are anonymous within a recording. They do not identify an enrolled
person and are not guaranteed to match labels in another recording.
## Can I use a speaker model other than cam++?
Yes. `spk_model` can be any FunASR `AutoModel` speaker embedding model, including a ModelScope model ID or a local model path. For example, ERes2NetV2 can be used directly:
```python
from funasr import AutoModel
model = AutoModel(
model="paraformer-zh",
vad_model="fsmn-vad",
punc_model="ct-punc",
spk_model="iic/speech_eres2netv2_sv_zh-cn_16k-common",
device="cuda",
)
result = model.generate(input="meeting.wav")
```
For a fully custom speaker model, make sure the model can be loaded by FunASR `AutoModel` and that its `inference()` method returns `spk_embedding`; the user-facing call still goes through `AutoModel.generate()`. The diarization post-processing and clustering steps use those embeddings to assign speaker labels. If your diarization model emits speaker segments directly instead of embeddings, add an adapter layer that converts its output to the current speaker-label structure.
For the supported pipeline and result fields, see [VAD, timestamps, and speakers](../python_api.md#vad-timestamps-and-speakers).
For the embedding-based pipeline, use a compatible registered speaker model
whose inference output contains `spk_embedding`; then test clustering with the
chosen ASR/VAD pipeline. See the
[SDK contract](../python_api.md#vad-timestamps-and-speakers) and
[model registration](../model_registration.md).
A model that emits diarized segments directly is not interchangeable with a
speaker-embedding model. MOSS uses its dedicated adapter, not `spk_model`.
## The same command works on CPU but fails on CUDA
This usually points to a CUDA, driver, PyTorch, or GPU memory mismatch. Include these checks in your issue:
```bash
nvidia-smi
python -c "import torch; print(torch.__version__, torch.version.cuda, torch.cuda.is_available())"
python -c "import torchaudio; print(torchaudio.__version__)"
```
Try a smaller model or lower batch size to rule out GPU memory pressure.
Record the driver, GPU, Python, PyTorch, torchaudio and CUDA build versions,
then check device support and peak memory for that model. Start with the
[environment checks](../installation/installation.md#3-verify-the-interpreter-and-imports)
and [troubleshooting](../troubleshooting.md). A CPU success does not establish GPU
wheel or model compatibility.
## What information should I include in an issue?
Please include:
- OS and Python version
- FunASR version and install method (`pip`, source, Docker)
- PyTorch, torchaudio, CUDA, and GPU information
- Exact command or minimal Python snippet
- Full traceback or server logs
- Model name and hub (`modelscope`, `hf`, or local path)
- Audio duration, sample rate, format, language, speaker count, and whether the audio can be shared
Provide a minimal command or script, expected versus actual output, package and
source versions, model ID/revision, runtime/device details, and relevant logs.
Include audio duration, format, sample rate and a shareable reproducer when
permitted. Remove credentials, private endpoints and personal audio before
posting. Keep the issue open while the proposed fix is being verified.
## Existing ModelScope pipeline examples
These are historical community discussions. Check their source/package versions
before reusing code; begin new integrations with the maintained SDK guide.
- [VAD model with ModelScope pipeline](https://github.com/modelscope/FunASR/discussions/236)
- [Punctuation model with ModelScope pipeline](https://github.com/modelscope/FunASR/discussions/238)
- [Paraformer streaming with ModelScope pipeline](https://github.com/modelscope/FunASR/discussions/241)
docs/reference/application.md
+36 -1
View File
@@ -1,5 +1,40 @@
# Speech Applications
Choose the application first, then select its model and serving contract.
The [use-case guide](../use_case_showcase.md) and
[community integrations](../community_projects.md) provide the maintained recipes.
## Audio Cut
Use [FunClip](https://github.com/modelscope/FunClip) for transcript-driven audio
and video clipping. For speaker-attributed recordings, start with
[MOSS-Transcribe-Diarize](../moss_transcribe_diarize.md): it produces transcription,
timestamps and anonymous speaker labels without an external VAD/speaker pipeline.
These labels are not verified real-world identities. Check the chosen FunClip
release and backend before assuming every ASR option has the same output fields.
## Realtime Speech Recognition
## Audio Chat
Start with the [deployment matrix](../deployment_matrix.md) and
[C++ WebSocket protocol](../../runtime/docs/websocket_protocol.md).
Python vLLM preview sessions and native C++ streaming services use different
messages and state transitions; their clients are not interchangeable.
Use the [realtime benchmark methodology](../benchmark/realtime_ws_benchmark.md)
to measure end-to-end latency under your traffic pattern.
MOSS is a long-form transcription/diarization option, not a replacement for a
native low-latency streaming model. Receiving an upload in chunks does not make
the underlying model a streaming recognizer.
## Audio Chat
For file or utterance transcription in an agent workflow, follow the
[Python HTTP service](../../examples/openai_api/README.md) and its
[workflow integration guide](../../examples/openai_api/WORKFLOWS.md).
FunASR supplies recognition; dialogue generation, speech synthesis, turn-taking
and interruption handling are separate application components. Validate their
combined latency and privacy requirements before deployment.
Next: [model selection](../model_selection.md),
[HTTP security](../../examples/openai_api/SECURITY.md),
and [troubleshooting](../troubleshooting.md).
docs/reference/build_task.md
+28 -117
View File
@@ -1,125 +1,36 @@
# Build custom tasks
FunASR is similar to ESPNet, which applies `Task` as the general interface ti achieve the training and inference of models. Each `Task` is a class inherited from `AbsTask` and its corresponding code can be seen in `funasr/tasks/abs_task.py`. The main functions of `AbsTask` are shown as follows:
```python
class AbsTask(ABC):
@classmethod
def add_task_arguments(cls, parser: argparse.ArgumentParser):
pass
@classmethod
def build_preprocess_fn(cls, args, train):
(...)
@classmethod
def build_collate_fn(cls, args: argparse.Namespace):
(...)
@classmethod
def build_model(cls, args):
(...)
@classmethod
def main(cls, args):
(...)
```
- add_task_arguments：Add parameters required by a specified `Task`
- build_preprocess_fn：定义如何处理对样本进行预处理 define how to preprocess samples
- build_collate_fn：define how to combine multiple samples into a `batch`
- build_model：define the model
- main：training interface, starting training through `Task.main()`
This is the migration entry for the former Task-based tutorial. That tutorial
described an older FunASR architecture; its `AbsTask` / `ASRTask` examples are not
interfaces in the current checkout. Do not copy them into a new integration.
Next, we take the speech recognition as an example to introduce how to define a new `Task`. For the corresponding code, please see `ASRTask` in `funasr/tasks/asr.py`. The procedure of defining a new `Task` is actually the procedure of redefining the above functions according to the requirements of the specified `Task`.
## Choose an extension point
- add_task_arguments
```python
@classmethod
def add_task_arguments(cls, parser: argparse.ArgumentParser):
group = parser.add_argument_group(description="Task related")
group.add_argument(
"--token_list",
type=str_or_none,
default=None,
help="A text mapping int-id to token",
)
(...)
```
For speech recognition tasks, specific parameters required include `token_list`, etc. According to the specific requirements of different tasks, users can define corresponding parameters in this function.
| What you need | Maintained guide | What to verify |
| --- | --- | --- |
| Run an existing checkpoint | [Python SDK](../python_api.md) | Model, input and output contract |
| Fine-tune an existing architecture | [Training](../training.md) | Recipe, data format, checkpoint and evaluation |
| Add a model architecture | [Model registration](../model_registration.md) | Registry name, configuration, `inference()` and result fields |
| Connect a third-party model | [MOSS integration](../moss_transcribe_diarize.md) | Upstream attribution, dependencies and adapter boundary |
| Expose a transcription service | [Deployment matrix](../deployment_matrix.md) | Runtime, protocol and supported model combination |
- build_preprocess_fn
```python
@classmethod
def build_preprocess_fn(cls, args, train):
if args.use_preprocessor:
retval = CommonPreprocessor(
train=train,
token_type=args.token_type,
token_list=args.token_list,
bpemodel=args.bpemodel,
non_linguistic_symbols=args.non_linguistic_symbols,
text_cleaner=args.cleaner,
...
)
else:
retval = None
return retval
```
This function defines how to preprocess samples. Specifically, the input of speech recognition tasks includes speech and text. For speech, functions such as (optional) adding noise and reverberation to the speech are supported. For text, functions such as (optional) processing text according to bpe and mapping text to `tokenid` are supported. Users can choose the preprocessing operation that needs to be performed on the sample. For the detail implementation, please refer to `CommonPreprocessor`.
The current model registration mechanism is implemented in
[register.py](../../funasr/register.py); inference orchestration is in
[auto_model.py](../../funasr/auto/auto_model.py). Registering a model is not the
same as adding support to every HTTP, WebSocket or C++ runtime. Follow the
specific service contract and test the intended route separately.
- build_collate_fn
```python
@classmethod
def build_collate_fn(cls, args, train):
return CommonCollateFn(float_pad_value=0.0, int_pad_value=-1)
```
This function defines how to combine multiple samples into a `batch`. For speech recognition tasks, `padding` is employed to obtain equal-length data from different speech and text. Specifically, we set `0.0` as the default padding value for speech and `-1` as the default padding value for text. Users can define different `batch` operations here. For the detail implementation, please refer to `CommonCollateFn`.
## Migrate an older extension
- build_model
```python
@classmethod
def build_model(cls, args, train):
with open(args.token_list, encoding="utf-8") as f:
token_list = [line.rstrip() for line in f]
vocab_size = len(token_list)
frontend = frontend_class(**args.frontend_conf)
specaug = specaug_class(**args.specaug_conf)
normalize = normalize_class(**args.normalize_conf)
preencoder = preencoder_class(**args.preencoder_conf)
encoder = encoder_class(input_size=input_size, **args.encoder_conf)
postencoder = postencoder_class(input_size=encoder_output_size, **args.postencoder_conf)
decoder = decoder_class(vocab_size=vocab_size, encoder_output_size=encoder_output_size, **args.decoder_conf)
ctc = CTC(odim=vocab_size, encoder_output_size=encoder_output_size, **args.ctc_conf)
model = model_class(
vocab_size=vocab_size,
frontend=frontend,
specaug=specaug,
normalize=normalize,
preencoder=preencoder,
encoder=encoder,
postencoder=postencoder,
decoder=decoder,
ctc=ctc,
token_list=token_list,
**args.model_conf,
)
return model
```
This function defines the detail of the model. For different speech recognition models, the same speech recognition `Task` can usually be shared and the remaining thing needed to be done is to define a specific model in this function. For example, a speech recognition model with a standard encoder-decoder structure has been shown above. Specifically, it first defines each module of the model, including encoder, decoder, etc. and then combine these modules together to generate a complete model. In FunASR, the model needs to inherit `FunASRModel` and the corresponding code can be seen in `funasr/train/abs_espnet_model.py`. The main function needed to be implemented is the `forward` function.
1. Record the original FunASR commit, configuration, checkpoint and dependency
versions before changing the environment.
2. Choose a current recipe or registered model with matching inputs and outputs.
Map the old preprocessing, collation and model construction responsibilities
to that recipe; do not assume the old Task class has a drop-in replacement.
3. Test configuration loading, checkpoint compatibility and a small inference
sample before training or serving. Compare outputs against the old pinned
environment, then run the relevant evaluation.
Next, we take `SANMEncoder` as an example to introduce how to use a custom encoder as a part of the model when defining the specified model and the corresponding code can be seen in `funasr/models/encoder/sanm_encoder.py`. For a custom encoder, in addition to inheriting the common encoder class `AbsEncoder`, it is also necessary to define the `forward` function to achieve the forward computation of the `encoder`. After defining the `encoder`, it should also be registered in the `Task`. The corresponding code example can be seen as below:
```python
encoder_choices = ClassChoices(
"encoder",
classes=dict(
conformer=ConformerEncoder,
transformer=TransformerEncoder,
rnn=RNNEncoder,
sanm=SANMEncoder,
sanm_chunk_opt=SANMEncoderChunkOpt,
data2vec_encoder=Data2VecEncoder,
mfcca_enc=MFCCAEncoder,
),
type_check=AbsEncoder,
default="rnn",
)
```
In this code, `sanm=SANMEncoder` takes the newly defined `SANMEncoder` as an optional choice of the `encoder`. Once the user specifies the `encoder` as `sanm` in the configuration file, the `SANMEncoder` will be correspondingly employed as the `encoder` module of the model.
For historical experiments, keep their matching source revision in an isolated
environment. This page preserves the old documentation URL; current implementation
instructions live in the guides above.
docs/vllm_guide.md
+72 -44
View File
@@ -14,7 +14,7 @@
| Fun-ASR-Nano | **Offline service (+SPK)** | dynamic | **46** | 8.19% | SPK off by default |
| GLM-ASR-Nano | **vLLM batch** | fixed | **265** | 12.93% | No long-audio support |
> vLLM matches PyTorch CER exactly (delta < 0.2%) while achieving 16–340x speedup.
> In the reported table, Fun-ASR-Nano batch throughput is `340 / 21 = 16.2` times the PyTorch baseline when timing scopes match. `RTFx 340` means 340 times realtime, not 340 times faster than PyTorch. CER changes from `8.06%` to `8.20%` (+0.14 percentage points), not identical accuracy. These historical measurements are not a guarantee for other hardware, workloads, or configurations.
---
@@ -169,14 +169,14 @@ FunASR's vLLM integration splits the ASR model into two independently running co
| Batching | Manual padding required | Continuous Batching, automatic scheduling |
| CUDA optimization | None | CUDA Graph + operator fusion |
| Multi-GPU parallelism | Manual implementation | Tensor Parallel with one-line config |
| Throughput | RTFx ~20 | **RTFx 340+** |
| Reported batch throughput | RTFx 21 | RTFx 340; see Benchmark scope |
### Supported Models
| Model | LLM component | Audio encoder | vLLM speedup |
| Model | LLM component | Audio encoder | Integration |
|-------|--------------|---------------|-------------|
| **Fun-ASR-Nano** | Qwen3-0.6B | SenseVoice | ✓ 21.7x |
| **GLM-ASR-Nano** | Llama-2B | Whisper-like | ✓ 7.6x |
| **Fun-ASR-Nano** | Qwen3-0.6B | SenseVoice | Specialized split engine |
| **GLM-ASR-Nano** | Llama-2B | Whisper-like | Specialized split engine |
| LLMASR | Qwen/Vicuna | Whisper | ✓ |
| Paraformer | No LLM | — | ✗ Non-autoregressive |
| SenseVoice | No LLM | — | ✗ Encoder-decoder |
@@ -259,7 +259,7 @@ Offline SDK inference splits the ASR pipeline into two stages executed independe
| Batching | Manual padding alignment | Continuous Batching auto-scheduling |
| CUDA | Sequential per-sample execution | CUDA Graph + operator fusion |
| Multi-GPU | Manual implementation | Tensor Parallel one-line config |
| Result | RTFx ~20 | **RTFx 340+** (16x speedup) |
| Reported batch result | RTFx 21 | RTFx 340; hardware and timing scope must match |
### Universal Interface (Recommended)
@@ -293,13 +293,15 @@ engine = FunASRNanoVLLM.from_pretrained(
)
results = engine.generate(
inputs="wav.scp", # supports scp/jsonl/file lists
inputs=["audio1.wav", "audio2.wav"], # existing audio files, not manifests
hotwords=["开放时间"],
language="中文",
max_new_tokens=512,
)
```
The direct engine accepts an audio path, a list of audio paths, or 16 kHz waveform arrays/tensors. It does not expand SCP/JSONL manifests. Use [demo_vllm.py](../examples/industrial_data_pretraining/fun_asr_nano/demo_vllm.py) below for manifests: SCP rows contain a path or `key path`; JSONL rows contain a `source` audio-path field. Paths must resolve from the process working directory.
### Command Line
```bash
@@ -451,24 +453,26 @@ The most feature-complete interface, supporting speaker diarization, timestamps,
**Response**:
Illustrative values, not a measured run. HTTP timestamps and duration are in seconds. Optional `words` and `speaker` fields require their respective processing paths; see [the serializer and service](../examples/industrial_data_pretraining/fun_asr_nano/serve_vllm.py).
```json
{
"text": "Full transcription text",
"text": "你好",
"segments": [
{
"text": "Segment text",
"start": 1.7,
"end": 14.8,
"text": "你好",
"start": 0.3,
"end": 1.2,
"speaker": "SPK0",
"words": [
{"word": "砸", "start": 2.02, "end": 2.08},
{"word": "了", "start": 2.26, "end": 2.32}
{"word": "你", "start": 0.3, "end": 0.6},
{"word": "好", "start": 0.6, "end": 1.2}
]
}
],
"duration": 227.4,
"processing_time": 3.422,
"rtf": 0.015
"duration": 2.0,
"processing_time": 0.1,
"rtf": 0.05
}
```
@@ -516,17 +520,22 @@ Compatible with the OpenAI Whisper API standard; works directly with the OpenAI
**Response** (`verbose_json`):
Illustrative complete response; timestamps are in seconds and are not a recognition-quality guarantee.
```json
{
"task": "transcribe",
"language": "zh",
"duration": 5.17,
"text": "我一直没有照顾孩子，但是我想要抚养权。",
"duration": 2.0,
"text": "你好",
"segments": [
{
"id": 0, "start": 0.0, "end": 5.15,
"text": "我一直没有照顾孩子，但是我想要抚养权。",
"words": [{"word": "我", "start": 0.42, "end": 0.48}, ...]
"id": 0, "start": 0.3, "end": 1.2,
"text": "你好",
"words": [
{"word": "你", "start": 0.3, "end": 0.6},
{"word": "好", "start": 0.6, "end": 1.2}
]
}
]
}
@@ -568,10 +577,12 @@ WebSocket interface for the offline service. Send complete audio, then receive r
**Server → Client**:
```json
Each line below represents a separate JSON WebSocket message, not one JSON document. Values are illustrative; WebSocket offsets are in milliseconds.
```text
{"event": "started"}
{"event": "language_set", "language": "中文"}
{"sentences": [{"text":"...","start":..,"end":..}], "is_final": true, "duration_ms": 5170}
{"sentences": [{"text": "你好", "start": 300, "end": 1200}], "is_final": true, "duration_ms": 2000}
{"event": "stopped"}
```
@@ -633,7 +644,7 @@ Client (microphone / audio stream) serve_realtime_ws.py
- Natural sentence segmentation based on VAD endpoints
- Confirmed segment text is locked and never changes; partial text updates in real time
- Optional streaming speaker assignment (`--enable-spk`) + global re-clustering on STOP
- First-word latency ~480 ms
- The partial-decode interval is a scheduling setting, not a first-word latency guarantee; measure end-to-end latency on your workload
### 6.2 Starting the Service
@@ -699,10 +710,12 @@ the final disconnect log.
**Server → Client**:
```json
Illustrative message sequence, one JSON object per WebSocket message. Offsets are in milliseconds; this is not a measured latency trace.
```text
{"event": "started"}
{"sentences": [{"text":"你好","start":300,"end":1200}], "partial": "世界", "is_final": false}
{"sentences": [...], "is_final": true}
{"sentences": [{"text": "你好", "start": 300, "end": 1200}], "partial": "世界", "is_final": false}
{"sentences": [{"text": "你好", "start": 300, "end": 1200}, {"text": "世界", "start": 1500, "end": 2200}], "is_final": true}
{"event": "stopped"}
```
@@ -826,28 +839,43 @@ CUDA_VISIBLE_DEVICES=0 python examples/industrial_data_pretraining/fun_asr_nano/
## 7. Dynamic VAD
fsmn-vad enables dynamic silence thresholds by default. Offline and streaming modes use different configurations.
Dynamic silence is a **VAD-stage** option, not an ASR decoder option. [FSMN-VAD](../funasr/models/fsmn_vad_streaming/model.py) reads `dynamic_silence` and `silence_schedule`; an explicit `max_end_silence_time` disables the dynamic default unless overridden. `AutoModelVLLM.generate(inputs, **kwargs)` forwards to the ASR engine and does not run VAD.
| Accumulated duration | Offline (preserve long segs ≤60 s) | Streaming (balance latency) |
|---------------------|-----------------------------------|-----------------------------|
| ≤ 5 s | 2000 ms | 2000 ms |
| 5–10 s | 2000 ms | 1500 ms |
| 10–15 s | 1000 ms | 1000 ms |
| 15–20 s | 1000 ms | 800 ms |
| 20–30 s | 800 ms | 800 ms |
| 30–45 s | 600 ms | 400 ms |
| 45–60 s | 200–400 ms | 100 ms |
| > 60 s | 100 ms | 100 ms |
The table samples the SDK's `DEFAULT_SILENCE_SCHEDULE` and `STREAMING_SILENCE_SCHEDULE` constants at boundary durations. Each schedule selects the first entry whose duration limit is greater than or equal to accumulated speech; these are silence thresholds, not maximum utterance lengths.
Offline mode favors longer segments to reduce boundary-cut losses; streaming mode tightens faster to reduce latency.
| Accumulated speech sample | DEFAULT_SILENCE_SCHEDULE | STREAMING_SILENCE_SCHEDULE |
| --- | --- | --- |
| 5000 ms | 2000 ms | 2000 ms |
| 10000 ms | 2000 ms | 1500 ms |
| 15000 ms | 1000 ms | 1000 ms |
| 20000 ms | 1000 ms | 800 ms |
| 30000 ms | 800 ms | 800 ms |
| 40000 ms | 600 ms | 400 ms |
| 45000 ms | 400 ms | 400 ms |
| 50000 ms | 400 ms | 100 ms |
| 60000 ms | 200 ms | 100 ms |
| 60001 ms | 100 ms | 100 ms |
The streaming-named constant is not automatically selected just because audio arrives in chunks. Service wrappers can choose their own policy; for example [DynamicStreamingVAD](../funasr/models/fsmn_vad_streaming/dynamic_vad.py) maintains its own schedule and calls the underlying VAD with `dynamic_silence=False`. Do not treat this SDK table as the configuration of every server.
### Customization
```python
model.generate(input="audio.wav", silence_schedule=[(5000,1500), (20000,800), (float('inf'),300)])
from funasr import AutoModel
vad = AutoModel(model="fsmn-vad", device="cpu", disable_update=True)
segments = vad.generate(
input="audio.wav",
cache={},
dynamic_silence=True,
silence_schedule=[(5000, 1500), (20000, 800), (float("inf"), 300)],
)
print(segments[0]["value"])
```
> GLM-ASR does not support long-segment inference; pass `dynamic_silence=False` when using it.
Replace `audio.wav` with an existing recording. This standalone VAD call returns `[start_ms, end_ms]` intervals in `value`, not transcripts. To feed an ASR engine, load/resample audio to its required sample rate, slice by these millisecond offsets, and pass the waveform list as `model.generate(inputs=audio_segments)`. Preserve the offsets when merging results. This example only demonstrates segmentation; it does not automatically connect VAD to vLLM.
> For GLM-ASR, pre-segment and validate duration limits for the selected checkpoint. To request fixed silence, pass `dynamic_silence=False` to the **VAD** call, not the ASR engine. Fixed silence alone does not cap segment duration.
---
@@ -870,17 +898,17 @@ model.generate(input="audio.wav", silence_schedule=[(5000,1500), (20000,800), (f
Complete files → offline (high throughput). Microphone / live stream → streaming (low latency).
**Q: Can GLM-ASR use dynamic VAD?**
It does not support long-segment inference. Use `dynamic_silence=False`.
Pre-segment long recordings and validate segment lengths with the chosen GLM checkpoint. `dynamic_silence=False` configures the separate FSMN-VAD stage, not `AutoModelVLLM`; disabling dynamic silence alone does not enforce an ASR-safe maximum segment length.
**Q: Performance impact of SPK?**
RTFx drops from 102 to 46. CER is unchanged. Disabled by default.
In the reported offline-service table, RTFx is 102 without SPK and 46 with SPK; CER is 8.14% and 8.19%, respectively. SPK is disabled by default. These measurements do not predict the cost or accuracy of another deployment.
**Q: Entry points for custom development?**
Offline: `serve_vllm.process_audio()` / `FunASRNanoVLLM.generate()`
Streaming: `serve_realtime_ws.RealtimeASRSession`
**Q: Slow first startup?**
vLLM initialization takes 60–90 s (KV Cache + CUDA Graph warmup). Subsequent inferences are instant.
Model loading, KV-cache allocation, and optional CUDA Graph warmup contribute to startup time. Measure cold startup and warm inference separately; neither has a fixed duration or an instant-response guarantee.
**Q: What happens when Fun-ASR-Nano vLLM uses `dtype="fp16"`?**
The audio frontend and adaptor remain float16, but FunASR runs the Qwen3 decoder
docs/vllm_guide_zh.md
+72 -44
View File
@@ -14,7 +14,7 @@
| Fun-ASR-Nano | **离线服务 (+SPK)** | dynamic | **46** | 8.19% | SPK 默认关闭 |
| GLM-ASR-Nano | **vLLM batch** | fixed | **265** | 12.93% | 不支持长音频推理 |
> vLLM 与 PyTorch CER 完全一致（差 < 0.2%），速度提升 16-340x。
> 表中 Fun-ASR-Nano 的 batch 吞吐量比值为 `340 / 21 = 16.2`，前提是计时范围相同。`RTFx 340` 表示实时倍率，不是相对 PyTorch 加速 340 倍。CER 从 `8.06%` 变为 `8.20%`，相差 0.14 个百分点，并非完全相同。以上历史测量不保证其他硬件、话务和配置的性能或精度。
---
@@ -164,14 +164,14 @@ FunASR 的 vLLM 集成将 ASR 模型拆分为两部分独立运行：
| 批处理 | 需手动 padding | Continuous Batching，自动调度 |
| CUDA 优化 | 无 | CUDA Graph + 算子融合 |
| 多卡并行 | 手动实现 | Tensor Parallel 一行配置 |
| 吞吐量 | RTFx ~20 | **RTFx 340+** |
| 已报告的 batch 吞吐量 | RTFx 21 | RTFx 340；范围见 Benchmark |
### 支持模型
| 模型 | LLM 部分 | audio encoder | vLLM 加速 |
| 模型 | LLM 部分 | audio encoder | 集成路径 |
|------|---------|---------------|-----------|
| **Fun-ASR-Nano** | Qwen3-0.6B | SenseVoice | ✓ 21.7x |
| **GLM-ASR-Nano** | Llama-2B | Whisper-like | ✓ 7.6x |
| **Fun-ASR-Nano** | Qwen3-0.6B | SenseVoice | 专用 split engine |
| **GLM-ASR-Nano** | Llama-2B | Whisper-like | 专用 split engine |
| LLMASR | Qwen/Vicuna | Whisper | ✓ |
| Paraformer | 无 LLM | — | ✗ 非自回归 |
| SenseVoice | 无 LLM | — | ✗ encoder-decoder |
@@ -255,7 +255,7 @@ FunASR 的 vLLM 集成将 ASR 模型拆分为两部分独立运行：
| 批处理 | 需手动 padding 对齐 | Continuous Batching 自动调度 |
| CUDA | 逐 sample 串行 | CUDA Graph + 算子融合 |
| 多卡 | 需手动实现 | Tensor Parallel 一行配置 |
| 结果 | RTFx ~20 | **RTFx 340+**（16倍加速） |
| 已报告的 batch 结果 | RTFx 21 | RTFx 340；需匹配硬件和计时范围 |
### 通用接口（推荐）
@@ -289,13 +289,15 @@ engine = FunASRNanoVLLM.from_pretrained(
)
results = engine.generate(
inputs="wav.scp", # 支持 scp/jsonl/文件列表
inputs=["audio1.wav", "audio2.wav"], # 已存在的音频文件，不是清单文件
hotwords=["开放时间"],
language="中文",
max_new_tokens=512,
)
```
直接引擎接收音频路径、音频路径列表或 16 kHz 波形数组/tensor，不展开 SCP/JSONL 清单。清单使用下方 [demo_vllm.py](../examples/industrial_data_pretraining/fun_asr_nano/demo_vllm.py)：SCP 每行为路径或 `key path`，JSONL 每行对象的 `source` 字段为音频路径。相对路径按进程工作目录解析。
### 命令行
```bash
@@ -447,24 +449,26 @@ CUDA_VISIBLE_DEVICES=0 python examples/industrial_data_pretraining/fun_asr_nano/
**响应**：
以下数值用于说明结构，不是实测结果。HTTP 时间戳和时长单位为秒；可选 `words`、`speaker` 字段取决于相应处理路径，见[服务与序列化实现](../examples/industrial_data_pretraining/fun_asr_nano/serve_vllm.py)。
```json
{
"text": "完整识别文本",
"text": "你好",
"segments": [
{
"text": "段文本",
"start": 1.7,
"end": 14.8,
"text": "你好",
"start": 0.3,
"end": 1.2,
"speaker": "SPK0",
"words": [
{"word": "砸", "start": 2.02, "end": 2.08},
{"word": "了", "start": 2.26, "end": 2.32}
{"word": "你", "start": 0.3, "end": 0.6},
{"word": "好", "start": 0.6, "end": 1.2}
]
}
],
"duration": 227.4,
"processing_time": 3.422,
"rtf": 0.015
"duration": 2.0,
"processing_time": 0.1,
"rtf": 0.05
}
```
@@ -512,17 +516,22 @@ const result = await resp.json();
**响应**（`verbose_json`）：
以下为结构完整的示例响应；时间戳单位为秒，不是识别质量保证。
```json
{
"task": "transcribe",
"language": "zh",
"duration": 5.17,
"text": "我一直没有照顾孩子，但是我想要抚养权。",
"duration": 2.0,
"text": "你好",
"segments": [
{
"id": 0, "start": 0.0, "end": 5.15,
"text": "我一直没有照顾孩子，但是我想要抚养权。",
"words": [{"word": "我", "start": 0.42, "end": 0.48}, ...]
"id": 0, "start": 0.3, "end": 1.2,
"text": "你好",
"words": [
{"word": "你", "start": 0.3, "end": 0.6},
{"word": "好", "start": 0.6, "end": 1.2}
]
}
]
}
@@ -565,10 +574,12 @@ curl -X POST http://localhost:8899/v1/audio/transcriptions \
**服务端 → 客户端**：
```json
下面每行是独立的 JSON WebSocket 消息，不是一个 JSON 文档。数值为示例，WebSocket 偏移单位为毫秒。
```text
{"event": "started"}
{"event": "language_set", "language": "中文"}
{"sentences": [{"text":"...","start":..,"end":..}], "is_final": true, "duration_ms": 5170}
{"sentences": [{"text": "你好", "start": 300, "end": 1200}], "is_final": true, "duration_ms": 2000}
{"event": "stopped"}
```
@@ -630,7 +641,7 @@ asyncio.run(offline_ws("audio.wav"))
- 基于 VAD 端点自然分句
- 确认段文字锁定不变，partial 实时更新
- 可选流式说话人分配（`--enable-spk`）+ STOP 时全局重聚类
- 首字延迟 ~480ms
- partial 解码间隔是调度参数，不是首字延迟保证；需按实际话务测量端到端延迟
### 6.2 启动服务
@@ -692,10 +703,12 @@ CUDA_VISIBLE_DEVICES=0 python examples/industrial_data_pretraining/fun_asr_nano/
**服务端 → 客户端**：
```json
示例消息序列，每个 JSON 对象对应一条 WebSocket 消息。偏移单位为毫秒，不是延迟实测记录。
```text
{"event": "started"}
{"sentences": [{"text":"你好","start":300,"end":1200}], "partial": "世界", "is_final": false}
{"sentences": [...], "is_final": true}
{"sentences": [{"text": "你好", "start": 300, "end": 1200}], "partial": "世界", "is_final": false}
{"sentences": [{"text": "你好", "start": 300, "end": 1200}, {"text": "世界", "start": 1500, "end": 2200}], "is_final": true}
{"event": "stopped"}
```
@@ -819,28 +832,43 @@ CUDA_VISIBLE_DEVICES=0 python examples/industrial_data_pretraining/fun_asr_nano/
## 7. 动态 VAD
fsmn-vad 默认启用动态静音阈值。离线和流式使用不同配置。
动态静音属于 **VAD 阶段**，不是 ASR 解码参数。[FSMN-VAD](../funasr/models/fsmn_vad_streaming/model.py) 读取 `dynamic_silence` 和 `silence_schedule`；显式设置 `max_end_silence_time` 会关闭默认动态策略，除非另行覆盖。`AutoModelVLLM.generate(inputs, **kwargs)` 只转交给 ASR 引擎，不运行 VAD。
| 累积时长 | 离线（保留长段 ≤60s） | 流式（平衡延迟） |
|---------|-------------------|----------------|
| ≤ 5s | 2000ms | 2000ms |
| 5-10s | 2000ms | 1500ms |
| 10-15s | 1000ms | 1000ms |
| 15-20s | 1000ms | 800ms |
| 20-30s | 800ms | 800ms |
| 30-45s | 600ms | 400ms |
| 45-60s | 200-400ms | 100ms |
| > 60s | 100ms | 100ms |
下表在边界时长采样 SDK 的 `DEFAULT_SILENCE_SCHEDULE` 与 `STREAMING_SILENCE_SCHEDULE` 常量。每个 schedule 选择首个大于或等于累积语音时长的上限对应项；这些是静音阈值，不是语音段长上限。
离线倾向保留长段减少边界损失；流式更快收紧以降低延迟。
| 累积语音时长采样点 | DEFAULT_SILENCE_SCHEDULE | STREAMING_SILENCE_SCHEDULE |
| --- | --- | --- |
| 5000 ms | 2000 ms | 2000 ms |
| 10000 ms | 2000 ms | 1500 ms |
| 15000 ms | 1000 ms | 1000 ms |
| 20000 ms | 1000 ms | 800 ms |
| 30000 ms | 800 ms | 800 ms |
| 40000 ms | 600 ms | 400 ms |
| 45000 ms | 400 ms | 400 ms |
| 50000 ms | 400 ms | 100 ms |
| 60000 ms | 200 ms | 100 ms |
| 60001 ms | 100 ms | 100 ms |
分块输入不会自动选择名称带 streaming 的常量。服务包装层可能使用独立策略，例如 [DynamicStreamingVAD](../funasr/models/fsmn_vad_streaming/dynamic_vad.py) 自己维护 schedule，并以 `dynamic_silence=False` 调用底层 VAD。不要把此 SDK 表格当作所有服务的配置。
### 自定义
```python
model.generate(input="audio.wav", silence_schedule=[(5000,1500), (20000,800), (float('inf'),300)])
from funasr import AutoModel
vad = AutoModel(model="fsmn-vad", device="cpu", disable_update=True)
segments = vad.generate(
input="audio.wav",
cache={},
dynamic_silence=True,
silence_schedule=[(5000, 1500), (20000, 800), (float("inf"), 300)],
)
print(segments[0]["value"])
```
> GLM-ASR 不支持长段，使用时传 `dynamic_silence=False`。
将 `audio.wav` 替换为实际录音。此独立 VAD 调用在 `value` 中返回毫秒单位的 `[start_ms, end_ms]` 区间，不返回转写文本。接入 ASR 时，按引擎要求加载并重采样音频，依据毫秒区间切片，再将波形列表传给 `model.generate(inputs=audio_segments)`；合并结果时保留原始偏移。本示例只演示分段，不会自动把 VAD 连接到 vLLM。
> GLM-ASR 应先分段，并验证所选 checkpoint 的段长限制。需要固定静音阈值时，将 `dynamic_silence=False` 传给 **VAD** 调用，而不是 ASR 引擎；固定静音本身不限制最大段长。
---
@@ -863,17 +891,17 @@ model.generate(input="audio.wav", silence_schedule=[(5000,1500), (20000,800), (f
完整文件 → 离线（高吞吐）。麦克风/直播 → 流式（低延迟）。
**Q: GLM-ASR 用动态 VAD？**
不支持长段推理，用 `dynamic_silence=False`。
长录音应先分段，并针对所选 GLM checkpoint 验证段长。`dynamic_silence=False` 配置独立的 FSMN-VAD 阶段，不是 `AutoModelVLLM`；关闭动态静音本身不保证段长符合 ASR 限制。
**Q: SPK 性能影响？**
RTFx 102 → 46。CER 不变。默认关闭。
表中离线服务关闭/开启 SPK 的 RTFx 分别为 102/46，CER 分别为 8.14%/8.19%。SPK 默认关闭；这组测量不代表其他部署的开销或精度。
**Q: 二次开发入口？**
离线：`serve_vllm.process_audio()` / `FunASRNanoVLLM.generate()`
流式：`serve_realtime_ws.RealtimeASRSession`
**Q: 首次慢？**
vLLM 初始化 60-90s，之后即时。
模型加载、KV cache 分配及可选 CUDA Graph 预热都会影响首次启动。请分别测量冷启动与预热后推理，二者均没有固定耗时或即时返回保证。
**Q: Fun-ASR-Nano vLLM 使用 `dtype="fp16"` 时实际会怎样？**
音频 frontend 与 adaptor 仍使用 FP16，但 FunASR 会让 Qwen3 decoder 使用 BF16，
docs/vllm_guide_zh_v2.md
+73 -45
View File
@@ -4,7 +4,7 @@
## Benchmark
**测试集**：184 文件，11541 秒，Fun-ASR-Nano / GLM-ASR-Nano。
**测试集**：184 文件，11541 秒，Fun-ASR-Nano / GLM-ASR-Nano。RTFx 定义、计时口径和可复现字段请见 [Benchmark RTF and Reproducibility Notes](./benchmark/rtf_reproducibility.md)。
| 模型 | 引擎 | VAD | RTFx | CER | 备注 |
|------|------|-----|------|-----|------|
@@ -14,7 +14,7 @@
| Fun-ASR-Nano | **离线服务 (+SPK)** | dynamic | **46** | 8.19% | SPK 默认关闭 |
| GLM-ASR-Nano | **vLLM batch** | fixed | **265** | 12.93% | 不支持长音频推理 |
> vLLM 与 PyTorch CER 完全一致（差 < 0.2%），速度提升 16-340x。
> 表中 Fun-ASR-Nano 的 batch 吞吐量比值为 `340 / 21 = 16.2`，前提是计时范围相同。`RTFx 340` 表示实时倍率，不是相对 PyTorch 加速 340 倍。CER 从 `8.06%` 变为 `8.20%`，相差 0.14 个百分点，并非完全相同。以上历史测量不保证其他硬件、话务和配置的性能或精度。
---
@@ -172,14 +172,14 @@ FunASR 的 vLLM 集成将 ASR 模型拆分为两部分独立运行：
| 批处理 | 需手动 padding | Continuous Batching，自动调度 |
| CUDA 优化 | 无 | CUDA Graph + 算子融合 |
| 多卡并行 | 手动实现 | Tensor Parallel 一行配置 |
| 吞吐量 | RTFx ~20 | **RTFx 340+** |
| 已报告的 batch 吞吐量 | RTFx 21 | RTFx 340；范围见 Benchmark |
### 支持模型
| 模型 | LLM 部分 | audio encoder | vLLM 加速 |
| 模型 | LLM 部分 | audio encoder | 集成路径 |
|------|---------|---------------|-----------|
| **Fun-ASR-Nano** | Qwen3-0.6B | SenseVoice | ✓ 21.7x |
| **GLM-ASR-Nano** | Llama-2B | Whisper-like | ✓ 7.6x |
| **Fun-ASR-Nano** | Qwen3-0.6B | SenseVoice | 专用 split engine |
| **GLM-ASR-Nano** | Llama-2B | Whisper-like | 专用 split engine |
| LLMASR | Qwen/Vicuna | Whisper | ✓ |
| Paraformer | 无 LLM | — | ✗ 非自回归 |
| SenseVoice | 无 LLM | — | ✗ encoder-decoder |
@@ -263,7 +263,7 @@ FunASR 的 vLLM 集成将 ASR 模型拆分为两部分独立运行：
| 批处理 | 需手动 padding 对齐 | Continuous Batching 自动调度 |
| CUDA | 逐 sample 串行 | CUDA Graph + 算子融合 |
| 多卡 | 需手动实现 | Tensor Parallel 一行配置 |
| 结果 | RTFx ~20 | **RTFx 340+**（16倍加速） |
| 已报告的 batch 结果 | RTFx 21 | RTFx 340；需匹配硬件和计时范围 |
### 通用接口（推荐）
@@ -297,13 +297,15 @@ engine = FunASRNanoVLLM.from_pretrained(
)
results = engine.generate(
inputs="wav.scp", # 支持 scp/jsonl/文件列表
inputs=["audio1.wav", "audio2.wav"], # 已存在的音频文件，不是清单文件
hotwords=["开放时间"],
language="中文",
max_new_tokens=512,
)
```
直接引擎接收音频路径、音频路径列表或 16 kHz 波形数组/tensor，不展开 SCP/JSONL 清单。清单使用下方 [demo_vllm.py](../examples/industrial_data_pretraining/fun_asr_nano/demo_vllm.py)：SCP 每行为路径或 `key path`，JSONL 每行对象的 `source` 字段为音频路径。相对路径按进程工作目录解析。
### 命令行
```bash
@@ -455,24 +457,26 @@ CUDA_VISIBLE_DEVICES=0 python examples/industrial_data_pretraining/fun_asr_nano/
**响应**：
以下数值用于说明结构，不是实测结果。HTTP 时间戳和时长单位为秒；可选 `words`、`speaker` 字段取决于相应处理路径，见[服务与序列化实现](../examples/industrial_data_pretraining/fun_asr_nano/serve_vllm.py)。
```json
{
"text": "完整识别文本",
"text": "你好",
"segments": [
{
"text": "段文本",
"start": 1.7,
"end": 14.8,
"text": "你好",
"start": 0.3,
"end": 1.2,
"speaker": "SPK0",
"words": [
{"word": "砸", "start": 2.02, "end": 2.08},
{"word": "了", "start": 2.26, "end": 2.32}
{"word": "你", "start": 0.3, "end": 0.6},
{"word": "好", "start": 0.6, "end": 1.2}
]
}
],
"duration": 227.4,
"processing_time": 3.422,
"rtf": 0.015
"duration": 2.0,
"processing_time": 0.1,
"rtf": 0.05
}
```
@@ -520,17 +524,22 @@ const result = await resp.json();
**响应**（`verbose_json`）：
以下为结构完整的示例响应；时间戳单位为秒，不是识别质量保证。
```json
{
"task": "transcribe",
"language": "zh",
"duration": 5.17,
"text": "我一直没有照顾孩子，但是我想要抚养权。",
"duration": 2.0,
"text": "你好",
"segments": [
{
"id": 0, "start": 0.0, "end": 5.15,
"text": "我一直没有照顾孩子，但是我想要抚养权。",
"words": [{"word": "我", "start": 0.42, "end": 0.48}, ...]
"id": 0, "start": 0.3, "end": 1.2,
"text": "你好",
"words": [
{"word": "你", "start": 0.3, "end": 0.6},
{"word": "好", "start": 0.6, "end": 1.2}
]
}
]
}
@@ -573,10 +582,12 @@ curl -X POST http://localhost:8899/v1/audio/transcriptions \
**服务端 → 客户端**：
```json
下面每行是独立的 JSON WebSocket 消息，不是一个 JSON 文档。数值为示例，WebSocket 偏移单位为毫秒。
```text
{"event": "started"}
{"event": "language_set", "language": "中文"}
{"sentences": [{"text":"...","start":..,"end":..}], "is_final": true, "duration_ms": 5170}
{"sentences": [{"text": "你好", "start": 300, "end": 1200}], "is_final": true, "duration_ms": 2000}
{"event": "stopped"}
```
@@ -638,7 +649,7 @@ asyncio.run(offline_ws("audio.wav"))
- 基于 VAD 端点自然分句
- 确认段文字锁定不变，partial 实时更新
- 可选流式说话人分配（`--enable-spk`）+ STOP 时全局重聚类
- 首字延迟 ~480ms
- partial 解码间隔是调度参数，不是首字延迟保证；需按实际话务测量端到端延迟
### 6.2 启动服务
@@ -673,10 +684,12 @@ CUDA_VISIBLE_DEVICES=0 python examples/industrial_data_pretraining/fun_asr_nano/
**服务端 → 客户端**：
```json
示例消息序列，每个 JSON 对象对应一条 WebSocket 消息。偏移单位为毫秒，不是延迟实测记录。
```text
{"event": "started"}
{"sentences": [{"text":"你好","start":300,"end":1200}], "partial": "世界", "is_final": false}
{"sentences": [...], "is_final": true}
{"sentences": [{"text": "你好", "start": 300, "end": 1200}], "partial": "世界", "is_final": false}
{"sentences": [{"text": "你好", "start": 300, "end": 1200}, {"text": "世界", "start": 1500, "end": 2200}], "is_final": true}
{"event": "stopped"}
```
@@ -784,28 +797,43 @@ Fun-ASR-Nano 的声学编码器（SenseVoice）是**全上下文、非流式**
## 7. 动态 VAD
fsmn-vad 默认启用动态静音阈值。离线和流式使用不同配置。
动态静音属于 **VAD 阶段**，不是 ASR 解码参数。[FSMN-VAD](../funasr/models/fsmn_vad_streaming/model.py) 读取 `dynamic_silence` 和 `silence_schedule`；显式设置 `max_end_silence_time` 会关闭默认动态策略，除非另行覆盖。`AutoModelVLLM.generate(inputs, **kwargs)` 只转交给 ASR 引擎，不运行 VAD。
| 累积时长 | 离线（保留长段 ≤60s） | 流式（平衡延迟） |
|---------|-------------------|----------------|
| ≤ 5s | 2000ms | 2000ms |
| 5-10s | 2000ms | 1500ms |
| 10-15s | 1000ms | 1000ms |
| 15-20s | 1000ms | 800ms |
| 20-30s | 800ms | 800ms |
| 30-45s | 600ms | 400ms |
| 45-60s | 200-400ms | 100ms |
| > 60s | 100ms | 100ms |
下表在边界时长采样 SDK 的 `DEFAULT_SILENCE_SCHEDULE` 与 `STREAMING_SILENCE_SCHEDULE` 常量。每个 schedule 选择首个大于或等于累积语音时长的上限对应项；这些是静音阈值，不是语音段长上限。
离线倾向保留长段减少边界损失；流式更快收紧以降低延迟。
| 累积语音时长采样点 | DEFAULT_SILENCE_SCHEDULE | STREAMING_SILENCE_SCHEDULE |
| --- | --- | --- |
| 5000 ms | 2000 ms | 2000 ms |
| 10000 ms | 2000 ms | 1500 ms |
| 15000 ms | 1000 ms | 1000 ms |
| 20000 ms | 1000 ms | 800 ms |
| 30000 ms | 800 ms | 800 ms |
| 40000 ms | 600 ms | 400 ms |
| 45000 ms | 400 ms | 400 ms |
| 50000 ms | 400 ms | 100 ms |
| 60000 ms | 200 ms | 100 ms |
| 60001 ms | 100 ms | 100 ms |
分块输入不会自动选择名称带 streaming 的常量。服务包装层可能使用独立策略，例如 [DynamicStreamingVAD](../funasr/models/fsmn_vad_streaming/dynamic_vad.py) 自己维护 schedule，并以 `dynamic_silence=False` 调用底层 VAD。不要把此 SDK 表格当作所有服务的配置。
### 自定义
```python
model.generate(input="audio.wav", silence_schedule=[(5000,1500), (20000,800), (float('inf'),300)])
from funasr import AutoModel
vad = AutoModel(model="fsmn-vad", device="cpu", disable_update=True)
segments = vad.generate(
input="audio.wav",
cache={},
dynamic_silence=True,
silence_schedule=[(5000, 1500), (20000, 800), (float("inf"), 300)],
)
print(segments[0]["value"])
```
> GLM-ASR 不支持长段，使用时传 `dynamic_silence=False`。
将 `audio.wav` 替换为实际录音。此独立 VAD 调用在 `value` 中返回毫秒单位的 `[start_ms, end_ms]` 区间，不返回转写文本。接入 ASR 时，按引擎要求加载并重采样音频，依据毫秒区间切片，再将波形列表传给 `model.generate(inputs=audio_segments)`；合并结果时保留原始偏移。本示例只演示分段，不会自动把 VAD 连接到 vLLM。
> GLM-ASR 应先分段，并验证所选 checkpoint 的段长限制。需要固定静音阈值时，将 `dynamic_silence=False` 传给 **VAD** 调用，而不是 ASR 引擎；固定静音本身不限制最大段长。
---
@@ -828,14 +856,14 @@ model.generate(input="audio.wav", silence_schedule=[(5000,1500), (20000,800), (f
完整文件 → 离线（高吞吐）。麦克风/直播 → 流式（低延迟）。
**Q: GLM-ASR 用动态 VAD？**
不支持长段推理，用 `dynamic_silence=False`。
长录音应先分段，并针对所选 GLM checkpoint 验证段长。`dynamic_silence=False` 配置独立的 FSMN-VAD 阶段，不是 `AutoModelVLLM`；关闭动态静音本身不保证段长符合 ASR 限制。
**Q: SPK 性能影响？**
RTFx 102 → 46。CER 不变。默认关闭。
表中离线服务关闭/开启 SPK 的 RTFx 分别为 102/46，CER 分别为 8.14%/8.19%。SPK 默认关闭；这组测量不代表其他部署的开销或精度。
**Q: 二次开发入口？**
离线：`serve_vllm.process_audio()` / `FunASRNanoVLLM.generate()`
流式：`serve_realtime_ws.RealtimeASRSession`
**Q: 首次慢？**
vLLM 初始化 60-90s，之后即时。
模型加载、KV cache 分配及可选 CUDA Graph 预热都会影响首次启动。请分别测量冷启动与预热后推理，二者均没有固定耗时或即时返回保证。
model_zoo/huggingface_models.md
+9 -1
View File
@@ -1,7 +1,15 @@
# Pretrained Models on Huggingface
## Model License
- Apache License 2.0
The FunASR toolkit uses the [MIT software license](../LICENSE). Model weights
have separate terms: check the exact checkpoint's model card, license file and
revision before use or redistribution. The [FunASR model agreement](../MODEL_LICENSE)
applies only to models whose published terms adopt it, not every entry in this
catalogue. Third-party models, including OpenMOSS MOSS-Transcribe-Diarize, retain
their original authorship and model-specific licenses.
For model selection and deployment boundaries, start with the
[Model Zoo guide](./readme.md) and [MOSS integration](../docs/moss_transcribe_diarize.md).
## Model Zoo
Here we provided several pretrained models on different datasets. The details of models and datasets can be found on [ModelScope](https://www.modelscope.cn/models?page=1&tasks=auto-speech-recognition).
model_zoo/modelscope_models.md
+10 -2
View File
@@ -3,10 +3,18 @@
# Pretrained Models Released on ModelScope
## Model License
You are free to use, copy, modify, and share FunASR models under the conditions of this agreement. You should indicate the model source and author information when using, copying, modifying and sharing FunASR models. You should keep the relevant names of models in [FunASR software].. Full model license could see [license](https://github.com/modelscope/FunASR/blob/main/MODEL_LICENSE)
The FunASR toolkit uses the [MIT software license](../LICENSE). Model weights
have separate terms: check the exact checkpoint's model card, license file and
revision before use or redistribution. The [FunASR model agreement](../MODEL_LICENSE)
applies only to models whose published terms adopt it, not every entry in this
catalogue. Third-party models, including OpenMOSS MOSS-Transcribe-Diarize, retain
their original authorship and model-specific licenses.
## Model Usage
Ref to [docs](https://modelscope.github.io/FunASR/tutorial.html)
Start with [model selection](../docs/model_selection.md), the
[Python SDK tutorial](../docs/tutorial/README.md), and the
[deployment matrix](../docs/deployment_matrix.md). For MOSS, follow its
[dedicated adapter and serving guide](../docs/moss_transcribe_diarize.md).
## Model Zoo
Here we provided several pretrained models on different datasets. The details of models and datasets can be found on [ModelScope](https://www.modelscope.cn/models?page=1&tasks=auto-speech-recognition).
model_zoo/modelscope_models_zh.md
+8 -2
View File
@@ -3,10 +3,16 @@
# ModelScope上的预训练模型
## 模型许可协议
您可以在本协议的条件下自由使用、复制、修改和分享FunASR模型。在使用、复制、修改和分享FunASR模型时，您应当标明模型来源和作者信息。您应当在[FunASR软件]中保留相关模型的名称。完整的模型许可证请参见 [模型许可协议](https://github.com/modelscope/FunASR/blob/main/MODEL_LICENSE)
FunASR 工具包采用 [MIT 软件许可](../LICENSE)，模型权重的许可需单独核对。
使用或再分发前，请检查所选 checkpoint 的模型卡、许可文件及具体 revision。
[FunASR 模型许可协议](../MODEL_LICENSE) 仅适用于发布条款明确采用它的模型，
不能套用于本目录全部条目。OpenMOSS MOSS-Transcribe-Diarize 等第三方模型
保留原作者归属和各自的模型许可。
## 模型用法
模型用法参考[文档](../runtime/quick_start_zh.md)
从[模型选型](../docs/model_selection_zh.md)、[Python SDK 入门](../docs/tutorial/README_zh.md)
和[部署矩阵](../docs/deployment_matrix_zh.md)开始。MOSS 请使用
[专门的适配器与服务指南](../docs/moss_transcribe_diarize_zh.md)，不要直接套用其他模型的流水线。
## 模型仓库
这里我们提供了在不同数据集上预训练的模型。模型和数据集的详细信息可在 [ModelScope](https://www.modelscope.cn/models?page=1&tasks=auto-speech-recognition)中找到.
tests/test_reference_docs_contract.py
+56
View File
@@ -0,0 +1,56 @@
"""Legacy URLs must lead to maintained, model-specific instructions."""
from pathlib import Path
import re
import pytest
ROOT = Path(__file__).resolve().parents[1]
@pytest.mark.parametrize('source,targets', (
('docs/reference/build_task.md', ('../model_registration.md', '../training.md')),
('docs/reference/application.md', ('../moss_transcribe_diarize.md', '../use_case_showcase.md')),
('docs/reference/FQA.md', ('../troubleshooting.md', '../moss_transcribe_diarize.md',
'../installation/installation.md')),
))
def test_legacy_reference_pages_link_to_current_guides(source, targets):
path = ROOT / source
content = path.read_text()
links = re.findall(r'\]\(([^)]+)\)', content)
for target in targets:
assert target in links
assert (path.parent / target).is_file()
def test_removed_task_architecture_is_not_an_executable_tutorial():
content = (ROOT / 'docs/reference/build_task.md').read_text()
assert content.startswith('# Build custom tasks\n')
assert '```python' not in content
assert 'migration' in content.lower()
assert 'funasr/tasks/abs_task.py' not in content
def test_legacy_faq_preserves_model_boundaries():
content = (ROOT / 'docs/reference/FQA.md').read_text()
assert 'MOSS' in content
assert 'anonymous' in content
assert 'external `vad_model` or `spk_model`' in content
assert 'Use a model pipeline that includes both VAD and speaker models:' not in content
assert 'retry after removing the `funasr-cache` Docker volume' not in content
assert '## Speaker diarization has no speaker labels' in content
assert '## Can I use a speaker model other than cam++?' in content
@pytest.mark.parametrize('source', (
'model_zoo/huggingface_models.md',
'model_zoo/modelscope_models.md',
'model_zoo/modelscope_models_zh.md',
))
def test_hub_catalogues_do_not_assign_one_license_to_all_models(source):
content = (ROOT / source).read_text()
intro = content[:content.index('### ')]
assert '../MODEL_LICENSE' in intro
assert '../LICENSE' in intro
assert 'OpenMOSS' in intro
assert '- Apache License 2.0' not in intro
tests/test_sphinx_links.py
+88 -1
View File
@@ -34,6 +34,15 @@ GUIDE = '''# Link fixture
[Body root](../../../outside.txt)
[Body missing](../../examples/missing.py)
[Body missing fragment](../training.md#absent-section)
[Numbered heading](../training.md#1-installation--environment)
[Unicode heading](../training.md#安装或-import-失败)
[Encoded heading](../training.md#%E5%AE%89%E8%A3%85%E6%88%96-import-%E5%A4%B1%E8%B4%A5)
[Punctuation heading](../training.md#llamacpp-or-gguf)
[Repeated heading](../training.md#repeat-1)
[Local numbered heading](#2-local--section)
[Colliding heading](../training.md#collision-1)
## 2. Local & section
| Case | Destination |
| --- | --- |
@@ -54,6 +63,9 @@ GUIDE = '''# Link fixture
| missing | [Table missing](../../examples/table-missing.py) |
| escape | [Table escape](../../../outside.txt) |
| symlink | [Table symlink](../../examples/escape.txt) |
| numbered | [Table numbered](../training.md#1-installation--environment) |
| Unicode | [Table Unicode heading](../training.md#安装或-import-失败) |
| collision | [Table colliding heading](../training.md#collision-1) |
<a href="//docs.example.test/raw.md" data-label="do not alter">Raw cross-host</a>
<a href="/other-site/guide.md">Root-relative</a>
@@ -62,6 +74,7 @@ GUIDE = '''# Link fixture
<span id="local-anchor"></span>
<a href="../training.md#raw-ABSENT">Raw missing fragment</a>
<a href="#self-ABSENT">Raw missing self fragment</a>
<a href="../training.md#collision-1">Raw colliding heading</a>
```python
print("../examples/demo.py is code, not a link")
@@ -83,7 +96,15 @@ def built(tmp_path_factory):
for name in ('guide.md', 'guide_zh.md'):
(docs / 'tutorial' / name).write_text(GUIDE, encoding='utf-8')
(docs / 'training.md').write_text(
'# Training\n\n## Details\n\nReal destination.\n\n<span id="raw-anchor"></span>\n')
'# Training\n\n## Details\n\nReal destination.\n\n<span id="raw-anchor"></span>\n'
'\n## 1. Installation & Environment\n\nInstall.\n'
'\n## 安装或 `import` 失败\n\nDiagnose.\n'
'\n## llama.cpp or GGUF\n\nRuntime.\n'
'\n## Repeat\n\nFirst.\n\n## Repeat\n\nSecond.\n'
'\n## Collision\n\nFirst collision.\n\n## Collision\n\nSecond collision.\n'
'\n## Collision-1\n\nThird collision.\n'
'\n<span id="rawcollision"></span>\n\n## Raw.Collision\n\nRaw collision.\n',
encoding='utf-8')
(docs / 'reference.rst').write_text('Reference\n=========\n\nExisting RST page.\n')
# Sphinx reads this target after tutorial/*, so early doctree access fails.
(docs / 'zz_later.md').write_text('# Later target\n\n## Details\n\nLate-read destination.\n')
@@ -198,6 +219,72 @@ def test_missing_targets_and_fragments_still_warn(built):
assert page.find('a', string='Table symlink')['href'] == '../../examples/escape.txt'
@pytest.mark.parametrize('language', ('guide', 'guide_zh'))
def test_source_heading_fragments_remain_clickable(built, language):
from urllib.parse import unquote
output, pages, log = built
target = BeautifulSoup((output / 'training.html').read_text(), 'html.parser')
for label, fragment in (
('Numbered heading', '1-installation--environment'),
('Unicode heading', '安装或-import-失败'),
('Encoded heading', '安装或-import-失败'),
('Punctuation heading', 'llamacpp-or-gguf'),
('Repeated heading', 'repeat-1'),
('Table numbered', '1-installation--environment'),
('Table Unicode heading', '安装或-import-失败'),
):
link = pages[language].find('a', string=label)
assert link is not None, label
assert unquote(link['href']) == '../training.html#' + fragment
assert target.find(id=fragment), fragment
assert not any(fragment in line for line in log.splitlines() if 'WARNING:' in line)
local = pages[language].find('a', string='Local numbered heading')
assert local is not None
assert local['href'] == '#2-local--section'
assert pages[language].find(id='2-local--section')
# Existing Sphinx IDs are public URLs too; aliases must not replace them.
assert target.find(id='details')
assert target.find(id='llama-cpp-or-gguf')
ids = [element['id'] for element in target.select('[id]')]
assert len(ids) == len(set(ids))
@pytest.mark.parametrize('language', ('guide', 'guide_zh'))
@pytest.mark.parametrize('label', ('Colliding heading', 'Table colliding heading', 'Raw colliding heading'))
def test_colliding_alias_links_reach_the_intended_heading(built, language, label):
output, pages, log = built
page = BeautifulSoup((output / 'training.html').read_text(), 'html.parser')
fragment = pages[language].find('a', string=label)['href'].split('#')[1]
target = page.find(id=fragment)
section = target if target.name == 'section' else target.find_parent('section')
assert 'Second collision.' in section.get_text()
assert 'Third collision.' not in section.get_text()
assert len(page.select('[id="rawcollision"]')) == 1
assert 'rawcollision' in log and '[repository_links.collision]' in log
assert page.find(id='collision-1'), 'Do not remove existing Sphinx URLs'
@pytest.mark.parametrize('builder', ('text', 'latex'))
def test_non_html_builders_keep_sphinx_fragment_resolution(tmp_path, builder):
docs = tmp_path / 'repo' / 'docs'
docs.mkdir(parents=True)
shutil.copy2(ROOT / 'docs/conf.py', docs / 'conf.py')
shutil.copytree(ROOT / 'docs/_ext', docs / '_ext')
(docs / 'index.rst').write_text('Fixture\n=======\n\n.. toctree::\n\n guide\n target\n')
(docs / 'guide.md').write_text('# Guide\n\n[Valid](target.md#details)\n\n[Missing](target.md#ABSENT)\n')
(docs / 'target.md').write_text('# Target\n\n## Details\n\nDestination.\n')
result = subprocess.run(
[sys.executable, '-m', 'sphinx', '-b', builder, '-n', '-E',
str(docs), str(tmp_path / 'output')],
text=True, capture_output=True, timeout=90,
env={**os.environ, 'PYTHONDONTWRITEBYTECODE': '1'},
)
assert result.returncode == 0, result.stdout + result.stderr
assert 'ABSENT' in result.stderr or 'absent' in result.stderr
assert 'WARNING' in result.stderr
def test_faq_points_to_the_existing_speaker_contract():
source = (ROOT / 'docs/reference/FQA.md').read_text()
assert 'speaker-verification--diarization-eres2netv2' not in source
tests/test_vllm_examples_contract.py
+270
View File
@@ -0,0 +1,270 @@
"""Check documented contracts against source without importing model runtimes."""
import ast
import hashlib
import inspect
import json
import re
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1]
GUIDES = ("vllm_guide.md", "vllm_guide_zh.md", "vllm_guide_zh_v2.md")
NANO = "funasr/models/fun_asr_nano/inference_vllm.py"
AUTO = "funasr/auto/auto_model_vllm.py"
VAD = "funasr/models/fsmn_vad_streaming/model.py"
SERVER = "examples/industrial_data_pretraining/fun_asr_nano/serve_vllm.py"
def blocks(text, language):
return re.findall(rf"^```{language}\n(.*?)^```", text, re.M | re.S)
@pytest.fixture(params=GUIDES)
def guide(request):
return request.param, (ROOT / "docs" / request.param).read_text()
def source_function(path, name, class_name=None):
tree = ast.parse((ROOT / path).read_text())
if class_name:
tree = next(
node
for node in tree.body
if isinstance(node, ast.ClassDef) and node.name == class_name
)
return next(
node
for node in tree.body
if isinstance(node, ast.FunctionDef) and node.name == name
)
def signature(path, class_name):
function = source_function(path, "generate", class_name)
function.body = [ast.Pass()]
function.decorator_list = []
function.returns = None
for argument in ast.walk(function.args):
if isinstance(argument, ast.arg):
argument.annotation = None
namespace = {}
exec(
compile(
ast.fix_missing_locations(ast.Module(body=[function], type_ignores=[])),
path,
"exec",
),
namespace,
)
return inspect.signature(namespace["generate"])
def generate_calls(text):
kinds = {}
for block in blocks(text, "python"):
tree = ast.parse(block)
for node in ast.walk(tree):
if not isinstance(node, ast.Assign) or not isinstance(node.value, ast.Call):
continue
constructor = node.value.func
if isinstance(constructor, ast.Attribute):
constructor = constructor.value
if not isinstance(constructor, ast.Name):
continue
kind = constructor.id
if kind == "AutoModel":
model = next(
(kw.value for kw in node.value.keywords if kw.arg == "model"), None
)
kind = (
"vad"
if isinstance(model, ast.Constant) and model.value == "fsmn-vad"
else kind
)
for target in node.targets:
if isinstance(target, ast.Name):
kinds[target.id] = kind
for node in ast.walk(tree):
if (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and node.func.attr == "generate"
and isinstance(node.func.value, ast.Name)
):
yield kinds.get(node.func.value.id), node
def test_generate_examples_bind_to_actual_signatures(guide):
_, text = guide
signatures = {
"AutoModelVLLM": signature(AUTO, "AutoModelVLLM"),
"FunASRNanoVLLM": signature(NANO, "FunASRNanoVLLM"),
"vad": signature("funasr/auto/auto_model.py", "AutoModel"),
}
checked = 0
for kind, call in generate_calls(text):
if kind in signatures:
signatures[kind].bind(
object(),
*[object() for _ in call.args],
**{kw.arg: object() for kw in call.keywords if kw.arg},
)
checked += 1
assert checked >= 3
def test_manifest_is_a_cli_contract_not_a_direct_engine_input(guide):
_, text = guide
for kind, call in generate_calls(text):
if kind not in {"AutoModelVLLM", "FunASRNanoVLLM"}:
continue
inputs = list(call.args) + [
kw.value for kw in call.keywords if kw.arg == "inputs"
]
for value in inputs:
for node in ast.walk(value):
if isinstance(node, ast.Constant) and isinstance(node.value, str):
assert not node.value.endswith((".scp", ".jsonl")), ast.unparse(
call
)
assert "demo_vllm.py --input wav.scp" in text
demo = (
ROOT / "examples/industrial_data_pretraining/fun_asr_nano/demo_vllm.py"
).read_text()
assert 'args.input.endswith(".scp")' in demo
assert 'args.input.endswith(".jsonl")' in demo
assert 'audio_files.append(item["source"])' in demo
assert "load_audio_text_image_video(audio_input" in (ROOT / NANO).read_text()
def test_dynamic_options_are_sent_to_the_actual_vad_consumer(guide):
_, text = guide
options = {"silence_schedule", "dynamic_silence"}
seen = set()
for kind, call in generate_calls(text):
specified = options.intersection(kw.arg for kw in call.keywords)
if specified:
assert kind == "vad", "VAD options do not configure the ASR engine"
seen.update(specified)
assert seen == options
inference = source_function(VAD, "inference", "FsmnVADStreaming")
consumed = {
node.args[0].value
for node in ast.walk(inference)
if isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and node.func.attr == "get"
and node.args
and isinstance(node.args[0], ast.Constant)
}
assert options <= consumed
def test_dynamic_schedule_table_matches_sdk_constants(guide):
_, text = guide
tree = ast.parse((ROOT / VAD).read_text())
schedules = {}
for node in tree.body:
if (
isinstance(node, ast.Assign)
and isinstance(node.targets[0], ast.Name)
and node.targets[0].id
in {"DEFAULT_SILENCE_SCHEDULE", "STREAMING_SILENCE_SCHEDULE"}
):
schedules[node.targets[0].id] = eval(
compile(ast.Expression(node.value), VAD, "eval"),
{"__builtins__": {}, "float": float},
)
rows = re.findall(r"^\| (\d+) ms \| (\d+) ms \| (\d+) ms \|$", text, re.M)
assert len(rows) >= 9
for duration, offline, streaming in rows:
for column, name in [
(offline, "DEFAULT_SILENCE_SCHEDULE"),
(streaming, "STREAMING_SILENCE_SCHEDULE"),
]:
expected = next(
threshold
for limit, threshold in schedules[name]
if int(duration) <= limit
)
assert int(column) == expected
def test_json_examples_are_valid_and_verbose_output_matches_serializer(guide):
_, text = guide
examples = [json.loads(block) for block in blocks(text, "json")]
assert len(examples) >= 2
function = source_function(SERVER, "build_openai_verbose_json")
namespace = {}
exec(
compile(ast.Module(body=[function], type_ignores=[]), SERVER, "exec"), namespace
)
verbose = next(
example for example in examples if example.get("task") == "transcribe"
)
result = {key: verbose[key] for key in ("text", "duration", "segments")}
assert (
namespace["build_openai_verbose_json"](result, language=verbose["language"])
== verbose
)
for example in examples:
for segment in example.get("segments", []):
assert 0 <= segment["start"] <= segment["end"] <= example["duration"]
for word in segment.get("words", []):
assert (
segment["start"] <= word["start"] <= word["end"] <= segment["end"]
)
def test_websocket_examples_are_individual_messages(guide):
_, text = guide
streams = [
block for block in blocks(text, "text") if '{"event": "started"}' in block
]
assert len(streams) == 2
for stream in streams:
events = [json.loads(line) for line in stream.splitlines() if line.strip()]
assert events[0] == {"event": "started"}
assert events[-1] == {"event": "stopped"}
final = next(event for event in events if event.get("is_final"))
assert final["sentences"]
for sentence in final["sentences"]:
assert isinstance(sentence["text"], str) and sentence["text"] != "..."
assert 0 <= sentence["start"] <= sentence["end"]
def test_performance_claims_distinguish_throughput_and_measured_accuracy(guide):
_, text = guide
assert "340 / 21" in text and "16.2" in text
assert "8.20%" in text and "8.06%" in text and "0.14" in text
assert "8.14%" in text and "8.19%" in text
assert "benchmark/rtf_reproducibility.md" in text
for stale in (
"16-340x",
"16–340x",
"21.7x",
"7.6x",
"RTFx 340+",
"CER 完全一致",
"CER exactly",
"CER 不变",
"CER is unchanged",
"Subsequent inferences are instant",
"之后即时",
):
assert stale not in text
def test_historical_heading_anchors_are_preserved(guide):
name, text = guide
prose = re.sub(r"^```[^\n]*\n.*?^```\s*$", "", text, flags=re.M | re.S)
headings = re.findall(r"^#{1,6} .+$", prose, re.M)
expected = (
"678bb9b4403454e8773e1470e7e90e9ed2cf2500266eaa06c58b6dda20b48134"
if name == "vllm_guide.md"
else "8b56033dfcfc2bf80e12cc8fa272a313311a1a0730f8f5c0c73fd95e8c56018f"
)
assert hashlib.sha256("\n".join(headings).encode()).hexdigest() == expected