mirror of
https://github.com/modelscope/FunASR.git
synced 2026-10-12 02:19:18 +00:00
docs: add Chinese (zh/) and Japanese (ja/) versions of all pages with language switcher
This commit is contained in:
+6
-1
@@ -18,7 +18,12 @@
|
||||
<a href="model-registration.html">Develop</a>
|
||||
<a href="api.html">API</a>
|
||||
</div>
|
||||
<a href="https://github.com/modelscope/FunASR" class="nav-github">GitHub</a>
|
||||
<div class="lang-switch">
|
||||
<a href="index.html" title="English">EN</a>
|
||||
<a href="zh/index.html" title="中文">中</a>
|
||||
<a href="ja/index.html" title="日本語">JP</a>
|
||||
</div>
|
||||
<a href="https://github.com/modelscope/FunASR" class="nav-github">GitHub</a>
|
||||
</div>
|
||||
</nav>
|
||||
|
||||
|
||||
+256
@@ -0,0 +1,256 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="ja">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>FunASR - End-to-End Speech Recognition Toolkit</title>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700;800&display=swap" rel="stylesheet">
|
||||
<link rel="stylesheet" href="../style.css">
|
||||
</head>
|
||||
<body>
|
||||
<nav class="nav">
|
||||
<div class="container">
|
||||
<a href="index.html" class="nav-logo">FunASR</a>
|
||||
<div class="nav-links">
|
||||
<a href="index.html" class="active">ホーム</a>
|
||||
<a href="tutorial.html">チュートリアル</a>
|
||||
<a href="training.html">トレーニング</a>
|
||||
<a href="model-registration.html">開発</a>
|
||||
<a href="../api.html">API</a>
|
||||
</div>
|
||||
<div class="lang-switch">
|
||||
<a href="index.html" title="English">EN</a>
|
||||
<a href="zh/index.html" title="中文">中</a>
|
||||
<a href="ja/index.html" title="日本語">JP</a>
|
||||
</div>
|
||||
<div class="lang-switch">
|
||||
<a href="../index.html" title="English">EN</a>
|
||||
<a href="../zh/index.html" title="中文">中</a>
|
||||
<a href="../ja/index.html" title="日本語">JP</a>
|
||||
</div>
|
||||
<a href="https://github.com/modelscope/FunASR" class="nav-github">GitHub</a>
|
||||
</div>
|
||||
</nav>
|
||||
|
||||
<div class="hero">
|
||||
<div class="container hero-content">
|
||||
<div class="hero-copy">
|
||||
<p class="eyebrow">Open-source speech understanding toolkit</p>
|
||||
<h1>FunASR</h1>
|
||||
<p>Production-ready ASR, VAD, punctuation, speaker diarization, emotion detection, and audio event recognition with one unified Python interface.</p>
|
||||
<div class="hero-badges">
|
||||
<a href="https://pypi.org/project/funasr/"><img src="https://img.shields.io/pypi/v/funasr?style=flat-square&color=2563eb" alt="PyPI"></a>
|
||||
<a href="https://github.com/modelscope/FunASR/stargazers"><img src="https://img.shields.io/github/stars/modelscope/FunASR?style=flat-square&color=b45309" alt="Stars"></a>
|
||||
<a href="https://github.com/modelscope/FunASR/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-MIT-059669?style=flat-square" alt="License"></a>
|
||||
</div>
|
||||
<div class="hero-actions">
|
||||
<a href="tutorial.html" class="btn btn-white">始める</a>
|
||||
<a href="training.html" class="btn btn-outline-white">Training guide</a>
|
||||
<a href="../api.html" class="btn btn-outline-white">API reference</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="hero-panel" aria-label="FunASR recognition preview">
|
||||
<div class="terminal-bar">
|
||||
<span class="terminal-dots"><span></span><span></span><span></span></span>
|
||||
<span>funasr-pipeline.py</span>
|
||||
</div>
|
||||
<pre>from funasr import AutoModel
|
||||
|
||||
model = AutoModel(
|
||||
model="paraformer-zh",
|
||||
vad_model="fsmn-vad",
|
||||
punc_model="ct-punc",
|
||||
spk_model="cam++",
|
||||
)
|
||||
res = model.generate(input="meeting.wav")
|
||||
print(res[0]["sentence_info"])</pre>
|
||||
<div class="hero-metrics">
|
||||
<div><strong>50+</strong><span>languages</span></div>
|
||||
<div><strong>15x</strong><span>faster</span></div>
|
||||
<div><strong>1 API</strong><span>pipeline</span></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<section class="docs-band">
|
||||
<div class="container">
|
||||
<h2 class="section-title">Documentation</h2>
|
||||
<p class="section-subtitle">Start with examples, tune on your own data, extend the registry, or jump into source-linked API docs.</p>
|
||||
<div class="doc-grid">
|
||||
<a class="doc-card" href="tutorial.html">
|
||||
<span class="doc-kicker">Learn</span>
|
||||
<h3>チュートリアル</h3>
|
||||
<p>Install FunASR, choose a model, and run common ASR, VAD, diarization, and export flows.</p>
|
||||
</a>
|
||||
<a class="doc-card" href="training.html">
|
||||
<span class="doc-kicker">Tune</span>
|
||||
<h3>トレーニング</h3>
|
||||
<p>Prepare JSONL data, fine-tune Paraformer, SenseVoice, and Fun-ASR-Nano, then monitor runs.</p>
|
||||
</a>
|
||||
<a class="doc-card" href="model-registration.html">
|
||||
<span class="doc-kicker">Extend</span>
|
||||
<h3>Developer Guide</h3>
|
||||
<p>Understand the registry, add a model, package remote code, and avoid integration pitfalls.</p>
|
||||
</a>
|
||||
<a class="doc-card" href="../api.html">
|
||||
<span class="doc-kicker">Reference</span>
|
||||
<h3>API</h3>
|
||||
<p>Browse generated classes, methods, source previews, and GitHub line links.</p>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="container">
|
||||
<h2 class="section-title">機能</h2>
|
||||
<p class="section-subtitle">Everything needed for speech understanding, from raw audio segmentation to speaker-aware transcripts.</p>
|
||||
<div class="grid-3">
|
||||
<div class="card">
|
||||
<div class="card-icon">ASR</div>
|
||||
<h3>音声認識</h3>
|
||||
<p>Streaming and offline ASR with VAD segmentation. Process long-form audio with a single API call.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div class="card-icon">50+</div>
|
||||
<h3>Multilingual Models</h3>
|
||||
<p>Fun-ASR-Nano covers 31 languages and Qwen3-ASR covers 52 languages with language detection.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div class="card-icon">SPK</div>
|
||||
<h3>話者分離</h3>
|
||||
<p>Identify who spoke when, then attach speaker labels to sentence-level ASR output.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div class="card-icon">SFX</div>
|
||||
<h3>感情・イベント</h3>
|
||||
<p>SenseVoice detects emotion and audio events including background music, applause, laughter, and crying.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div class="card-icon">RTF</div>
|
||||
<h3>Low Latency</h3>
|
||||
<p>Non-autoregressive models support fast batch and realtime workloads across common deployment targets.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div class="card-icon">OPS</div>
|
||||
<h3>トレーニング・デプロイ</h3>
|
||||
<p>Fine-tune with DeepSpeed, export to ONNX, and deploy through Docker runtime or the Python SDK.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="alt">
|
||||
<div class="container">
|
||||
<h2 class="section-title">モデル</h2>
|
||||
<p class="section-subtitle">Pre-trained industrial models ready for recognition, segmentation, and speech understanding workflows.</p>
|
||||
<div class="grid-2">
|
||||
<div class="card">
|
||||
<h3>Fun-ASR-Nano</h3>
|
||||
<p>End-to-end ASR trained on tens of millions of hours. 31 languages, dialects, accents, lyrics, timestamps, and hotwords.</p>
|
||||
<div class="card-tags">
|
||||
<span class="tag">800M</span>
|
||||
<span class="tag">31 Languages</span>
|
||||
<span class="tag">Timestamps</span>
|
||||
<span class="tag">話者分離</span>
|
||||
<span class="tag">Hotwords</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h3>Paraformer</h3>
|
||||
<p>Non-autoregressive Chinese and English ASR with streaming and offline variants for production systems.</p>
|
||||
<div class="card-tags">
|
||||
<span class="tag">Streaming</span>
|
||||
<span class="tag">Offline</span>
|
||||
<span class="tag">Hotwords</span>
|
||||
<span class="tag">ONNX</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h3>SenseVoice</h3>
|
||||
<p>Multi-task speech understanding for ASR, language ID, emotion, and audio events across five languages.</p>
|
||||
<div class="card-tags">
|
||||
<span class="tag">Emotion</span>
|
||||
<span class="tag">Audio Events</span>
|
||||
<span class="tag">話者分離</span>
|
||||
<span class="tag">70ms/10s</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h3>Qwen3-ASR</h3>
|
||||
<p>LLM-based ASR with 52 languages, contextual understanding, and automatic language detection.</p>
|
||||
<div class="card-tags">
|
||||
<span class="tag">52 Languages</span>
|
||||
<span class="tag">0.6B / 1.7B</span>
|
||||
<span class="tag">Context-aware</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="container narrow">
|
||||
<h2 class="section-title">クイックスタート</h2>
|
||||
<p class="section-subtitle">Install the package, compose the pipeline, and run recognition from Python.</p>
|
||||
<pre>pip install funasr
|
||||
# Or latest: pip install git+https://github.com/modelscope/FunASR.git</pre>
|
||||
<pre>from funasr import AutoModel
|
||||
|
||||
model = AutoModel(
|
||||
model="paraformer-zh",
|
||||
vad_model="fsmn-vad",
|
||||
punc_model="ct-punc",
|
||||
spk_model="cam++",
|
||||
)
|
||||
res = model.generate(input="meeting.wav", batch_size_s=300)
|
||||
|
||||
for sent in res[0]["sentence_info"]:
|
||||
print(f"[Speaker {sent['spk']}] {sent['text']}")</pre>
|
||||
<div class="section-cta">
|
||||
<a href="tutorial.html" class="btn btn-primary">Full tutorial</a>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="alt">
|
||||
<div class="container">
|
||||
<h2 class="section-title">エコシステム</h2>
|
||||
<p class="section-subtitle">Related projects around ASR, speech understanding, video clipping, and voice generation.</p>
|
||||
<div class="grid-2">
|
||||
<div class="card">
|
||||
<h3><a class="card-title-link" href="https://github.com/FunAudioLLM/Fun-ASR">Fun-ASR-Nano</a></h3>
|
||||
<p>The latest ASR large model with multilingual recognition, timestamps, speaker diarization, and hotwords.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h3><a class="card-title-link" href="https://github.com/FunAudioLLM/SenseVoice">SenseVoice</a></h3>
|
||||
<p>Multi-task speech understanding for ASR, emotion detection, and audio event recognition.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h3><a class="card-title-link" href="https://github.com/modelscope/FunClip">FunClip</a></h3>
|
||||
<p>AI video clipping powered by FunASR and LLM-assisted editing workflows.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h3><a class="card-title-link" href="https://github.com/FunAudioLLM/CosyVoice">CosyVoice</a></h3>
|
||||
<p>Natural speech generation with multi-language, timbre, and emotion control.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<footer>
|
||||
<p>FunASR · Tongyi Lab, Alibaba Group</p>
|
||||
<p>
|
||||
<a href="https://github.com/modelscope/FunASR">GitHub</a> ·
|
||||
<a href="training.html">トレーニング</a> ·
|
||||
<a href="../api.html">API</a> ·
|
||||
<a href="https://github.com/modelscope/FunASR/blob/main/MODEL_LICENSE">License</a> ·
|
||||
<a href="https://pypi.org/project/funasr/">PyPI</a> ·
|
||||
<a href="https://github.com/modelscope/FunASR/issues">Issues</a>
|
||||
</p>
|
||||
</footer>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,410 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="ja"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0">
|
||||
<title>FunASR Developer Guide</title>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700;800&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet">
|
||||
<link rel="stylesheet" href="../style.css">
|
||||
</head><body>
|
||||
<nav class="nav"><div class="container">
|
||||
<a href="index.html" class="nav-logo">FunASR</a>
|
||||
<div class="nav-links">
|
||||
<a href="index.html">ホーム</a>
|
||||
<a href="tutorial.html">チュートリアル</a>
|
||||
<a href="training.html">トレーニング</a>
|
||||
<a href="model-registration.html" class="active">開発</a>
|
||||
<a href="../api.html">API</a>
|
||||
</div>
|
||||
<div class="lang-switch">
|
||||
<a href="model-registration.html" title="English">EN</a>
|
||||
<a href="zh/model-registration.html" title="中文">中</a>
|
||||
<a href="ja/model-registration.html" title="日本語">JP</a>
|
||||
</div>
|
||||
<div class="lang-switch">
|
||||
<a href="../model-registration.html" title="English">EN</a>
|
||||
<a href="../zh/model-registration.html" title="中文">中</a>
|
||||
<a href="../ja/model-registration.html" title="日本語">JP</a>
|
||||
</div>
|
||||
<a href="https://github.com/modelscope/FunASR" class="nav-github">GitHub</a>
|
||||
</div></nav>
|
||||
|
||||
<div class="content"><div class="container narrow">
|
||||
<h1>Developer Guide</h1>
|
||||
<p>From understanding the architecture to contributing your own model — a step-by-step guide.</p>
|
||||
|
||||
<div class="toc-grid">
|
||||
<a href="#architecture">1. Architecture Overview</a>
|
||||
<a href="#setup">2. Development Setup</a>
|
||||
<a href="#understand">3. How Inference Works</a>
|
||||
<a href="#add-model">4. Add a New Model</a>
|
||||
<a href="#add-component">5. Add Frontend/Tokenizer</a>
|
||||
<a href="#standalone">6. Standalone Repository</a>
|
||||
<a href="#testing">7. Testing Your Model</a>
|
||||
<a href="#pitfalls">8. Common Pitfalls</a>
|
||||
<a href="#contribute">9. Contributing</a>
|
||||
</div>
|
||||
|
||||
<!-- 1. Architecture -->
|
||||
<h2 id="architecture">1. Architecture Overview</h2>
|
||||
<p>FunASR is built around three core ideas: a <strong>registry</strong> for component discovery, <strong>AutoModel</strong> as the unified entry point, and <strong>config.yaml</strong> as the declarative model definition.</p>
|
||||
|
||||
<div class="arch-box">User Code FunASR Framework Model Hub
|
||||
───────── ──────────────── ─────────
|
||||
AutoModel(model="name")
|
||||
│
|
||||
├─→ download_model() ──────→ ModelScope / HuggingFace
|
||||
│ ↓ ↓
|
||||
│ read config.yaml download model.pt
|
||||
│ ↓
|
||||
├─→ tables.model_classes["Name"] ← @tables.register decorator
|
||||
│ ↓
|
||||
├─→ model_class(**config) ← __init__: build encoder/decoder
|
||||
│ ↓
|
||||
├─→ load_pretrained_model() ← load weights from model.pt
|
||||
│ ↓
|
||||
└─→ model.eval() ← ready for inference
|
||||
|
||||
generate(input="audio.wav")
|
||||
│
|
||||
├─ No VAD → inference() ← single utterance
|
||||
│ ↓
|
||||
│ model.inference(data_in, tokenizer, frontend, **kwargs)
|
||||
│ ↓
|
||||
│ return [{"key", "text", "timestamp", ...}]
|
||||
│
|
||||
└─ With VAD → inference_with_vad() ← long audio
|
||||
↓
|
||||
1. VAD: segment audio → [[start_ms, end_ms], ...]
|
||||
2. Sort segments by length (for efficient batching)
|
||||
3. ASR: recognize each segment
|
||||
4. Merge timestamps (add VAD offset)
|
||||
5. Punctuation (optional)
|
||||
6. Speaker diarization (optional)
|
||||
↓
|
||||
return [{"key", "text", "timestamp", "sentence_info"}]
|
||||
</div>
|
||||
|
||||
<h3>Registry System</h3>
|
||||
<p>Every component in FunASR is registered by name. The registry is the lookup table that connects config strings to Python classes:</p>
|
||||
|
||||
<table>
|
||||
<tr><th>Registry</th><th>Purpose</th><th>Example</th></tr>
|
||||
<tr><td><code>model_classes</code></td><td>ASR, VAD, PUNC, SPK models</td><td>"Paraformer", "FsmnVADStreaming"</td></tr>
|
||||
<tr><td><code>encoder_classes</code></td><td>Encoder architectures</td><td>"SANMEncoder", "ConformerEncoder"</td></tr>
|
||||
<tr><td><code>decoder_classes</code></td><td>Decoder architectures</td><td>"ParaformerSANMDecoder"</td></tr>
|
||||
<tr><td><code>frontend_classes</code></td><td>Audio feature extraction</td><td>"WavFrontend", "WhisperFrontend"</td></tr>
|
||||
<tr><td><code>tokenizer_classes</code></td><td>Text tokenization</td><td>"SentencepiecesTokenizer"</td></tr>
|
||||
<tr><td><code>dataset_classes</code></td><td>Training data loading</td><td>"AudioDataset"</td></tr>
|
||||
</table>
|
||||
|
||||
<pre>from funasr.register import tables
|
||||
|
||||
# Register a new model
|
||||
@tables.register("model_classes", "MyModel")
|
||||
class MyModel(nn.Module):
|
||||
...
|
||||
|
||||
# View all registered models
|
||||
tables.print("model")</pre>
|
||||
|
||||
<!-- 2. Setup -->
|
||||
<h2 id="setup">2. Development Setup</h2>
|
||||
|
||||
<div class="step">
|
||||
<h4>Clone and install in development mode</h4>
|
||||
<pre>git clone https://github.com/modelscope/FunASR.git
|
||||
cd FunASR
|
||||
pip install -e . # editable install
|
||||
pip install -e ".[train]" # with training dependencies</pre>
|
||||
</div>
|
||||
|
||||
<div class="step">
|
||||
<h4>Verify installation</h4>
|
||||
<pre>python -c "from funasr import AutoModel; print('OK')"
|
||||
python -c "from funasr.register import tables; tables.print('model')"</pre>
|
||||
</div>
|
||||
|
||||
<div class="step">
|
||||
<h4>Run existing tests</h4>
|
||||
<pre># Quick smoke test
|
||||
python tests_models/test_fsmn_vad.py
|
||||
python tests_models/test_paraformer.py
|
||||
|
||||
# Full test suite
|
||||
cd tests_models && python run_all_tests.py</pre>
|
||||
</div>
|
||||
|
||||
<!-- 3. How Inference Works -->
|
||||
<h2 id="understand">3. How Inference Works (Deep Dive)</h2>
|
||||
|
||||
<p>Understanding the inference data flow is essential before adding a new model. Here's what happens when you call <code>model.generate(input="audio.wav")</code>:</p>
|
||||
|
||||
<h4>Step 1: Input Preparation</h4>
|
||||
<p><code>prepare_data_iterator()</code> normalizes any input type (file path, URL, numpy, bytes, list) into a uniform <code>(key_list, data_list)</code> format.</p>
|
||||
|
||||
<h4>Step 2: Model Inference</h4>
|
||||
<p>Each model's <code>inference()</code> method receives:</p>
|
||||
<pre>def inference(self, data_in, data_lengths=None, key=None,
|
||||
tokenizer=None, frontend=None, **kwargs):
|
||||
# data_in: list of audio samples (numpy arrays)
|
||||
# tokenizer: for decoding token IDs → text
|
||||
# frontend: for extracting fbank features
|
||||
# **kwargs: all config.yaml params + user runtime params
|
||||
|
||||
# Must return: (results_list, meta_data_dict)
|
||||
return [{"key": "id", "text": "hello", "timestamp": [...]}], {"batch_data_time": 5.5}</pre>
|
||||
|
||||
<h4>Step 3: Output Format</h4>
|
||||
<p>The <code>results_list</code> must be a list of dicts. Required/optional fields:</p>
|
||||
<table>
|
||||
<tr><th>Field</th><th>Type</th><th>Required</th><th>Description</th></tr>
|
||||
<tr><td><code>key</code></td><td>str</td><td>Yes</td><td>Sample identifier</td></tr>
|
||||
<tr><td><code>text</code></td><td>str</td><td>Yes (ASR)</td><td>Recognized text</td></tr>
|
||||
<tr><td><code>timestamp</code></td><td>list</td><td>For SPK</td><td><code>[[start_ms, end_ms], ...]</code> per character</td></tr>
|
||||
<tr><td><code>value</code></td><td>list</td><td>VAD only</td><td><code>[[start_ms, end_ms], ...]</code> speech segments</td></tr>
|
||||
<tr><td><code>spk_embedding</code></td><td>Tensor</td><td>SPK only</td><td>Shape <code>[N, 192]</code></td></tr>
|
||||
</table>
|
||||
|
||||
<div class="warning">
|
||||
<strong>Timestamp format matters!</strong> If your model outputs timestamps differently (e.g., dict format like Fun-ASR-Nano), FunASR's <code>inference_with_vad</code> will auto-convert it. But the standard expected format is <code>[[start_ms, end_ms], ...]</code> (list of 2-element lists in milliseconds).
|
||||
</div>
|
||||
|
||||
<!-- 4. Add a New Model -->
|
||||
<h2 id="add-model">4. Add a New Model</h2>
|
||||
|
||||
<div class="step">
|
||||
<h4>Create model directory</h4>
|
||||
<pre>funasr/models/my_model/
|
||||
├── __init__.py # empty file
|
||||
├── model.py # main model class
|
||||
├── encoder.py # (optional) custom encoder
|
||||
└── decoder.py # (optional) custom decoder</pre>
|
||||
<div class="warning"><strong>Rule:</strong> Each model directory is self-contained. Never import from other model directories. Never modify existing models.</div>
|
||||
</div>
|
||||
|
||||
<div class="step">
|
||||
<h4>Implement the model class</h4>
|
||||
<pre>import torch.nn as nn
|
||||
from funasr.register import tables
|
||||
|
||||
@tables.register("model_classes", "MyModel")
|
||||
class MyModel(nn.Module):
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__() # ← MUST call super().__init__()
|
||||
# Build your architecture from kwargs (comes from config.yaml)
|
||||
# kwargs includes: input_size, vocab_size, tokenizer, frontend, etc.
|
||||
|
||||
def forward(self, speech, speech_lengths, text, text_lengths, **kwargs):
|
||||
"""Training forward pass. Return (loss, stats_dict, weight)."""
|
||||
...
|
||||
|
||||
def inference(self, data_in, data_lengths=None, key=None,
|
||||
tokenizer=None, frontend=None, **kwargs):
|
||||
"""Inference. Return (results_list, meta_data)."""
|
||||
...
|
||||
</pre>
|
||||
</div>
|
||||
|
||||
<div class="step">
|
||||
<h4>Create config.yaml</h4>
|
||||
<pre># This file defines what components to use
|
||||
model: MyModel # matches @tables.register key
|
||||
model_conf:
|
||||
hidden_size: 512
|
||||
|
||||
frontend: WavFrontend # reuse existing frontend
|
||||
frontend_conf:
|
||||
fs: 16000
|
||||
n_mels: 80
|
||||
frame_length: 25
|
||||
frame_shift: 10
|
||||
cmvn_file: null
|
||||
|
||||
tokenizer: SentencepiecesTokenizer
|
||||
tokenizer_conf:
|
||||
bpemodel: null</pre>
|
||||
</div>
|
||||
|
||||
<div class="step">
|
||||
<h4>Create configuration.json (for Hub upload)</h4>
|
||||
<pre>{
|
||||
"framework": "pytorch",
|
||||
"task": "auto-speech-recognition",
|
||||
"model": {"type": "funasr"},
|
||||
"file_path_metas": {
|
||||
"init_param": "model.pt",
|
||||
"config": "config.yaml",
|
||||
"tokenizer_conf": {"bpemodel": "my_tokenizer.model"},
|
||||
"frontend_conf": {"cmvn_file": "am.mvn"}
|
||||
}
|
||||
}</pre>
|
||||
<p>This file tells AutoModel how to resolve relative paths. When the model is downloaded, each path in <code>file_path_metas</code> gets the model directory prepended.</p>
|
||||
</div>
|
||||
|
||||
<div class="step">
|
||||
<h4>Test locally</h4>
|
||||
<pre>from funasr import AutoModel
|
||||
|
||||
# From local directory
|
||||
model = AutoModel(model="./my_model_dir")
|
||||
res = model.generate(input="test.wav")
|
||||
print(res)</pre>
|
||||
</div>
|
||||
|
||||
<!-- 5. Add Component -->
|
||||
<h2 id="add-component">5. Add a New Frontend / Tokenizer / Dataset</h2>
|
||||
<p>The same registry pattern applies to all components. Example — adding a new frontend:</p>
|
||||
|
||||
<pre>from funasr.register import tables
|
||||
|
||||
@tables.register("frontend_classes", "MyFrontend")
|
||||
class MyFrontend(nn.Module):
|
||||
def __init__(self, fs=16000, **kwargs):
|
||||
super().__init__()
|
||||
self.fs = fs
|
||||
|
||||
def output_size(self):
|
||||
return 80 # feature dimension
|
||||
|
||||
def forward(self, input, input_lengths):
|
||||
# input: raw waveform (batch, samples)
|
||||
# return: features (batch, frames, dim), lengths
|
||||
...</pre>
|
||||
|
||||
<p>Then reference it in config.yaml:</p>
|
||||
<pre>frontend: MyFrontend
|
||||
frontend_conf:
|
||||
fs: 16000</pre>
|
||||
|
||||
<p>Same pattern for tokenizer (<code>tokenizer_classes</code>), dataset (<code>dataset_classes</code>), encoder (<code>encoder_classes</code>), etc.</p>
|
||||
|
||||
<!-- 6. Standalone -->
|
||||
<h2 id="standalone">6. Standalone Repository Mode</h2>
|
||||
<p>Your model doesn't need to live inside the FunASR source tree. With <code>trust_remote_code=True</code>, FunASR dynamically loads your model class from an external file:</p>
|
||||
|
||||
<pre># User code — loads YOUR model.py from a separate repo
|
||||
model = AutoModel(
|
||||
model="your-org/your-model", # HuggingFace/ModelScope repo
|
||||
trust_remote_code=True,
|
||||
remote_code="./model.py", # path to model class definition
|
||||
hub="hf",
|
||||
)</pre>
|
||||
|
||||
<p><strong>How it works:</strong></p>
|
||||
<ol>
|
||||
<li>FunASR downloads the model repo (weights + config + model.py)</li>
|
||||
<li><code>remote_code="./model.py"</code> is dynamically imported</li>
|
||||
<li>The <code>@tables.register</code> decorator in that file registers the model class</li>
|
||||
<li>Normal <code>build_model()</code> flow proceeds with the registered class</li>
|
||||
</ol>
|
||||
|
||||
<p><strong>Your repo structure:</strong></p>
|
||||
<pre>your-model-repo/
|
||||
├── model.py # model class with @tables.register
|
||||
├── config.yaml # model architecture config
|
||||
├── configuration.json # path resolution
|
||||
├── model.pt # trained weights
|
||||
└── example/test.wav # demo audio</pre>
|
||||
|
||||
<p>Examples: <a href="https://github.com/FunAudioLLM/Fun-ASR">Fun-ASR-Nano</a>, <a href="https://github.com/FunAudioLLM/SenseVoice">SenseVoice</a></p>
|
||||
|
||||
<!-- 7. Testing -->
|
||||
<h2 id="testing">7. Testing Your Model</h2>
|
||||
|
||||
<h4>Write a test script</h4>
|
||||
<pre># tests_models/test_my_model.py
|
||||
import sys, time
|
||||
from funasr import AutoModel
|
||||
|
||||
def main():
|
||||
model = AutoModel(model="path/to/model", device="cpu", disable_update=True)
|
||||
res = model.generate(input="test.wav")
|
||||
|
||||
assert res and len(res) > 0, "empty result"
|
||||
assert "text" in res[0], "missing text field"
|
||||
print("PASSED")
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())</pre>
|
||||
|
||||
<h4>Test with VAD + SPK pipeline</h4>
|
||||
<pre># If your model should work with speaker diarization:
|
||||
model = AutoModel(
|
||||
model="path/to/model",
|
||||
vad_model="fsmn-vad",
|
||||
spk_model="cam++",
|
||||
)
|
||||
res = model.generate(input="meeting.wav", cache={})
|
||||
assert "sentence_info" in res[0]
|
||||
assert "spk" in res[0]["sentence_info"][0]</pre>
|
||||
|
||||
<h4>Test streaming (if applicable)</h4>
|
||||
<pre>cache = {}
|
||||
for i in range(total_chunks):
|
||||
chunk = audio[i*stride:(i+1)*stride]
|
||||
res = model.generate(input=chunk, cache=cache,
|
||||
is_final=(i == total_chunks-1), ...)
|
||||
# Verify: same audio gives same result across multiple sessions</pre>
|
||||
|
||||
<!-- 8. Pitfalls -->
|
||||
<h2 id="pitfalls">8. Common Pitfalls</h2>
|
||||
|
||||
<h4>❌ Forgetting <code>super().__init__()</code></h4>
|
||||
<pre># WRONG — causes "object has no attribute '_state_dict_pre_hooks'"
|
||||
class MyEncoder(nn.Module):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
# CORRECT
|
||||
class MyEncoder(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()</pre>
|
||||
|
||||
<h4>❌ Checking <code>kwargs["batch_size"]</code> in your model</h4>
|
||||
<p><code>batch_size</code> in kwargs is set by <code>inference_with_vad</code> for segment batching (a large number in ms). Don't use it to check actual data batch size. Use <code>len(data_in)</code> instead.</p>
|
||||
|
||||
<h4>❌ Not handling empty/short input</h4>
|
||||
<p>VAD may produce empty segments. Your <code>inference()</code> should handle <code>data_in = []</code> gracefully.</p>
|
||||
|
||||
<h4>❌ Timestamp format mismatch</h4>
|
||||
<p>If your model returns timestamps as dicts (<code>{"start_time": 0.5, "end_time": 0.8}</code>), the pipeline handles conversion. But if you output <code>[start_time, end_time, text]</code> (3 elements), strip the text — downstream expects <code>[start_ms, end_ms]</code> (2 elements, in milliseconds).</p>
|
||||
|
||||
<h4>❌ Importing from other model directories</h4>
|
||||
<pre># WRONG — creates tight coupling
|
||||
from funasr.models.paraformer.model import Paraformer
|
||||
|
||||
# CORRECT — copy what you need into your own directory
|
||||
# Or inherit via the registry name in config.yaml</pre>
|
||||
|
||||
<h4>❌ Modifying <code>self.kwargs</code> during inference</h4>
|
||||
<p>Don't mutate <code>kwargs</code> that came from AutoModel. The framework resets state between calls, but persistent mutations can leak between sessions.</p>
|
||||
|
||||
<!-- 9. Contributing -->
|
||||
<h2 id="contribute">9. Contributing</h2>
|
||||
|
||||
<h4>Code Style</h4>
|
||||
<ul>
|
||||
<li>Follow existing patterns — look at <code>paraformer/model.py</code> as reference</li>
|
||||
<li>Add docstrings to all public methods (Args, Returns)</li>
|
||||
<li>No comments explaining WHAT — use clear naming. Comments for WHY only.</li>
|
||||
</ul>
|
||||
|
||||
<h4>PR Checklist</h4>
|
||||
<ul>
|
||||
<li>New model: self-contained directory, no cross-model imports</li>
|
||||
<li>Include a test script in <code>tests_models/</code></li>
|
||||
<li>Include a demo in <code>examples/industrial_data_pretraining/</code></li>
|
||||
<li>All existing tests still pass</li>
|
||||
<li>Add entry to README What's New if user-facing</li>
|
||||
</ul>
|
||||
|
||||
<h4>License</h4>
|
||||
<p>Code: MIT. Model weights: FunASR Model License (commercial use allowed with attribution).</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
<footer>
|
||||
<p>FunASR · Tongyi Lab, Alibaba Group</p>
|
||||
<p><a href="index.html">ホーム</a> · <a href="tutorial.html">チュートリアル</a> · <a href="training.html">トレーニング</a> · <a href="model-registration.html">開発</a> · <a href="../api.html">API</a> · <a href="https://github.com/modelscope/FunASR">GitHub</a></p>
|
||||
</footer>
|
||||
</body></html>
|
||||
@@ -0,0 +1,335 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="ja"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0">
|
||||
<title>FunASR Training Guide</title>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700;800&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet">
|
||||
<link rel="stylesheet" href="../style.css">
|
||||
</head><body>
|
||||
<nav class="nav"><div class="container">
|
||||
<a href="index.html" class="nav-logo">FunASR</a>
|
||||
<div class="nav-links">
|
||||
<a href="index.html">ホーム</a>
|
||||
<a href="tutorial.html">チュートリアル</a>
|
||||
<a href="training.html" class="active">トレーニング</a>
|
||||
<a href="model-registration.html">開発</a>
|
||||
<a href="../api.html">API</a>
|
||||
</div>
|
||||
<div class="lang-switch">
|
||||
<a href="training.html" title="English">EN</a>
|
||||
<a href="zh/training.html" title="中文">中</a>
|
||||
<a href="ja/training.html" title="日本語">JP</a>
|
||||
</div>
|
||||
<div class="lang-switch">
|
||||
<a href="../training.html" title="English">EN</a>
|
||||
<a href="../zh/training.html" title="中文">中</a>
|
||||
<a href="../ja/training.html" title="日本語">JP</a>
|
||||
</div>
|
||||
<a href="https://github.com/modelscope/FunASR" class="nav-github">GitHub</a>
|
||||
</div></nav>
|
||||
|
||||
<div class="content"><div class="container narrow">
|
||||
<h1>Training & Fine-tuning</h1>
|
||||
<p>Fine-tune pretrained models on your own data using FunASR's training framework.</p>
|
||||
|
||||
<div class="toc-grid">
|
||||
<a href="#overview">Overview</a>
|
||||
<a href="#data">Data Preparation</a>
|
||||
<a href="#paraformer">Fine-tune Paraformer</a>
|
||||
<a href="#sensevoice">Fine-tune SenseVoice</a>
|
||||
<a href="#nano">Fine-tune Fun-ASR-Nano</a>
|
||||
<a href="#params">Parameter Reference</a>
|
||||
<a href="#multi-gpu">Multi-GPU Training</a>
|
||||
<a href="#deepspeed">DeepSpeed</a>
|
||||
<a href="#monitor">Monitoring</a>
|
||||
<a href="#inference-after">Use Fine-tuned Model</a>
|
||||
<a href="#tips">Tips & Troubleshooting</a>
|
||||
</div>
|
||||
|
||||
<!-- Overview -->
|
||||
<h2 id="overview">Overview</h2>
|
||||
<p>FunASR's training framework supports:</p>
|
||||
<ul>
|
||||
<li><strong>Fine-tuning</strong> any pretrained model on custom domain data</li>
|
||||
<li><strong>Multi-GPU</strong> training with PyTorch DDP (single/multi-node)</li>
|
||||
<li><strong>DeepSpeed</strong> ZeRO Stage 1/2/3 for large model training</li>
|
||||
<li><strong>Dynamic batching</strong> by token count or example count</li>
|
||||
<li><strong>Checkpoint averaging</strong> for best performance</li>
|
||||
<li><strong>Resume training</strong> from interruption</li>
|
||||
</ul>
|
||||
|
||||
<p>The training entry point is <code>funasr-train-ds</code> (or <code>funasr/bin/train_ds.py</code>), launched via <code>torchrun</code> for distributed training.</p>
|
||||
|
||||
<!-- Data -->
|
||||
<h2 id="data">Data Preparation</h2>
|
||||
|
||||
<h3>Standard Format (Paraformer, SenseVoice)</h3>
|
||||
<p>Training data uses JSONL format — one JSON object per line:</p>
|
||||
<pre>{"key": "utt001", "source": "/path/to/audio.wav", "source_len": 90, "target": "这是转写文本", "target_len": 6}
|
||||
{"key": "utt002", "source": "/path/to/audio2.wav", "source_len": 150, "target": "hello world", "target_len": 2}</pre>
|
||||
|
||||
<table>
|
||||
<tr><th>Field</th><th>Type</th><th>Description</th></tr>
|
||||
<tr><td><code>key</code></td><td>str</td><td>Unique utterance ID</td></tr>
|
||||
<tr><td><code>source</code></td><td>str</td><td>Audio file path (local path or URL)</td></tr>
|
||||
<tr><td><code>source_len</code></td><td>int</td><td>Audio length in fbank frames (1 frame = 10ms)</td></tr>
|
||||
<tr><td><code>target</code></td><td>str</td><td>Transcription text</td></tr>
|
||||
<tr><td><code>target_len</code></td><td>int</td><td>Number of text tokens</td></tr>
|
||||
</table>
|
||||
|
||||
<h4>Generate from wav.scp + text.txt</h4>
|
||||
<p>If you have Kaldi-style data files, convert them:</p>
|
||||
<pre># train_wav.scp (tab-separated: id \t path)
|
||||
utt001 /data/audio/001.wav
|
||||
utt002 /data/audio/002.wav
|
||||
|
||||
# train_text.txt (tab-separated: id \t text)
|
||||
utt001 这是转写文本
|
||||
utt002 hello world</pre>
|
||||
|
||||
<pre># Convert to jsonl
|
||||
scp2jsonl \
|
||||
++scp_file_list='["/data/list/train_wav.scp", "/data/list/train_text.txt"]' \
|
||||
++data_type_list='["source", "target"]' \
|
||||
++jsonl_file_out="/data/list/train.jsonl"</pre>
|
||||
|
||||
<h3>ChatML Format (Fun-ASR-Nano)</h3>
|
||||
<p>Fun-ASR-Nano uses ChatML conversation format:</p>
|
||||
<pre>{"messages": [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "语音转写:<|startofspeech|>!/path/to/audio.wav<|endofspeech|>"},
|
||||
{"role": "assistant", "content": "几点了?"}
|
||||
], "speech_length": 145, "text_length": 3}</pre>
|
||||
|
||||
<table>
|
||||
<tr><th>Field</th><th>Description</th></tr>
|
||||
<tr><td><code>messages[0]</code></td><td>System prompt (fixed: "You are a helpful assistant.")</td></tr>
|
||||
<tr><td><code>messages[1]</code></td><td>User: prompt + audio path wrapped in <code><|startofspeech|>!...<|endofspeech|></code></td></tr>
|
||||
<tr><td><code>messages[2]</code></td><td>Assistant: transcription text</td></tr>
|
||||
<tr><td><code>speech_length</code></td><td>Number of fbank frames (10ms each)</td></tr>
|
||||
<tr><td><code>text_length</code></td><td>Number of tokens (tokenized by Qwen3-0.6B)</td></tr>
|
||||
</table>
|
||||
|
||||
<div class="tip"><strong>Prompt variations:</strong><br>
|
||||
• Chinese: <code>语音转写:</code><br>
|
||||
• English: <code>Speech transcription:</code><br>
|
||||
• Cross-language: <code>语音转写成英文:</code><br>
|
||||
• No ITN: <code>语音转写,不进行文本规整:</code></div>
|
||||
|
||||
<p>Convert from wav.scp + text.txt:</p>
|
||||
<pre>python tools/scp2jsonl.py \
|
||||
++scp_file=data/train_wav.scp \
|
||||
++transcript_file=data/train_text.txt \
|
||||
++jsonl_file=data/train_example.jsonl</pre>
|
||||
|
||||
<!-- Paraformer -->
|
||||
<h2 id="paraformer">Fine-tune Paraformer</h2>
|
||||
<pre>cd examples/industrial_data_pretraining/paraformer
|
||||
bash finetune.sh</pre>
|
||||
|
||||
<p>Or customize the key parameters:</p>
|
||||
<pre>export CUDA_VISIBLE_DEVICES="0,1"
|
||||
gpu_num=2
|
||||
|
||||
torchrun --nproc_per_node $gpu_num \
|
||||
funasr/bin/train_ds.py \
|
||||
++model="iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch" \
|
||||
++train_data_set_list="data/train.jsonl" \
|
||||
++valid_data_set_list="data/val.jsonl" \
|
||||
++dataset_conf.batch_size=6000 \
|
||||
++dataset_conf.batch_type="token" \
|
||||
++dataset_conf.num_workers=4 \
|
||||
++train_conf.max_epoch=50 \
|
||||
++train_conf.validate_interval=2000 \
|
||||
++train_conf.save_checkpoint_interval=2000 \
|
||||
++train_conf.keep_nbest_models=20 \
|
||||
++train_conf.avg_nbest_model=10 \
|
||||
++optim_conf.lr=0.0002 \
|
||||
++output_dir="./outputs"</pre>
|
||||
|
||||
<!-- SenseVoice -->
|
||||
<h2 id="sensevoice">Fine-tune SenseVoice</h2>
|
||||
<pre>cd examples/industrial_data_pretraining/sense_voice
|
||||
bash finetune.sh</pre>
|
||||
|
||||
<p>Same data format as Paraformer (source/target JSONL). Key difference: SenseVoice uses its own dataset class internally.</p>
|
||||
|
||||
<!-- Fun-ASR-Nano -->
|
||||
<h2 id="nano">Fine-tune Fun-ASR-Nano</h2>
|
||||
<pre>cd examples/industrial_data_pretraining/fun_asr_nano
|
||||
bash finetune.sh</pre>
|
||||
|
||||
<p>Key differences from Paraformer:</p>
|
||||
<ul>
|
||||
<li>Uses ChatML data format (see above)</li>
|
||||
<li><code>++trust_remote_code=true</code> required</li>
|
||||
<li>Supports <strong>selective freezing</strong>: freeze encoder/adaptor while training LLM decoder</li>
|
||||
</ul>
|
||||
|
||||
<pre># Freeze encoder + adaptor, only train LLM (recommended for domain adaptation)
|
||||
++audio_encoder_conf.freeze=true
|
||||
++audio_adaptor_conf.freeze=true
|
||||
++llm_conf.freeze=false
|
||||
|
||||
# Full fine-tune (all parameters)
|
||||
++audio_encoder_conf.freeze=false
|
||||
++audio_adaptor_conf.freeze=false
|
||||
++llm_conf.freeze=false</pre>
|
||||
|
||||
<div class="note"><strong>Recommended strategy:</strong> Start with LLM-only fine-tuning (faster, less data needed). If results are insufficient, unfreeze adaptor. Only unfreeze encoder with very large datasets (>1000h).</div>
|
||||
|
||||
<!-- Parameters -->
|
||||
<h2 id="params">Parameter Reference</h2>
|
||||
|
||||
<h3>Dataset Parameters</h3>
|
||||
<table>
|
||||
<tr><th>Parameter</th><th>Default</th><th>Description</th></tr>
|
||||
<tr><td><code>dataset_conf.batch_type</code></td><td>"token"</td><td><code>"token"</code>: dynamic batch by total tokens. <code>"example"</code>: fixed batch count.</td></tr>
|
||||
<tr><td><code>dataset_conf.batch_size</code></td><td>6000</td><td>Token mode: total frames per batch. Example mode: number of samples.</td></tr>
|
||||
<tr><td><code>dataset_conf.sort_size</code></td><td>1024</td><td>Buffer size for length-based sorting (improves padding efficiency).</td></tr>
|
||||
<tr><td><code>dataset_conf.num_workers</code></td><td>4</td><td>Data loading threads.</td></tr>
|
||||
<tr><td><code>dataset_conf.data_split_num</code></td><td>1</td><td>Split data into N groups for large-scale training (reduces memory).</td></tr>
|
||||
<tr><td><code>dataset_conf.max_token_length</code></td><td>—</td><td>Filter: skip samples longer than this (in frames/tokens).</td></tr>
|
||||
<tr><td><code>dataset_conf.min_token_length</code></td><td>—</td><td>Filter: skip samples shorter than this.</td></tr>
|
||||
</table>
|
||||
|
||||
<h3>Training Parameters</h3>
|
||||
<table>
|
||||
<tr><th>Parameter</th><th>Default</th><th>Description</th></tr>
|
||||
<tr><td><code>train_conf.max_epoch</code></td><td>50</td><td>Total training epochs.</td></tr>
|
||||
<tr><td><code>train_conf.log_interval</code></td><td>1</td><td>Print loss every N steps.</td></tr>
|
||||
<tr><td><code>train_conf.validate_interval</code></td><td>2000</td><td>Run validation every N steps.</td></tr>
|
||||
<tr><td><code>train_conf.save_checkpoint_interval</code></td><td>2000</td><td>Save model every N steps.</td></tr>
|
||||
<tr><td><code>train_conf.keep_nbest_models</code></td><td>20</td><td>Keep top N models (by validation accuracy).</td></tr>
|
||||
<tr><td><code>train_conf.avg_nbest_model</code></td><td>10</td><td>Average top N models for final checkpoint.</td></tr>
|
||||
<tr><td><code>train_conf.resume</code></td><td>true</td><td>Resume from last checkpoint if exists.</td></tr>
|
||||
<tr><td><code>train_conf.use_deepspeed</code></td><td>false</td><td>Enable DeepSpeed ZeRO optimization.</td></tr>
|
||||
<tr><td><code>optim_conf.lr</code></td><td>0.0002</td><td>Learning rate.</td></tr>
|
||||
</table>
|
||||
|
||||
<!-- Multi-GPU -->
|
||||
<h2 id="multi-gpu">Multi-GPU Training</h2>
|
||||
|
||||
<h3>Single Machine, Multiple GPUs</h3>
|
||||
<pre>export CUDA_VISIBLE_DEVICES="0,1,2,3"
|
||||
gpu_num=4
|
||||
|
||||
torchrun --nnodes 1 --nproc_per_node $gpu_num \
|
||||
funasr/bin/train_ds.py ${train_args}</pre>
|
||||
|
||||
<h3>Multiple Machines</h3>
|
||||
<pre># Machine 1 (master, IP=192.168.1.1)
|
||||
torchrun --nnodes 2 --node_rank 0 --nproc_per_node 4 \
|
||||
--master_addr=192.168.1.1 --master_port=12345 \
|
||||
funasr/bin/train_ds.py ${train_args}
|
||||
|
||||
# Machine 2
|
||||
torchrun --nnodes 2 --node_rank 1 --nproc_per_node 4 \
|
||||
--master_addr=192.168.1.1 --master_port=12345 \
|
||||
funasr/bin/train_ds.py ${train_args}</pre>
|
||||
|
||||
<!-- DeepSpeed -->
|
||||
<h2 id="deepspeed">DeepSpeed</h2>
|
||||
<p>For large models (Fun-ASR-Nano 800M params), enable DeepSpeed ZeRO:</p>
|
||||
<pre>++train_conf.use_deepspeed=true
|
||||
++train_conf.deepspeed_config=./deepspeed_conf/ds_stage1.json</pre>
|
||||
|
||||
<p>Stage 1 config (recommended starting point):</p>
|
||||
<pre>{
|
||||
"train_micro_batch_size_per_gpu": 1,
|
||||
"gradient_accumulation_steps": 1,
|
||||
"bf16": {"enabled": true},
|
||||
"zero_optimization": {
|
||||
"stage": 1,
|
||||
"reduce_bucket_size": 5e8,
|
||||
"allgather_bucket_size": 5e8
|
||||
}
|
||||
}</pre>
|
||||
|
||||
<div class="note"><strong>When to use which stage:</strong><br>
|
||||
• Stage 1: Optimizer state partitioned. Good for most cases.<br>
|
||||
• Stage 2: + Gradient partitioned. For larger models.<br>
|
||||
• Stage 3: + Parameter partitioned. Maximum memory savings but slower.</div>
|
||||
|
||||
<!-- Monitor -->
|
||||
<h2 id="monitor">Monitoring Training</h2>
|
||||
|
||||
<h4>Log file</h4>
|
||||
<pre>tail -f outputs/log.txt
|
||||
|
||||
# Example output:
|
||||
# train, rank: 0, epoch: 0/50, step: 6990, (loss_avg_rank: 0.327),
|
||||
# (acc_avg_epoch: 0.795), (lr: 1.165e-04),
|
||||
# GPU memory: usage: 3.8GB, peak: 18.3GB</pre>
|
||||
|
||||
<p>Key metrics to watch:</p>
|
||||
<ul>
|
||||
<li><code>loss_avg_epoch</code>: should decrease over time</li>
|
||||
<li><code>acc_avg_epoch</code>: should increase (most important metric)</li>
|
||||
<li><code>lr</code>: learning rate at current step</li>
|
||||
<li><code>GPU memory</code>: peak should not exceed your GPU VRAM</li>
|
||||
</ul>
|
||||
|
||||
<h4>TensorBoard</h4>
|
||||
<pre>tensorboard --logdir outputs/log/tensorboard
|
||||
# Open http://localhost:6006</pre>
|
||||
|
||||
<!-- Use Fine-tuned Model -->
|
||||
<h2 id="inference-after">Use Your Fine-tuned Model</h2>
|
||||
|
||||
<h4>If outputs/ has configuration.json</h4>
|
||||
<pre>from funasr import AutoModel
|
||||
model = AutoModel(model="./outputs")
|
||||
res = model.generate(input="test.wav")
|
||||
print(res[0]["text"])</pre>
|
||||
|
||||
<h4>If no configuration.json</h4>
|
||||
<pre>funasr ++model="./outputs" \
|
||||
++config-path="./outputs" \
|
||||
++config-name="config.yaml" \
|
||||
++init_param="./outputs/model.pt" \
|
||||
++input="test.wav"</pre>
|
||||
|
||||
<!-- Tips -->
|
||||
<h2 id="tips">Tips & Troubleshooting</h2>
|
||||
|
||||
<h4>OOM during training</h4>
|
||||
<ol>
|
||||
<li>Reduce <code>dataset_conf.batch_size</code></li>
|
||||
<li>Add <code>dataset_conf.max_token_length=2000</code> to filter long utterances</li>
|
||||
<li>Enable DeepSpeed (partitions optimizer states)</li>
|
||||
<li>Reduce <code>dataset_conf.num_workers</code></li>
|
||||
</ol>
|
||||
|
||||
<h4>Training loss stuck / NaN gradients</h4>
|
||||
<ul>
|
||||
<li>Reduce learning rate (try 0.00005)</li>
|
||||
<li>Check data quality — corrupted audio files cause NaN</li>
|
||||
<li>For Fun-ASR-Nano: start with encoder frozen</li>
|
||||
</ul>
|
||||
|
||||
<h4>Validation accuracy not improving</h4>
|
||||
<ul>
|
||||
<li>Increase training data (min ~10h for fine-tuning)</li>
|
||||
<li>Check domain match — model may not generalize to very different domains</li>
|
||||
<li>Try unfreezing more layers gradually</li>
|
||||
</ul>
|
||||
|
||||
<h4>Large-scale data (>10,000 hours)</h4>
|
||||
<p>Use data splitting to avoid memory issues:</p>
|
||||
<pre># Split data into chunks, load 2 at a time
|
||||
++dataset_conf.data_split_num=256
|
||||
# data.list contains paths to split jsonl files:
|
||||
# data/train.0.jsonl
|
||||
# data/train.1.jsonl
|
||||
# ...
|
||||
++train_data_set_list="data/data.list"</pre>
|
||||
|
||||
<h4>Resume after crash</h4>
|
||||
<p>Set <code>++train_conf.resume=true</code> (default). Training automatically restarts from the latest checkpoint in <code>output_dir</code>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
<footer>
|
||||
<p>FunASR · Tongyi Lab, Alibaba Group</p>
|
||||
<p><a href="index.html">ホーム</a> · <a href="tutorial.html">チュートリアル</a> · <a href="training.html">トレーニング</a> · <a href="model-registration.html">開発</a> · <a href="../api.html">API</a> · <a href="https://github.com/modelscope/FunASR">GitHub</a></p>
|
||||
</footer>
|
||||
</body></html>
|
||||
@@ -0,0 +1,383 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="ja"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0">
|
||||
<title>FunASR Tutorial</title>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700;800&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet">
|
||||
<link rel="stylesheet" href="../style.css">
|
||||
</head><body>
|
||||
<nav class="nav"><div class="container">
|
||||
<a href="index.html" class="nav-logo">FunASR</a>
|
||||
<div class="nav-links">
|
||||
<a href="index.html">ホーム</a>
|
||||
<a href="tutorial.html" class="active">チュートリアル</a>
|
||||
<a href="training.html">トレーニング</a>
|
||||
<a href="model-registration.html">開発</a>
|
||||
<a href="../api.html">API</a>
|
||||
</div>
|
||||
<div class="lang-switch">
|
||||
<a href="tutorial.html" title="English">EN</a>
|
||||
<a href="zh/tutorial.html" title="中文">中</a>
|
||||
<a href="ja/tutorial.html" title="日本語">JP</a>
|
||||
</div>
|
||||
<div class="lang-switch">
|
||||
<a href="../tutorial.html" title="English">EN</a>
|
||||
<a href="../zh/tutorial.html" title="中文">中</a>
|
||||
<a href="../ja/tutorial.html" title="日本語">JP</a>
|
||||
</div>
|
||||
<a href="https://github.com/modelscope/FunASR" class="nav-github">GitHub</a>
|
||||
</div></nav>
|
||||
|
||||
<div class="content"><div class="container narrow">
|
||||
<h1>チュートリアル</h1>
|
||||
<p>From zero to production — learn FunASR in 10 minutes.</p>
|
||||
|
||||
<div class="toc-grid">
|
||||
<a href="#quick">30秒デモ</a>
|
||||
<a href="#install">インストール</a>
|
||||
<a href="#choose">Choose a Model</a>
|
||||
<a href="#scenarios">使用シナリオ</a>
|
||||
<a href="#offline">オフラインASR</a>
|
||||
<a href="#streaming">ストリーミングASR</a>
|
||||
<a href="#spk">話者分離</a>
|
||||
<a href="#emotion">感情検出</a>
|
||||
<a href="#vad">VAD</a>
|
||||
<a href="#punc">句読点復元</a>
|
||||
<a href="#export">ONNXエクスポート</a>
|
||||
<a href="#faq">FAQ・トラブルシューティング</a>
|
||||
</div>
|
||||
|
||||
<!-- 30-Second Demo -->
|
||||
<h2 id="quick">30秒デモ</h2>
|
||||
<p>Copy-paste this and run. No config needed — FunASR downloads everything automatically:</p>
|
||||
<pre>pip install funasr
|
||||
|
||||
python -c "
|
||||
from funasr import AutoModel
|
||||
model = AutoModel(model='paraformer-zh', vad_model='fsmn-vad', punc_model='ct-punc')
|
||||
res = model.generate(input='https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav')
|
||||
print(res[0]['text'])
|
||||
"</pre>
|
||||
<p><strong>Output:</strong></p>
|
||||
<div class="output-box">欢迎大家来体验达摩院推出的语音识别模型。</div>
|
||||
|
||||
<p>That's it. One model object handles everything: loading audio, VAD segmentation, ASR, and punctuation.</p>
|
||||
|
||||
<!-- Install -->
|
||||
<h2 id="install">インストール</h2>
|
||||
<pre># Stable release
|
||||
pip install funasr
|
||||
|
||||
# Latest (recommended — has newest models and bug fixes)
|
||||
pip install git+https://github.com/modelscope/FunASR.git</pre>
|
||||
|
||||
<div class="tip"><strong>China users:</strong> Models download from ModelScope by default (fast in China). International users: add <code>hub="hf"</code> to download from HuggingFace.</div>
|
||||
|
||||
<!-- Choose a Model -->
|
||||
<h2 id="choose">どのモデルを使うべきか?</h2>
|
||||
|
||||
<table>
|
||||
<tr><th>Model</th><th>Best For</th><th>Languages</th><th>Speed</th><th>句読点復元</th></tr>
|
||||
<tr><td><strong>Paraformer</strong></td><td>Chinese production ASR</td><td>Chinese, English</td><td>Fast</td><td>Needs punc_model</td></tr>
|
||||
<tr><td><strong>Fun-ASR-Nano</strong></td><td>Multi-language, dialects, lyrics</td><td>31 languages</td><td>Medium</td><td>Built-in ✓</td></tr>
|
||||
<tr><td><strong>SenseVoice</strong></td><td>Emotion + events + ASR</td><td>5 languages</td><td>Ultra-fast (70ms/10s)</td><td>Built-in ✓</td></tr>
|
||||
<tr><td><strong>Qwen3-ASR</strong></td><td>Highest accuracy, context-aware</td><td>52 languages</td><td>Slow (LLM)</td><td>Built-in ✓</td></tr>
|
||||
<tr><td><strong>Paraformer-Streaming</strong></td><td>Real-time transcription</td><td>Chinese</td><td>Real-time</td><td>Needs punc_model</td></tr>
|
||||
</table>
|
||||
|
||||
<div class="note">
|
||||
<strong>Quick recommendation:</strong><br>
|
||||
• Chinese meeting/call → <code>paraformer-zh</code> + <code>fsmn-vad</code> + <code>ct-punc</code> + <code>cam++</code><br>
|
||||
• Multi-language → <code>Fun-ASR-Nano</code><br>
|
||||
• Need emotion/sound events → <code>SenseVoice</code><br>
|
||||
• Real-time subtitles → <code>paraformer-zh-streaming</code><br>
|
||||
• Highest quality, no latency concern → <code>Qwen3-ASR</code>
|
||||
</div>
|
||||
|
||||
<!-- Scenarios -->
|
||||
<h2 id="scenarios">使用シナリオ</h2>
|
||||
|
||||
<div class="scenario-card">
|
||||
<h4>🎤 "I have a meeting recording and want text + who said what"</h4>
|
||||
<p>→ Paraformer + VAD + Punctuation + Speaker Diarization → <a href="#spk">Jump to Speaker Diarization</a></p>
|
||||
</div>
|
||||
|
||||
<div class="scenario-card">
|
||||
<h4>📺 "I want real-time subtitles for a live stream"</h4>
|
||||
<p>→ Paraformer-Streaming, feed audio chunks every 600ms → <a href="#streaming">Jump to Streaming ASR</a></p>
|
||||
</div>
|
||||
|
||||
<div class="scenario-card">
|
||||
<h4>😊 "I want to detect user emotions from voice"</h4>
|
||||
<p>→ SenseVoice (outputs emotion tags: happy, sad, angry, neutral) → <a href="#emotion">Jump to Emotion Detection</a></p>
|
||||
</div>
|
||||
|
||||
<div class="scenario-card">
|
||||
<h4>🌍 "I have audio in Japanese/Korean/Arabic/etc."</h4>
|
||||
<p>→ Fun-ASR-Nano (31 languages) or Qwen3-ASR (52 languages) → <a href="#offline">Jump to Offline ASR</a></p>
|
||||
</div>
|
||||
|
||||
<div class="scenario-card">
|
||||
<h4>✂️ "I want to clip a video by spoken content"</h4>
|
||||
<p>→ Use <a href="https://github.com/modelscope/FunClip">FunClip</a> which integrates FunASR for smart video editing</p>
|
||||
</div>
|
||||
|
||||
<!-- Offline ASR -->
|
||||
<h2 id="offline">オフラインASR</h2>
|
||||
|
||||
<h3>Paraformer (Chinese)</h3>
|
||||
<pre>from funasr import AutoModel
|
||||
|
||||
model = AutoModel(
|
||||
model="paraformer-zh", # Chinese ASR
|
||||
vad_model="fsmn-vad", # handles any audio length
|
||||
vad_kwargs={"max_single_segment_time": 60000},
|
||||
punc_model="ct-punc", # adds punctuation
|
||||
)
|
||||
res = model.generate(input="meeting.wav", batch_size_s=300, hotword='达摩院 语音识别')
|
||||
print(res[0]["text"]) # "欢迎大家来体验达摩院推出的语音识别模型。"
|
||||
print(res[0]["timestamp"]) # [[880,1120],[1120,1360],...] (ms per character)</pre>
|
||||
|
||||
<h3>Fun-ASR-Nano (31 Languages)</h3>
|
||||
<pre>from funasr import AutoModel
|
||||
|
||||
model = AutoModel(
|
||||
model="FunAudioLLM/Fun-ASR-Nano-2512",
|
||||
trust_remote_code=True,
|
||||
remote_code="./model.py",
|
||||
vad_model="fsmn-vad",
|
||||
vad_kwargs={"max_single_segment_time": 30000},
|
||||
device="cuda:0",
|
||||
hub="hf",
|
||||
)
|
||||
res = model.generate(input=["audio.wav"], cache={}, batch_size=1,
|
||||
hotwords=["keyword"], language="中文")
|
||||
print(res[0]["text"]) # recognized text with punctuation
|
||||
print(res[0]["timestamps"]) # [{"token":"开","start_time":0.42,"end_time":0.48}, ...]</pre>
|
||||
|
||||
<div class="note"><strong>Fun-ASR-Nano outputs punctuation natively</strong> — no need for <code>punc_model</code>.</div>
|
||||
|
||||
<h3>SenseVoice (ASR + Emotion + Events)</h3>
|
||||
<pre>from funasr import AutoModel
|
||||
from funasr.utils.postprocess_utils import rich_transcription_postprocess
|
||||
|
||||
model = AutoModel(
|
||||
model="iic/SenseVoiceSmall",
|
||||
vad_model="fsmn-vad",
|
||||
vad_kwargs={"max_single_segment_time": 30000},
|
||||
device="cuda:0",
|
||||
)
|
||||
res = model.generate(input="audio.wav", cache={}, language="auto",
|
||||
use_itn=True, batch_size_s=60, merge_vad=True, merge_length_s=15)
|
||||
|
||||
# Raw output contains emotion/event tags: <|zh|><|HAPPY|><|Speech|>你好
|
||||
text = rich_transcription_postprocess(res[0]["text"])
|
||||
print(text) # "你好" (clean text)</pre>
|
||||
|
||||
<h3>Qwen3-ASR (52 Languages, Highest Accuracy)</h3>
|
||||
<pre># pip install qwen-asr
|
||||
from funasr import AutoModel
|
||||
|
||||
model = AutoModel(model="Qwen/Qwen3-ASR-1.7B", hub="hf", device="cuda:0")
|
||||
res = model.generate(input="audio.wav")
|
||||
print(res[0]["text"]) # recognized text
|
||||
print(res[0].get("language")) # auto-detected language</pre>
|
||||
|
||||
<!-- Streaming -->
|
||||
<h2 id="streaming">Streaming ASR (Real-time)</h2>
|
||||
<p>Process audio chunk-by-chunk for real-time transcription. Each chunk produces partial text immediately.</p>
|
||||
|
||||
<pre>from funasr import AutoModel
|
||||
import soundfile
|
||||
|
||||
model = AutoModel(model="paraformer-zh-streaming")
|
||||
|
||||
speech, sr = soundfile.read("audio.wav")
|
||||
chunk_size = [0, 10, 5] # 600ms display, 300ms lookahead
|
||||
chunk_stride = chunk_size[1] * 960 # 9600 samples per chunk
|
||||
|
||||
cache = {}
|
||||
total_chunks = int((len(speech) - 1) / chunk_stride + 1)
|
||||
for i in range(total_chunks):
|
||||
chunk = speech[i * chunk_stride:(i + 1) * chunk_stride]
|
||||
is_final = (i == total_chunks - 1)
|
||||
|
||||
res = model.generate(input=chunk, cache=cache, is_final=is_final,
|
||||
chunk_size=chunk_size,
|
||||
encoder_chunk_look_back=4,
|
||||
decoder_chunk_look_back=1)
|
||||
if res[0]["text"]:
|
||||
print(res[0]["text"], end="", flush=True) # incremental output</pre>
|
||||
|
||||
<p><strong>Output (printed incrementally):</strong></p>
|
||||
<div class="output-box">欢迎大 | 家来 | 体验达 | 摩院推 | 出的语 | 音识 | 别模型</div>
|
||||
|
||||
<div class="note">
|
||||
<strong>Key points:</strong><br>
|
||||
• <code>cache={}</code> must persist across all chunks (don't recreate it)<br>
|
||||
• <code>is_final=True</code> on last chunk flushes remaining buffered text<br>
|
||||
• <code>chunk_size=[0,10,5]</code>: first number unused, second=display granularity (×60ms), third=lookahead (×60ms)
|
||||
</div>
|
||||
|
||||
<!-- Speaker Diarization -->
|
||||
<h2 id="spk">Speaker Diarization ("Who Said What")</h2>
|
||||
<p>Works with <strong>all three major models</strong>: Paraformer, Fun-ASR-Nano, and SenseVoice. Add <code>spk_model="cam++"</code> to get speaker labels per sentence.</p>
|
||||
|
||||
<h3>Paraformer + Speaker</h3>
|
||||
<pre>from funasr import AutoModel
|
||||
|
||||
model = AutoModel(
|
||||
model="paraformer-zh",
|
||||
vad_model="fsmn-vad",
|
||||
punc_model="ct-punc", # Paraformer needs this for sentence segmentation
|
||||
spk_model="cam++",
|
||||
)
|
||||
res = model.generate(input="meeting.wav", batch_size_s=300)
|
||||
|
||||
for sent in res[0]["sentence_info"]:
|
||||
print(f"[Speaker {sent['spk']}] [{sent['start']}-{sent['end']}ms] {sent['text']}")</pre>
|
||||
|
||||
<p><strong>Output:</strong></p>
|
||||
<div class="output-box">[Speaker 0] [880-5195ms] 欢迎大家来体验达摩院推出的语音识别模型。</div>
|
||||
|
||||
<h3>Fun-ASR-Nano + Speaker (no punc needed)</h3>
|
||||
<pre>model = AutoModel(
|
||||
model="FunAudioLLM/Fun-ASR-Nano-2512",
|
||||
trust_remote_code=True, remote_code="./model.py",
|
||||
vad_model="fsmn-vad", vad_kwargs={"max_single_segment_time": 30000},
|
||||
spk_model="cam++", # no punc_model needed!
|
||||
device="cuda:0", hub="hf",
|
||||
)
|
||||
res = model.generate(input=["meeting.wav"], cache={}, batch_size=1, language="中文")
|
||||
for sent in res[0]["sentence_info"]:
|
||||
print(f"Speaker {sent['spk']}: {sent.get('text', sent.get('sentence', ''))}")</pre>
|
||||
|
||||
<h3>SenseVoice + Speaker (no punc needed)</h3>
|
||||
<pre>model = AutoModel(
|
||||
model="iic/SenseVoiceSmall",
|
||||
vad_model="fsmn-vad", vad_kwargs={"max_single_segment_time": 30000},
|
||||
spk_model="cam++", # no punc_model needed!
|
||||
device="cuda:0",
|
||||
)
|
||||
res = model.generate(input="meeting.wav", cache={}, language="auto",
|
||||
use_itn=True, batch_size_s=60, merge_vad=True, merge_length_s=15)
|
||||
for sent in res[0]["sentence_info"]:
|
||||
print(f"Speaker {sent['spk']}: {rich_transcription_postprocess(sent['text'])}")</pre>
|
||||
|
||||
<!-- Emotion -->
|
||||
<h2 id="emotion">感情検出</h2>
|
||||
|
||||
<h3>Dedicated Emotion Model (emotion2vec)</h3>
|
||||
<pre>from funasr import AutoModel
|
||||
|
||||
model = AutoModel(model="iic/emotion2vec_plus_large", device="cuda:0")
|
||||
res = model.generate(input="audio.wav", granularity="utterance")
|
||||
|
||||
print(res[0]["labels"]) # ['angry', 'happy', 'neutral', 'sad', ...]
|
||||
print(res[0]["scores"]) # [0.01, 0.03, 0.89, 0.05, ...]
|
||||
# → This audio is "neutral" with 89% confidence</pre>
|
||||
|
||||
<h3>SenseVoice (ASR + Emotion in one shot)</h3>
|
||||
<p>SenseVoice embeds emotion tags directly in the transcription output:</p>
|
||||
<pre># Raw output: "<|zh|><|HAPPY|><|Speech|><|withitn|>今天真是太开心了。"
|
||||
# The <|HAPPY|> tag tells you the emotion
|
||||
# Use rich_transcription_postprocess() to get clean text</pre>
|
||||
|
||||
<!-- VAD -->
|
||||
<h2 id="vad">VAD音声検出</h2>
|
||||
|
||||
<h3>Offline (full audio)</h3>
|
||||
<pre>from funasr import AutoModel
|
||||
|
||||
model = AutoModel(model="fsmn-vad")
|
||||
res = model.generate(input="audio.wav")
|
||||
print(res[0]["value"]) # [[610, 5530], [7200, 12400], ...]
|
||||
# Each pair: [start_ms, end_ms] of speech</pre>
|
||||
|
||||
<h3>Streaming (chunk-by-chunk)</h3>
|
||||
<pre>import soundfile
|
||||
from funasr import AutoModel
|
||||
|
||||
model = AutoModel(model="fsmn-vad")
|
||||
speech, sr = soundfile.read("audio.wav")
|
||||
chunk_stride = int(200 * sr / 1000) # 200ms chunks
|
||||
|
||||
cache = {}
|
||||
for i in range(int((len(speech)-1)/chunk_stride+1)):
|
||||
chunk = speech[i*chunk_stride:(i+1)*chunk_stride]
|
||||
is_final = i == int((len(speech)-1)/chunk_stride)
|
||||
res = model.generate(input=chunk, cache=cache, is_final=is_final, chunk_size=200)
|
||||
if res[0]["value"]:
|
||||
print(res[0]["value"])
|
||||
# [[610, -1]] → speech started at 610ms
|
||||
# [[-1, 5530]] → speech ended at 5530ms
|
||||
# [[610, 5530]] → complete segment</pre>
|
||||
|
||||
<!-- Punctuation -->
|
||||
<h2 id="punc">Punctuation Restoration</h2>
|
||||
<pre>from funasr import AutoModel
|
||||
|
||||
model = AutoModel(model="ct-punc")
|
||||
res = model.generate(input="那今天的会就到这里吧 happy new year 明年见")
|
||||
print(res[0]["text"]) # "那今天的会就到这里吧,happy new year,明年见。"</pre>
|
||||
<div class="note"><strong>When do you need this?</strong> Only with Paraformer (which outputs raw text without punctuation). Fun-ASR-Nano, SenseVoice, and Qwen3-ASR output punctuation natively.</div>
|
||||
|
||||
<!-- Export -->
|
||||
<h2 id="export">ONNXエクスポート</h2>
|
||||
<pre># Export model to ONNX format
|
||||
from funasr import AutoModel
|
||||
model = AutoModel(model="paraformer", device="cpu")
|
||||
model.export(quantize=False) # saves to model cache directory
|
||||
|
||||
# Use ONNX model (faster, no PyTorch needed)
|
||||
# pip install funasr-onnx
|
||||
from funasr_onnx import Paraformer
|
||||
model = Paraformer("damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
|
||||
batch_size=1, quantize=True)
|
||||
result = model(["audio.wav"])
|
||||
print(result)</pre>
|
||||
|
||||
<!-- FAQ -->
|
||||
<h2 id="faq">FAQ・トラブルシューティング</h2>
|
||||
|
||||
<h4>Model download is slow</h4>
|
||||
<p>Set <code>hub="hf"</code> for HuggingFace (faster outside China), or download once and use local path:</p>
|
||||
<pre>model = AutoModel(model="/path/to/local/model", disable_update=True)</pre>
|
||||
|
||||
<h4>Out of Memory (OOM)</h4>
|
||||
<p>Three knobs to reduce memory:</p>
|
||||
<ol>
|
||||
<li>Reduce <code>batch_size_s</code> (e.g., from 300 to 60)</li>
|
||||
<li>Reduce <code>max_single_segment_time</code> in vad_kwargs (e.g., 30000 → 15000)</li>
|
||||
<li>Add <code>batch_size_threshold_s=30</code> to force batch=1 for long segments</li>
|
||||
</ol>
|
||||
|
||||
<h4>Every startup shows "Downloading Model..."</h4>
|
||||
<p>Not actually re-downloading — just verifying cache. To skip entirely:</p>
|
||||
<pre>model = AutoModel(model="/local/path/to/model", disable_update=True)</pre>
|
||||
|
||||
<h4>"ModelName is not registered"</h4>
|
||||
<p>Usually means pypi version is outdated. Install from source:</p>
|
||||
<pre>pip install git+https://github.com/modelscope/FunASR.git</pre>
|
||||
|
||||
<h4>Want to suppress all progress bars and logs?</h4>
|
||||
<pre>model = AutoModel(model="...", disable_update=True, disable_pbar=True, log_level="ERROR")</pre>
|
||||
|
||||
<h4>How to pass pre-loaded numpy audio?</h4>
|
||||
<pre>import soundfile as sf
|
||||
audio, sr = sf.read("audio.wav") # numpy array, 16kHz
|
||||
res = model.generate(input=audio) # pass directly — no file needed</pre>
|
||||
|
||||
<h4>GPU not being used?</h4>
|
||||
<pre># Check:
|
||||
import torch
|
||||
print(torch.cuda.is_available()) # must be True
|
||||
|
||||
# Specify GPU explicitly:
|
||||
model = AutoModel(model="...", device="cuda:0")</pre>
|
||||
|
||||
</div></div>
|
||||
|
||||
<footer>
|
||||
<p>FunASR · Tongyi Lab, Alibaba Group</p>
|
||||
<p><a href="index.html">ホーム</a> · <a href="tutorial.html">チュートリアル</a> · <a href="training.html">トレーニング</a> · <a href="model-registration.html">開発</a> · <a href="../api.html">API</a> · <a href="https://github.com/modelscope/FunASR">GitHub</a></p>
|
||||
</footer>
|
||||
</body></html>
|
||||
@@ -13,6 +13,11 @@
|
||||
<a href="model-registration.html" class="active">Develop</a>
|
||||
<a href="api.html">API</a>
|
||||
</div>
|
||||
<div class="lang-switch">
|
||||
<a href="model-registration.html" title="English">EN</a>
|
||||
<a href="zh/model-registration.html" title="中文">中</a>
|
||||
<a href="ja/model-registration.html" title="日本語">JP</a>
|
||||
</div>
|
||||
<a href="https://github.com/modelscope/FunASR" class="nav-github">GitHub</a>
|
||||
</div></nav>
|
||||
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -13,6 +13,11 @@
|
||||
<a href="model-registration.html">Develop</a>
|
||||
<a href="api.html">API</a>
|
||||
</div>
|
||||
<div class="lang-switch">
|
||||
<a href="training.html" title="English">EN</a>
|
||||
<a href="zh/training.html" title="中文">中</a>
|
||||
<a href="ja/training.html" title="日本語">JP</a>
|
||||
</div>
|
||||
<a href="https://github.com/modelscope/FunASR" class="nav-github">GitHub</a>
|
||||
</div></nav>
|
||||
|
||||
|
||||
@@ -13,6 +13,11 @@
|
||||
<a href="model-registration.html">Develop</a>
|
||||
<a href="api.html">API</a>
|
||||
</div>
|
||||
<div class="lang-switch">
|
||||
<a href="tutorial.html" title="English">EN</a>
|
||||
<a href="zh/tutorial.html" title="中文">中</a>
|
||||
<a href="ja/tutorial.html" title="日本語">JP</a>
|
||||
</div>
|
||||
<a href="https://github.com/modelscope/FunASR" class="nav-github">GitHub</a>
|
||||
</div></nav>
|
||||
|
||||
|
||||
+256
@@ -0,0 +1,256 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>FunASR - End-to-End Speech Recognition Toolkit</title>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700;800&display=swap" rel="stylesheet">
|
||||
<link rel="stylesheet" href="../style.css">
|
||||
</head>
|
||||
<body>
|
||||
<nav class="nav">
|
||||
<div class="container">
|
||||
<a href="index.html" class="nav-logo">FunASR</a>
|
||||
<div class="nav-links">
|
||||
<a href="index.html" class="active">首页</a>
|
||||
<a href="tutorial.html">使用教程</a>
|
||||
<a href="training.html">训练</a>
|
||||
<a href="model-registration.html">开发</a>
|
||||
<a href="../api.html">API</a>
|
||||
</div>
|
||||
<div class="lang-switch">
|
||||
<a href="index.html" title="English">EN</a>
|
||||
<a href="zh/index.html" title="中文">中</a>
|
||||
<a href="ja/index.html" title="日本語">JP</a>
|
||||
</div>
|
||||
<div class="lang-switch">
|
||||
<a href="../index.html" title="English">EN</a>
|
||||
<a href="../zh/index.html" title="中文">中</a>
|
||||
<a href="../ja/index.html" title="日本語">JP</a>
|
||||
</div>
|
||||
<a href="https://github.com/modelscope/FunASR" class="nav-github">GitHub</a>
|
||||
</div>
|
||||
</nav>
|
||||
|
||||
<div class="hero">
|
||||
<div class="container hero-content">
|
||||
<div class="hero-copy">
|
||||
<p class="eyebrow">Open-source speech understanding toolkit</p>
|
||||
<h1>FunASR</h1>
|
||||
<p>Production-ready ASR, VAD, punctuation, speaker diarization, emotion detection, and audio event recognition with one unified Python interface.</p>
|
||||
<div class="hero-badges">
|
||||
<a href="https://pypi.org/project/funasr/"><img src="https://img.shields.io/pypi/v/funasr?style=flat-square&color=2563eb" alt="PyPI"></a>
|
||||
<a href="https://github.com/modelscope/FunASR/stargazers"><img src="https://img.shields.io/github/stars/modelscope/FunASR?style=flat-square&color=b45309" alt="Stars"></a>
|
||||
<a href="https://github.com/modelscope/FunASR/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-MIT-059669?style=flat-square" alt="License"></a>
|
||||
</div>
|
||||
<div class="hero-actions">
|
||||
<a href="tutorial.html" class="btn btn-white">快速开始</a>
|
||||
<a href="training.html" class="btn btn-outline-white">Training guide</a>
|
||||
<a href="../api.html" class="btn btn-outline-white">API reference</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="hero-panel" aria-label="FunASR recognition preview">
|
||||
<div class="terminal-bar">
|
||||
<span class="terminal-dots"><span></span><span></span><span></span></span>
|
||||
<span>funasr-pipeline.py</span>
|
||||
</div>
|
||||
<pre>from funasr import AutoModel
|
||||
|
||||
model = AutoModel(
|
||||
model="paraformer-zh",
|
||||
vad_model="fsmn-vad",
|
||||
punc_model="ct-punc",
|
||||
spk_model="cam++",
|
||||
)
|
||||
res = model.generate(input="meeting.wav")
|
||||
print(res[0]["sentence_info"])</pre>
|
||||
<div class="hero-metrics">
|
||||
<div><strong>50+</strong><span>languages</span></div>
|
||||
<div><strong>15x</strong><span>faster</span></div>
|
||||
<div><strong>1 API</strong><span>pipeline</span></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<section class="docs-band">
|
||||
<div class="container">
|
||||
<h2 class="section-title">Documentation</h2>
|
||||
<p class="section-subtitle">Start with examples, tune on your own data, extend the registry, or jump into source-linked API docs.</p>
|
||||
<div class="doc-grid">
|
||||
<a class="doc-card" href="tutorial.html">
|
||||
<span class="doc-kicker">Learn</span>
|
||||
<h3>使用教程</h3>
|
||||
<p>Install FunASR, choose a model, and run common ASR, VAD, diarization, and export flows.</p>
|
||||
</a>
|
||||
<a class="doc-card" href="training.html">
|
||||
<span class="doc-kicker">Tune</span>
|
||||
<h3>训练</h3>
|
||||
<p>Prepare JSONL data, fine-tune Paraformer, SenseVoice, and Fun-ASR-Nano, then monitor runs.</p>
|
||||
</a>
|
||||
<a class="doc-card" href="model-registration.html">
|
||||
<span class="doc-kicker">Extend</span>
|
||||
<h3>Developer Guide</h3>
|
||||
<p>Understand the registry, add a model, package remote code, and avoid integration pitfalls.</p>
|
||||
</a>
|
||||
<a class="doc-card" href="../api.html">
|
||||
<span class="doc-kicker">Reference</span>
|
||||
<h3>API</h3>
|
||||
<p>Browse generated classes, methods, source previews, and GitHub line links.</p>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="container">
|
||||
<h2 class="section-title">核心能力</h2>
|
||||
<p class="section-subtitle">Everything needed for speech understanding, from raw audio segmentation to speaker-aware transcripts.</p>
|
||||
<div class="grid-3">
|
||||
<div class="card">
|
||||
<div class="card-icon">ASR</div>
|
||||
<h3>语音识别</h3>
|
||||
<p>Streaming and offline ASR with VAD segmentation. Process long-form audio with a single API call.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div class="card-icon">50+</div>
|
||||
<h3>Multilingual Models</h3>
|
||||
<p>Fun-ASR-Nano covers 31 languages and Qwen3-ASR covers 52 languages with language detection.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div class="card-icon">SPK</div>
|
||||
<h3>说话人分离</h3>
|
||||
<p>Identify who spoke when, then attach speaker labels to sentence-level ASR output.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div class="card-icon">SFX</div>
|
||||
<h3>情感与事件</h3>
|
||||
<p>SenseVoice detects emotion and audio events including background music, applause, laughter, and crying.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div class="card-icon">RTF</div>
|
||||
<h3>Low Latency</h3>
|
||||
<p>Non-autoregressive models support fast batch and realtime workloads across common deployment targets.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<div class="card-icon">OPS</div>
|
||||
<h3>训练与部署</h3>
|
||||
<p>Fine-tune with DeepSpeed, export to ONNX, and deploy through Docker runtime or the Python SDK.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="alt">
|
||||
<div class="container">
|
||||
<h2 class="section-title">模型</h2>
|
||||
<p class="section-subtitle">Pre-trained industrial models ready for recognition, segmentation, and speech understanding workflows.</p>
|
||||
<div class="grid-2">
|
||||
<div class="card">
|
||||
<h3>Fun-ASR-Nano</h3>
|
||||
<p>End-to-end ASR trained on tens of millions of hours. 31 languages, dialects, accents, lyrics, timestamps, and hotwords.</p>
|
||||
<div class="card-tags">
|
||||
<span class="tag">800M</span>
|
||||
<span class="tag">31 Languages</span>
|
||||
<span class="tag">Timestamps</span>
|
||||
<span class="tag">说话人分离</span>
|
||||
<span class="tag">Hotwords</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h3>Paraformer</h3>
|
||||
<p>Non-autoregressive Chinese and English ASR with streaming and offline variants for production systems.</p>
|
||||
<div class="card-tags">
|
||||
<span class="tag">Streaming</span>
|
||||
<span class="tag">Offline</span>
|
||||
<span class="tag">Hotwords</span>
|
||||
<span class="tag">ONNX</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h3>SenseVoice</h3>
|
||||
<p>Multi-task speech understanding for ASR, language ID, emotion, and audio events across five languages.</p>
|
||||
<div class="card-tags">
|
||||
<span class="tag">Emotion</span>
|
||||
<span class="tag">Audio Events</span>
|
||||
<span class="tag">说话人分离</span>
|
||||
<span class="tag">70ms/10s</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h3>Qwen3-ASR</h3>
|
||||
<p>LLM-based ASR with 52 languages, contextual understanding, and automatic language detection.</p>
|
||||
<div class="card-tags">
|
||||
<span class="tag">52 Languages</span>
|
||||
<span class="tag">0.6B / 1.7B</span>
|
||||
<span class="tag">Context-aware</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<div class="container narrow">
|
||||
<h2 class="section-title">快速开始</h2>
|
||||
<p class="section-subtitle">Install the package, compose the pipeline, and run recognition from Python.</p>
|
||||
<pre>pip install funasr
|
||||
# Or latest: pip install git+https://github.com/modelscope/FunASR.git</pre>
|
||||
<pre>from funasr import AutoModel
|
||||
|
||||
model = AutoModel(
|
||||
model="paraformer-zh",
|
||||
vad_model="fsmn-vad",
|
||||
punc_model="ct-punc",
|
||||
spk_model="cam++",
|
||||
)
|
||||
res = model.generate(input="meeting.wav", batch_size_s=300)
|
||||
|
||||
for sent in res[0]["sentence_info"]:
|
||||
print(f"[Speaker {sent['spk']}] {sent['text']}")</pre>
|
||||
<div class="section-cta">
|
||||
<a href="tutorial.html" class="btn btn-primary">Full tutorial</a>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="alt">
|
||||
<div class="container">
|
||||
<h2 class="section-title">生态系统</h2>
|
||||
<p class="section-subtitle">Related projects around ASR, speech understanding, video clipping, and voice generation.</p>
|
||||
<div class="grid-2">
|
||||
<div class="card">
|
||||
<h3><a class="card-title-link" href="https://github.com/FunAudioLLM/Fun-ASR">Fun-ASR-Nano</a></h3>
|
||||
<p>The latest ASR large model with multilingual recognition, timestamps, speaker diarization, and hotwords.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h3><a class="card-title-link" href="https://github.com/FunAudioLLM/SenseVoice">SenseVoice</a></h3>
|
||||
<p>Multi-task speech understanding for ASR, emotion detection, and audio event recognition.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h3><a class="card-title-link" href="https://github.com/modelscope/FunClip">FunClip</a></h3>
|
||||
<p>AI video clipping powered by FunASR and LLM-assisted editing workflows.</p>
|
||||
</div>
|
||||
<div class="card">
|
||||
<h3><a class="card-title-link" href="https://github.com/FunAudioLLM/CosyVoice">CosyVoice</a></h3>
|
||||
<p>Natural speech generation with multi-language, timbre, and emotion control.</p>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<footer>
|
||||
<p>FunASR · Tongyi Lab, Alibaba Group</p>
|
||||
<p>
|
||||
<a href="https://github.com/modelscope/FunASR">GitHub</a> ·
|
||||
<a href="training.html">训练</a> ·
|
||||
<a href="../api.html">API</a> ·
|
||||
<a href="https://github.com/modelscope/FunASR/blob/main/MODEL_LICENSE">License</a> ·
|
||||
<a href="https://pypi.org/project/funasr/">PyPI</a> ·
|
||||
<a href="https://github.com/modelscope/FunASR/issues">Issues</a>
|
||||
</p>
|
||||
</footer>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,410 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0">
|
||||
<title>FunASR Developer Guide</title>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700;800&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet">
|
||||
<link rel="stylesheet" href="../style.css">
|
||||
</head><body>
|
||||
<nav class="nav"><div class="container">
|
||||
<a href="index.html" class="nav-logo">FunASR</a>
|
||||
<div class="nav-links">
|
||||
<a href="index.html">首页</a>
|
||||
<a href="tutorial.html">使用教程</a>
|
||||
<a href="training.html">训练</a>
|
||||
<a href="model-registration.html" class="active">开发</a>
|
||||
<a href="../api.html">API</a>
|
||||
</div>
|
||||
<div class="lang-switch">
|
||||
<a href="model-registration.html" title="English">EN</a>
|
||||
<a href="zh/model-registration.html" title="中文">中</a>
|
||||
<a href="ja/model-registration.html" title="日本語">JP</a>
|
||||
</div>
|
||||
<div class="lang-switch">
|
||||
<a href="../model-registration.html" title="English">EN</a>
|
||||
<a href="../zh/model-registration.html" title="中文">中</a>
|
||||
<a href="../ja/model-registration.html" title="日本語">JP</a>
|
||||
</div>
|
||||
<a href="https://github.com/modelscope/FunASR" class="nav-github">GitHub</a>
|
||||
</div></nav>
|
||||
|
||||
<div class="content"><div class="container narrow">
|
||||
<h1>Developer Guide</h1>
|
||||
<p>From understanding the architecture to contributing your own model — a step-by-step guide.</p>
|
||||
|
||||
<div class="toc-grid">
|
||||
<a href="#architecture">1. Architecture Overview</a>
|
||||
<a href="#setup">2. Development Setup</a>
|
||||
<a href="#understand">3. How Inference Works</a>
|
||||
<a href="#add-model">4. Add a New Model</a>
|
||||
<a href="#add-component">5. Add Frontend/Tokenizer</a>
|
||||
<a href="#standalone">6. Standalone Repository</a>
|
||||
<a href="#testing">7. Testing Your Model</a>
|
||||
<a href="#pitfalls">8. Common Pitfalls</a>
|
||||
<a href="#contribute">9. Contributing</a>
|
||||
</div>
|
||||
|
||||
<!-- 1. Architecture -->
|
||||
<h2 id="architecture">1. Architecture Overview</h2>
|
||||
<p>FunASR is built around three core ideas: a <strong>registry</strong> for component discovery, <strong>AutoModel</strong> as the unified entry point, and <strong>config.yaml</strong> as the declarative model definition.</p>
|
||||
|
||||
<div class="arch-box">User Code FunASR Framework Model Hub
|
||||
───────── ──────────────── ─────────
|
||||
AutoModel(model="name")
|
||||
│
|
||||
├─→ download_model() ──────→ ModelScope / HuggingFace
|
||||
│ ↓ ↓
|
||||
│ read config.yaml download model.pt
|
||||
│ ↓
|
||||
├─→ tables.model_classes["Name"] ← @tables.register decorator
|
||||
│ ↓
|
||||
├─→ model_class(**config) ← __init__: build encoder/decoder
|
||||
│ ↓
|
||||
├─→ load_pretrained_model() ← load weights from model.pt
|
||||
│ ↓
|
||||
└─→ model.eval() ← ready for inference
|
||||
|
||||
generate(input="audio.wav")
|
||||
│
|
||||
├─ No VAD → inference() ← single utterance
|
||||
│ ↓
|
||||
│ model.inference(data_in, tokenizer, frontend, **kwargs)
|
||||
│ ↓
|
||||
│ return [{"key", "text", "timestamp", ...}]
|
||||
│
|
||||
└─ With VAD → inference_with_vad() ← long audio
|
||||
↓
|
||||
1. VAD: segment audio → [[start_ms, end_ms], ...]
|
||||
2. Sort segments by length (for efficient batching)
|
||||
3. ASR: recognize each segment
|
||||
4. Merge timestamps (add VAD offset)
|
||||
5. Punctuation (optional)
|
||||
6. Speaker diarization (optional)
|
||||
↓
|
||||
return [{"key", "text", "timestamp", "sentence_info"}]
|
||||
</div>
|
||||
|
||||
<h3>Registry System</h3>
|
||||
<p>Every component in FunASR is registered by name. The registry is the lookup table that connects config strings to Python classes:</p>
|
||||
|
||||
<table>
|
||||
<tr><th>Registry</th><th>Purpose</th><th>Example</th></tr>
|
||||
<tr><td><code>model_classes</code></td><td>ASR, VAD, PUNC, SPK models</td><td>"Paraformer", "FsmnVADStreaming"</td></tr>
|
||||
<tr><td><code>encoder_classes</code></td><td>Encoder architectures</td><td>"SANMEncoder", "ConformerEncoder"</td></tr>
|
||||
<tr><td><code>decoder_classes</code></td><td>Decoder architectures</td><td>"ParaformerSANMDecoder"</td></tr>
|
||||
<tr><td><code>frontend_classes</code></td><td>Audio feature extraction</td><td>"WavFrontend", "WhisperFrontend"</td></tr>
|
||||
<tr><td><code>tokenizer_classes</code></td><td>Text tokenization</td><td>"SentencepiecesTokenizer"</td></tr>
|
||||
<tr><td><code>dataset_classes</code></td><td>Training data loading</td><td>"AudioDataset"</td></tr>
|
||||
</table>
|
||||
|
||||
<pre>from funasr.register import tables
|
||||
|
||||
# Register a new model
|
||||
@tables.register("model_classes", "MyModel")
|
||||
class MyModel(nn.Module):
|
||||
...
|
||||
|
||||
# View all registered models
|
||||
tables.print("model")</pre>
|
||||
|
||||
<!-- 2. Setup -->
|
||||
<h2 id="setup">2. Development Setup</h2>
|
||||
|
||||
<div class="step">
|
||||
<h4>Clone and install in development mode</h4>
|
||||
<pre>git clone https://github.com/modelscope/FunASR.git
|
||||
cd FunASR
|
||||
pip install -e . # editable install
|
||||
pip install -e ".[train]" # with training dependencies</pre>
|
||||
</div>
|
||||
|
||||
<div class="step">
|
||||
<h4>Verify installation</h4>
|
||||
<pre>python -c "from funasr import AutoModel; print('OK')"
|
||||
python -c "from funasr.register import tables; tables.print('model')"</pre>
|
||||
</div>
|
||||
|
||||
<div class="step">
|
||||
<h4>Run existing tests</h4>
|
||||
<pre># Quick smoke test
|
||||
python tests_models/test_fsmn_vad.py
|
||||
python tests_models/test_paraformer.py
|
||||
|
||||
# Full test suite
|
||||
cd tests_models && python run_all_tests.py</pre>
|
||||
</div>
|
||||
|
||||
<!-- 3. How Inference Works -->
|
||||
<h2 id="understand">3. How Inference Works (Deep Dive)</h2>
|
||||
|
||||
<p>Understanding the inference data flow is essential before adding a new model. Here's what happens when you call <code>model.generate(input="audio.wav")</code>:</p>
|
||||
|
||||
<h4>Step 1: Input Preparation</h4>
|
||||
<p><code>prepare_data_iterator()</code> normalizes any input type (file path, URL, numpy, bytes, list) into a uniform <code>(key_list, data_list)</code> format.</p>
|
||||
|
||||
<h4>Step 2: Model Inference</h4>
|
||||
<p>Each model's <code>inference()</code> method receives:</p>
|
||||
<pre>def inference(self, data_in, data_lengths=None, key=None,
|
||||
tokenizer=None, frontend=None, **kwargs):
|
||||
# data_in: list of audio samples (numpy arrays)
|
||||
# tokenizer: for decoding token IDs → text
|
||||
# frontend: for extracting fbank features
|
||||
# **kwargs: all config.yaml params + user runtime params
|
||||
|
||||
# Must return: (results_list, meta_data_dict)
|
||||
return [{"key": "id", "text": "hello", "timestamp": [...]}], {"batch_data_time": 5.5}</pre>
|
||||
|
||||
<h4>Step 3: Output Format</h4>
|
||||
<p>The <code>results_list</code> must be a list of dicts. Required/optional fields:</p>
|
||||
<table>
|
||||
<tr><th>Field</th><th>Type</th><th>Required</th><th>Description</th></tr>
|
||||
<tr><td><code>key</code></td><td>str</td><td>Yes</td><td>Sample identifier</td></tr>
|
||||
<tr><td><code>text</code></td><td>str</td><td>Yes (ASR)</td><td>Recognized text</td></tr>
|
||||
<tr><td><code>timestamp</code></td><td>list</td><td>For SPK</td><td><code>[[start_ms, end_ms], ...]</code> per character</td></tr>
|
||||
<tr><td><code>value</code></td><td>list</td><td>VAD only</td><td><code>[[start_ms, end_ms], ...]</code> speech segments</td></tr>
|
||||
<tr><td><code>spk_embedding</code></td><td>Tensor</td><td>SPK only</td><td>Shape <code>[N, 192]</code></td></tr>
|
||||
</table>
|
||||
|
||||
<div class="warning">
|
||||
<strong>Timestamp format matters!</strong> If your model outputs timestamps differently (e.g., dict format like Fun-ASR-Nano), FunASR's <code>inference_with_vad</code> will auto-convert it. But the standard expected format is <code>[[start_ms, end_ms], ...]</code> (list of 2-element lists in milliseconds).
|
||||
</div>
|
||||
|
||||
<!-- 4. Add a New Model -->
|
||||
<h2 id="add-model">4. Add a New Model</h2>
|
||||
|
||||
<div class="step">
|
||||
<h4>Create model directory</h4>
|
||||
<pre>funasr/models/my_model/
|
||||
├── __init__.py # empty file
|
||||
├── model.py # main model class
|
||||
├── encoder.py # (optional) custom encoder
|
||||
└── decoder.py # (optional) custom decoder</pre>
|
||||
<div class="warning"><strong>Rule:</strong> Each model directory is self-contained. Never import from other model directories. Never modify existing models.</div>
|
||||
</div>
|
||||
|
||||
<div class="step">
|
||||
<h4>Implement the model class</h4>
|
||||
<pre>import torch.nn as nn
|
||||
from funasr.register import tables
|
||||
|
||||
@tables.register("model_classes", "MyModel")
|
||||
class MyModel(nn.Module):
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__() # ← MUST call super().__init__()
|
||||
# Build your architecture from kwargs (comes from config.yaml)
|
||||
# kwargs includes: input_size, vocab_size, tokenizer, frontend, etc.
|
||||
|
||||
def forward(self, speech, speech_lengths, text, text_lengths, **kwargs):
|
||||
"""Training forward pass. Return (loss, stats_dict, weight)."""
|
||||
...
|
||||
|
||||
def inference(self, data_in, data_lengths=None, key=None,
|
||||
tokenizer=None, frontend=None, **kwargs):
|
||||
"""Inference. Return (results_list, meta_data)."""
|
||||
...
|
||||
</pre>
|
||||
</div>
|
||||
|
||||
<div class="step">
|
||||
<h4>Create config.yaml</h4>
|
||||
<pre># This file defines what components to use
|
||||
model: MyModel # matches @tables.register key
|
||||
model_conf:
|
||||
hidden_size: 512
|
||||
|
||||
frontend: WavFrontend # reuse existing frontend
|
||||
frontend_conf:
|
||||
fs: 16000
|
||||
n_mels: 80
|
||||
frame_length: 25
|
||||
frame_shift: 10
|
||||
cmvn_file: null
|
||||
|
||||
tokenizer: SentencepiecesTokenizer
|
||||
tokenizer_conf:
|
||||
bpemodel: null</pre>
|
||||
</div>
|
||||
|
||||
<div class="step">
|
||||
<h4>Create configuration.json (for Hub upload)</h4>
|
||||
<pre>{
|
||||
"framework": "pytorch",
|
||||
"task": "auto-speech-recognition",
|
||||
"model": {"type": "funasr"},
|
||||
"file_path_metas": {
|
||||
"init_param": "model.pt",
|
||||
"config": "config.yaml",
|
||||
"tokenizer_conf": {"bpemodel": "my_tokenizer.model"},
|
||||
"frontend_conf": {"cmvn_file": "am.mvn"}
|
||||
}
|
||||
}</pre>
|
||||
<p>This file tells AutoModel how to resolve relative paths. When the model is downloaded, each path in <code>file_path_metas</code> gets the model directory prepended.</p>
|
||||
</div>
|
||||
|
||||
<div class="step">
|
||||
<h4>Test locally</h4>
|
||||
<pre>from funasr import AutoModel
|
||||
|
||||
# From local directory
|
||||
model = AutoModel(model="./my_model_dir")
|
||||
res = model.generate(input="test.wav")
|
||||
print(res)</pre>
|
||||
</div>
|
||||
|
||||
<!-- 5. Add Component -->
|
||||
<h2 id="add-component">5. Add a New Frontend / Tokenizer / Dataset</h2>
|
||||
<p>The same registry pattern applies to all components. Example — adding a new frontend:</p>
|
||||
|
||||
<pre>from funasr.register import tables
|
||||
|
||||
@tables.register("frontend_classes", "MyFrontend")
|
||||
class MyFrontend(nn.Module):
|
||||
def __init__(self, fs=16000, **kwargs):
|
||||
super().__init__()
|
||||
self.fs = fs
|
||||
|
||||
def output_size(self):
|
||||
return 80 # feature dimension
|
||||
|
||||
def forward(self, input, input_lengths):
|
||||
# input: raw waveform (batch, samples)
|
||||
# return: features (batch, frames, dim), lengths
|
||||
...</pre>
|
||||
|
||||
<p>Then reference it in config.yaml:</p>
|
||||
<pre>frontend: MyFrontend
|
||||
frontend_conf:
|
||||
fs: 16000</pre>
|
||||
|
||||
<p>Same pattern for tokenizer (<code>tokenizer_classes</code>), dataset (<code>dataset_classes</code>), encoder (<code>encoder_classes</code>), etc.</p>
|
||||
|
||||
<!-- 6. Standalone -->
|
||||
<h2 id="standalone">6. Standalone Repository Mode</h2>
|
||||
<p>Your model doesn't need to live inside the FunASR source tree. With <code>trust_remote_code=True</code>, FunASR dynamically loads your model class from an external file:</p>
|
||||
|
||||
<pre># User code — loads YOUR model.py from a separate repo
|
||||
model = AutoModel(
|
||||
model="your-org/your-model", # HuggingFace/ModelScope repo
|
||||
trust_remote_code=True,
|
||||
remote_code="./model.py", # path to model class definition
|
||||
hub="hf",
|
||||
)</pre>
|
||||
|
||||
<p><strong>How it works:</strong></p>
|
||||
<ol>
|
||||
<li>FunASR downloads the model repo (weights + config + model.py)</li>
|
||||
<li><code>remote_code="./model.py"</code> is dynamically imported</li>
|
||||
<li>The <code>@tables.register</code> decorator in that file registers the model class</li>
|
||||
<li>Normal <code>build_model()</code> flow proceeds with the registered class</li>
|
||||
</ol>
|
||||
|
||||
<p><strong>Your repo structure:</strong></p>
|
||||
<pre>your-model-repo/
|
||||
├── model.py # model class with @tables.register
|
||||
├── config.yaml # model architecture config
|
||||
├── configuration.json # path resolution
|
||||
├── model.pt # trained weights
|
||||
└── example/test.wav # demo audio</pre>
|
||||
|
||||
<p>Examples: <a href="https://github.com/FunAudioLLM/Fun-ASR">Fun-ASR-Nano</a>, <a href="https://github.com/FunAudioLLM/SenseVoice">SenseVoice</a></p>
|
||||
|
||||
<!-- 7. Testing -->
|
||||
<h2 id="testing">7. Testing Your Model</h2>
|
||||
|
||||
<h4>Write a test script</h4>
|
||||
<pre># tests_models/test_my_model.py
|
||||
import sys, time
|
||||
from funasr import AutoModel
|
||||
|
||||
def main():
|
||||
model = AutoModel(model="path/to/model", device="cpu", disable_update=True)
|
||||
res = model.generate(input="test.wav")
|
||||
|
||||
assert res and len(res) > 0, "empty result"
|
||||
assert "text" in res[0], "missing text field"
|
||||
print("PASSED")
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())</pre>
|
||||
|
||||
<h4>Test with VAD + SPK pipeline</h4>
|
||||
<pre># If your model should work with speaker diarization:
|
||||
model = AutoModel(
|
||||
model="path/to/model",
|
||||
vad_model="fsmn-vad",
|
||||
spk_model="cam++",
|
||||
)
|
||||
res = model.generate(input="meeting.wav", cache={})
|
||||
assert "sentence_info" in res[0]
|
||||
assert "spk" in res[0]["sentence_info"][0]</pre>
|
||||
|
||||
<h4>Test streaming (if applicable)</h4>
|
||||
<pre>cache = {}
|
||||
for i in range(total_chunks):
|
||||
chunk = audio[i*stride:(i+1)*stride]
|
||||
res = model.generate(input=chunk, cache=cache,
|
||||
is_final=(i == total_chunks-1), ...)
|
||||
# Verify: same audio gives same result across multiple sessions</pre>
|
||||
|
||||
<!-- 8. Pitfalls -->
|
||||
<h2 id="pitfalls">8. Common Pitfalls</h2>
|
||||
|
||||
<h4>❌ Forgetting <code>super().__init__()</code></h4>
|
||||
<pre># WRONG — causes "object has no attribute '_state_dict_pre_hooks'"
|
||||
class MyEncoder(nn.Module):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
# CORRECT
|
||||
class MyEncoder(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()</pre>
|
||||
|
||||
<h4>❌ Checking <code>kwargs["batch_size"]</code> in your model</h4>
|
||||
<p><code>batch_size</code> in kwargs is set by <code>inference_with_vad</code> for segment batching (a large number in ms). Don't use it to check actual data batch size. Use <code>len(data_in)</code> instead.</p>
|
||||
|
||||
<h4>❌ Not handling empty/short input</h4>
|
||||
<p>VAD may produce empty segments. Your <code>inference()</code> should handle <code>data_in = []</code> gracefully.</p>
|
||||
|
||||
<h4>❌ Timestamp format mismatch</h4>
|
||||
<p>If your model returns timestamps as dicts (<code>{"start_time": 0.5, "end_time": 0.8}</code>), the pipeline handles conversion. But if you output <code>[start_time, end_time, text]</code> (3 elements), strip the text — downstream expects <code>[start_ms, end_ms]</code> (2 elements, in milliseconds).</p>
|
||||
|
||||
<h4>❌ Importing from other model directories</h4>
|
||||
<pre># WRONG — creates tight coupling
|
||||
from funasr.models.paraformer.model import Paraformer
|
||||
|
||||
# CORRECT — copy what you need into your own directory
|
||||
# Or inherit via the registry name in config.yaml</pre>
|
||||
|
||||
<h4>❌ Modifying <code>self.kwargs</code> during inference</h4>
|
||||
<p>Don't mutate <code>kwargs</code> that came from AutoModel. The framework resets state between calls, but persistent mutations can leak between sessions.</p>
|
||||
|
||||
<!-- 9. Contributing -->
|
||||
<h2 id="contribute">9. Contributing</h2>
|
||||
|
||||
<h4>Code Style</h4>
|
||||
<ul>
|
||||
<li>Follow existing patterns — look at <code>paraformer/model.py</code> as reference</li>
|
||||
<li>Add docstrings to all public methods (Args, Returns)</li>
|
||||
<li>No comments explaining WHAT — use clear naming. Comments for WHY only.</li>
|
||||
</ul>
|
||||
|
||||
<h4>PR Checklist</h4>
|
||||
<ul>
|
||||
<li>New model: self-contained directory, no cross-model imports</li>
|
||||
<li>Include a test script in <code>tests_models/</code></li>
|
||||
<li>Include a demo in <code>examples/industrial_data_pretraining/</code></li>
|
||||
<li>All existing tests still pass</li>
|
||||
<li>Add entry to README What's New if user-facing</li>
|
||||
</ul>
|
||||
|
||||
<h4>License</h4>
|
||||
<p>Code: MIT. Model weights: FunASR Model License (commercial use allowed with attribution).</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
<footer>
|
||||
<p>FunASR · Tongyi Lab, Alibaba Group</p>
|
||||
<p><a href="index.html">首页</a> · <a href="tutorial.html">使用教程</a> · <a href="training.html">训练</a> · <a href="model-registration.html">开发</a> · <a href="../api.html">API</a> · <a href="https://github.com/modelscope/FunASR">GitHub</a></p>
|
||||
</footer>
|
||||
</body></html>
|
||||
@@ -0,0 +1,335 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0">
|
||||
<title>FunASR Training Guide</title>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700;800&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet">
|
||||
<link rel="stylesheet" href="../style.css">
|
||||
</head><body>
|
||||
<nav class="nav"><div class="container">
|
||||
<a href="index.html" class="nav-logo">FunASR</a>
|
||||
<div class="nav-links">
|
||||
<a href="index.html">首页</a>
|
||||
<a href="tutorial.html">使用教程</a>
|
||||
<a href="training.html" class="active">训练</a>
|
||||
<a href="model-registration.html">开发</a>
|
||||
<a href="../api.html">API</a>
|
||||
</div>
|
||||
<div class="lang-switch">
|
||||
<a href="training.html" title="English">EN</a>
|
||||
<a href="zh/training.html" title="中文">中</a>
|
||||
<a href="ja/training.html" title="日本語">JP</a>
|
||||
</div>
|
||||
<div class="lang-switch">
|
||||
<a href="../training.html" title="English">EN</a>
|
||||
<a href="../zh/training.html" title="中文">中</a>
|
||||
<a href="../ja/training.html" title="日本語">JP</a>
|
||||
</div>
|
||||
<a href="https://github.com/modelscope/FunASR" class="nav-github">GitHub</a>
|
||||
</div></nav>
|
||||
|
||||
<div class="content"><div class="container narrow">
|
||||
<h1>Training & Fine-tuning</h1>
|
||||
<p>Fine-tune pretrained models on your own data using FunASR's training framework.</p>
|
||||
|
||||
<div class="toc-grid">
|
||||
<a href="#overview">Overview</a>
|
||||
<a href="#data">Data Preparation</a>
|
||||
<a href="#paraformer">Fine-tune Paraformer</a>
|
||||
<a href="#sensevoice">Fine-tune SenseVoice</a>
|
||||
<a href="#nano">Fine-tune Fun-ASR-Nano</a>
|
||||
<a href="#params">Parameter Reference</a>
|
||||
<a href="#multi-gpu">Multi-GPU Training</a>
|
||||
<a href="#deepspeed">DeepSpeed</a>
|
||||
<a href="#monitor">Monitoring</a>
|
||||
<a href="#inference-after">Use Fine-tuned Model</a>
|
||||
<a href="#tips">Tips & Troubleshooting</a>
|
||||
</div>
|
||||
|
||||
<!-- Overview -->
|
||||
<h2 id="overview">Overview</h2>
|
||||
<p>FunASR's training framework supports:</p>
|
||||
<ul>
|
||||
<li><strong>Fine-tuning</strong> any pretrained model on custom domain data</li>
|
||||
<li><strong>Multi-GPU</strong> training with PyTorch DDP (single/multi-node)</li>
|
||||
<li><strong>DeepSpeed</strong> ZeRO Stage 1/2/3 for large model training</li>
|
||||
<li><strong>Dynamic batching</strong> by token count or example count</li>
|
||||
<li><strong>Checkpoint averaging</strong> for best performance</li>
|
||||
<li><strong>Resume training</strong> from interruption</li>
|
||||
</ul>
|
||||
|
||||
<p>The training entry point is <code>funasr-train-ds</code> (or <code>funasr/bin/train_ds.py</code>), launched via <code>torchrun</code> for distributed training.</p>
|
||||
|
||||
<!-- Data -->
|
||||
<h2 id="data">Data Preparation</h2>
|
||||
|
||||
<h3>Standard Format (Paraformer, SenseVoice)</h3>
|
||||
<p>Training data uses JSONL format — one JSON object per line:</p>
|
||||
<pre>{"key": "utt001", "source": "/path/to/audio.wav", "source_len": 90, "target": "这是转写文本", "target_len": 6}
|
||||
{"key": "utt002", "source": "/path/to/audio2.wav", "source_len": 150, "target": "hello world", "target_len": 2}</pre>
|
||||
|
||||
<table>
|
||||
<tr><th>Field</th><th>Type</th><th>Description</th></tr>
|
||||
<tr><td><code>key</code></td><td>str</td><td>Unique utterance ID</td></tr>
|
||||
<tr><td><code>source</code></td><td>str</td><td>Audio file path (local path or URL)</td></tr>
|
||||
<tr><td><code>source_len</code></td><td>int</td><td>Audio length in fbank frames (1 frame = 10ms)</td></tr>
|
||||
<tr><td><code>target</code></td><td>str</td><td>Transcription text</td></tr>
|
||||
<tr><td><code>target_len</code></td><td>int</td><td>Number of text tokens</td></tr>
|
||||
</table>
|
||||
|
||||
<h4>Generate from wav.scp + text.txt</h4>
|
||||
<p>If you have Kaldi-style data files, convert them:</p>
|
||||
<pre># train_wav.scp (tab-separated: id \t path)
|
||||
utt001 /data/audio/001.wav
|
||||
utt002 /data/audio/002.wav
|
||||
|
||||
# train_text.txt (tab-separated: id \t text)
|
||||
utt001 这是转写文本
|
||||
utt002 hello world</pre>
|
||||
|
||||
<pre># Convert to jsonl
|
||||
scp2jsonl \
|
||||
++scp_file_list='["/data/list/train_wav.scp", "/data/list/train_text.txt"]' \
|
||||
++data_type_list='["source", "target"]' \
|
||||
++jsonl_file_out="/data/list/train.jsonl"</pre>
|
||||
|
||||
<h3>ChatML Format (Fun-ASR-Nano)</h3>
|
||||
<p>Fun-ASR-Nano uses ChatML conversation format:</p>
|
||||
<pre>{"messages": [
|
||||
{"role": "system", "content": "You are a helpful assistant."},
|
||||
{"role": "user", "content": "语音转写:<|startofspeech|>!/path/to/audio.wav<|endofspeech|>"},
|
||||
{"role": "assistant", "content": "几点了?"}
|
||||
], "speech_length": 145, "text_length": 3}</pre>
|
||||
|
||||
<table>
|
||||
<tr><th>Field</th><th>Description</th></tr>
|
||||
<tr><td><code>messages[0]</code></td><td>System prompt (fixed: "You are a helpful assistant.")</td></tr>
|
||||
<tr><td><code>messages[1]</code></td><td>User: prompt + audio path wrapped in <code><|startofspeech|>!...<|endofspeech|></code></td></tr>
|
||||
<tr><td><code>messages[2]</code></td><td>Assistant: transcription text</td></tr>
|
||||
<tr><td><code>speech_length</code></td><td>Number of fbank frames (10ms each)</td></tr>
|
||||
<tr><td><code>text_length</code></td><td>Number of tokens (tokenized by Qwen3-0.6B)</td></tr>
|
||||
</table>
|
||||
|
||||
<div class="tip"><strong>Prompt variations:</strong><br>
|
||||
• Chinese: <code>语音转写:</code><br>
|
||||
• English: <code>Speech transcription:</code><br>
|
||||
• Cross-language: <code>语音转写成英文:</code><br>
|
||||
• No ITN: <code>语音转写,不进行文本规整:</code></div>
|
||||
|
||||
<p>Convert from wav.scp + text.txt:</p>
|
||||
<pre>python tools/scp2jsonl.py \
|
||||
++scp_file=data/train_wav.scp \
|
||||
++transcript_file=data/train_text.txt \
|
||||
++jsonl_file=data/train_example.jsonl</pre>
|
||||
|
||||
<!-- Paraformer -->
|
||||
<h2 id="paraformer">Fine-tune Paraformer</h2>
|
||||
<pre>cd examples/industrial_data_pretraining/paraformer
|
||||
bash finetune.sh</pre>
|
||||
|
||||
<p>Or customize the key parameters:</p>
|
||||
<pre>export CUDA_VISIBLE_DEVICES="0,1"
|
||||
gpu_num=2
|
||||
|
||||
torchrun --nproc_per_node $gpu_num \
|
||||
funasr/bin/train_ds.py \
|
||||
++model="iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch" \
|
||||
++train_data_set_list="data/train.jsonl" \
|
||||
++valid_data_set_list="data/val.jsonl" \
|
||||
++dataset_conf.batch_size=6000 \
|
||||
++dataset_conf.batch_type="token" \
|
||||
++dataset_conf.num_workers=4 \
|
||||
++train_conf.max_epoch=50 \
|
||||
++train_conf.validate_interval=2000 \
|
||||
++train_conf.save_checkpoint_interval=2000 \
|
||||
++train_conf.keep_nbest_models=20 \
|
||||
++train_conf.avg_nbest_model=10 \
|
||||
++optim_conf.lr=0.0002 \
|
||||
++output_dir="./outputs"</pre>
|
||||
|
||||
<!-- SenseVoice -->
|
||||
<h2 id="sensevoice">Fine-tune SenseVoice</h2>
|
||||
<pre>cd examples/industrial_data_pretraining/sense_voice
|
||||
bash finetune.sh</pre>
|
||||
|
||||
<p>Same data format as Paraformer (source/target JSONL). Key difference: SenseVoice uses its own dataset class internally.</p>
|
||||
|
||||
<!-- Fun-ASR-Nano -->
|
||||
<h2 id="nano">Fine-tune Fun-ASR-Nano</h2>
|
||||
<pre>cd examples/industrial_data_pretraining/fun_asr_nano
|
||||
bash finetune.sh</pre>
|
||||
|
||||
<p>Key differences from Paraformer:</p>
|
||||
<ul>
|
||||
<li>Uses ChatML data format (see above)</li>
|
||||
<li><code>++trust_remote_code=true</code> required</li>
|
||||
<li>Supports <strong>selective freezing</strong>: freeze encoder/adaptor while training LLM decoder</li>
|
||||
</ul>
|
||||
|
||||
<pre># Freeze encoder + adaptor, only train LLM (recommended for domain adaptation)
|
||||
++audio_encoder_conf.freeze=true
|
||||
++audio_adaptor_conf.freeze=true
|
||||
++llm_conf.freeze=false
|
||||
|
||||
# Full fine-tune (all parameters)
|
||||
++audio_encoder_conf.freeze=false
|
||||
++audio_adaptor_conf.freeze=false
|
||||
++llm_conf.freeze=false</pre>
|
||||
|
||||
<div class="note"><strong>Recommended strategy:</strong> Start with LLM-only fine-tuning (faster, less data needed). If results are insufficient, unfreeze adaptor. Only unfreeze encoder with very large datasets (>1000h).</div>
|
||||
|
||||
<!-- Parameters -->
|
||||
<h2 id="params">Parameter Reference</h2>
|
||||
|
||||
<h3>Dataset Parameters</h3>
|
||||
<table>
|
||||
<tr><th>Parameter</th><th>Default</th><th>Description</th></tr>
|
||||
<tr><td><code>dataset_conf.batch_type</code></td><td>"token"</td><td><code>"token"</code>: dynamic batch by total tokens. <code>"example"</code>: fixed batch count.</td></tr>
|
||||
<tr><td><code>dataset_conf.batch_size</code></td><td>6000</td><td>Token mode: total frames per batch. Example mode: number of samples.</td></tr>
|
||||
<tr><td><code>dataset_conf.sort_size</code></td><td>1024</td><td>Buffer size for length-based sorting (improves padding efficiency).</td></tr>
|
||||
<tr><td><code>dataset_conf.num_workers</code></td><td>4</td><td>Data loading threads.</td></tr>
|
||||
<tr><td><code>dataset_conf.data_split_num</code></td><td>1</td><td>Split data into N groups for large-scale training (reduces memory).</td></tr>
|
||||
<tr><td><code>dataset_conf.max_token_length</code></td><td>—</td><td>Filter: skip samples longer than this (in frames/tokens).</td></tr>
|
||||
<tr><td><code>dataset_conf.min_token_length</code></td><td>—</td><td>Filter: skip samples shorter than this.</td></tr>
|
||||
</table>
|
||||
|
||||
<h3>Training Parameters</h3>
|
||||
<table>
|
||||
<tr><th>Parameter</th><th>Default</th><th>Description</th></tr>
|
||||
<tr><td><code>train_conf.max_epoch</code></td><td>50</td><td>Total training epochs.</td></tr>
|
||||
<tr><td><code>train_conf.log_interval</code></td><td>1</td><td>Print loss every N steps.</td></tr>
|
||||
<tr><td><code>train_conf.validate_interval</code></td><td>2000</td><td>Run validation every N steps.</td></tr>
|
||||
<tr><td><code>train_conf.save_checkpoint_interval</code></td><td>2000</td><td>Save model every N steps.</td></tr>
|
||||
<tr><td><code>train_conf.keep_nbest_models</code></td><td>20</td><td>Keep top N models (by validation accuracy).</td></tr>
|
||||
<tr><td><code>train_conf.avg_nbest_model</code></td><td>10</td><td>Average top N models for final checkpoint.</td></tr>
|
||||
<tr><td><code>train_conf.resume</code></td><td>true</td><td>Resume from last checkpoint if exists.</td></tr>
|
||||
<tr><td><code>train_conf.use_deepspeed</code></td><td>false</td><td>Enable DeepSpeed ZeRO optimization.</td></tr>
|
||||
<tr><td><code>optim_conf.lr</code></td><td>0.0002</td><td>Learning rate.</td></tr>
|
||||
</table>
|
||||
|
||||
<!-- Multi-GPU -->
|
||||
<h2 id="multi-gpu">Multi-GPU Training</h2>
|
||||
|
||||
<h3>Single Machine, Multiple GPUs</h3>
|
||||
<pre>export CUDA_VISIBLE_DEVICES="0,1,2,3"
|
||||
gpu_num=4
|
||||
|
||||
torchrun --nnodes 1 --nproc_per_node $gpu_num \
|
||||
funasr/bin/train_ds.py ${train_args}</pre>
|
||||
|
||||
<h3>Multiple Machines</h3>
|
||||
<pre># Machine 1 (master, IP=192.168.1.1)
|
||||
torchrun --nnodes 2 --node_rank 0 --nproc_per_node 4 \
|
||||
--master_addr=192.168.1.1 --master_port=12345 \
|
||||
funasr/bin/train_ds.py ${train_args}
|
||||
|
||||
# Machine 2
|
||||
torchrun --nnodes 2 --node_rank 1 --nproc_per_node 4 \
|
||||
--master_addr=192.168.1.1 --master_port=12345 \
|
||||
funasr/bin/train_ds.py ${train_args}</pre>
|
||||
|
||||
<!-- DeepSpeed -->
|
||||
<h2 id="deepspeed">DeepSpeed</h2>
|
||||
<p>For large models (Fun-ASR-Nano 800M params), enable DeepSpeed ZeRO:</p>
|
||||
<pre>++train_conf.use_deepspeed=true
|
||||
++train_conf.deepspeed_config=./deepspeed_conf/ds_stage1.json</pre>
|
||||
|
||||
<p>Stage 1 config (recommended starting point):</p>
|
||||
<pre>{
|
||||
"train_micro_batch_size_per_gpu": 1,
|
||||
"gradient_accumulation_steps": 1,
|
||||
"bf16": {"enabled": true},
|
||||
"zero_optimization": {
|
||||
"stage": 1,
|
||||
"reduce_bucket_size": 5e8,
|
||||
"allgather_bucket_size": 5e8
|
||||
}
|
||||
}</pre>
|
||||
|
||||
<div class="note"><strong>When to use which stage:</strong><br>
|
||||
• Stage 1: Optimizer state partitioned. Good for most cases.<br>
|
||||
• Stage 2: + Gradient partitioned. For larger models.<br>
|
||||
• Stage 3: + Parameter partitioned. Maximum memory savings but slower.</div>
|
||||
|
||||
<!-- Monitor -->
|
||||
<h2 id="monitor">Monitoring Training</h2>
|
||||
|
||||
<h4>Log file</h4>
|
||||
<pre>tail -f outputs/log.txt
|
||||
|
||||
# Example output:
|
||||
# train, rank: 0, epoch: 0/50, step: 6990, (loss_avg_rank: 0.327),
|
||||
# (acc_avg_epoch: 0.795), (lr: 1.165e-04),
|
||||
# GPU memory: usage: 3.8GB, peak: 18.3GB</pre>
|
||||
|
||||
<p>Key metrics to watch:</p>
|
||||
<ul>
|
||||
<li><code>loss_avg_epoch</code>: should decrease over time</li>
|
||||
<li><code>acc_avg_epoch</code>: should increase (most important metric)</li>
|
||||
<li><code>lr</code>: learning rate at current step</li>
|
||||
<li><code>GPU memory</code>: peak should not exceed your GPU VRAM</li>
|
||||
</ul>
|
||||
|
||||
<h4>TensorBoard</h4>
|
||||
<pre>tensorboard --logdir outputs/log/tensorboard
|
||||
# Open http://localhost:6006</pre>
|
||||
|
||||
<!-- Use Fine-tuned Model -->
|
||||
<h2 id="inference-after">Use Your Fine-tuned Model</h2>
|
||||
|
||||
<h4>If outputs/ has configuration.json</h4>
|
||||
<pre>from funasr import AutoModel
|
||||
model = AutoModel(model="./outputs")
|
||||
res = model.generate(input="test.wav")
|
||||
print(res[0]["text"])</pre>
|
||||
|
||||
<h4>If no configuration.json</h4>
|
||||
<pre>funasr ++model="./outputs" \
|
||||
++config-path="./outputs" \
|
||||
++config-name="config.yaml" \
|
||||
++init_param="./outputs/model.pt" \
|
||||
++input="test.wav"</pre>
|
||||
|
||||
<!-- Tips -->
|
||||
<h2 id="tips">Tips & Troubleshooting</h2>
|
||||
|
||||
<h4>OOM during training</h4>
|
||||
<ol>
|
||||
<li>Reduce <code>dataset_conf.batch_size</code></li>
|
||||
<li>Add <code>dataset_conf.max_token_length=2000</code> to filter long utterances</li>
|
||||
<li>Enable DeepSpeed (partitions optimizer states)</li>
|
||||
<li>Reduce <code>dataset_conf.num_workers</code></li>
|
||||
</ol>
|
||||
|
||||
<h4>Training loss stuck / NaN gradients</h4>
|
||||
<ul>
|
||||
<li>Reduce learning rate (try 0.00005)</li>
|
||||
<li>Check data quality — corrupted audio files cause NaN</li>
|
||||
<li>For Fun-ASR-Nano: start with encoder frozen</li>
|
||||
</ul>
|
||||
|
||||
<h4>Validation accuracy not improving</h4>
|
||||
<ul>
|
||||
<li>Increase training data (min ~10h for fine-tuning)</li>
|
||||
<li>Check domain match — model may not generalize to very different domains</li>
|
||||
<li>Try unfreezing more layers gradually</li>
|
||||
</ul>
|
||||
|
||||
<h4>Large-scale data (>10,000 hours)</h4>
|
||||
<p>Use data splitting to avoid memory issues:</p>
|
||||
<pre># Split data into chunks, load 2 at a time
|
||||
++dataset_conf.data_split_num=256
|
||||
# data.list contains paths to split jsonl files:
|
||||
# data/train.0.jsonl
|
||||
# data/train.1.jsonl
|
||||
# ...
|
||||
++train_data_set_list="data/data.list"</pre>
|
||||
|
||||
<h4>Resume after crash</h4>
|
||||
<p>Set <code>++train_conf.resume=true</code> (default). Training automatically restarts from the latest checkpoint in <code>output_dir</code>.</p>
|
||||
|
||||
</div></div>
|
||||
|
||||
<footer>
|
||||
<p>FunASR · Tongyi Lab, Alibaba Group</p>
|
||||
<p><a href="index.html">首页</a> · <a href="tutorial.html">使用教程</a> · <a href="training.html">训练</a> · <a href="model-registration.html">开发</a> · <a href="../api.html">API</a> · <a href="https://github.com/modelscope/FunASR">GitHub</a></p>
|
||||
</footer>
|
||||
</body></html>
|
||||
@@ -0,0 +1,383 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0">
|
||||
<title>FunASR Tutorial</title>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700;800&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet">
|
||||
<link rel="stylesheet" href="../style.css">
|
||||
</head><body>
|
||||
<nav class="nav"><div class="container">
|
||||
<a href="index.html" class="nav-logo">FunASR</a>
|
||||
<div class="nav-links">
|
||||
<a href="index.html">首页</a>
|
||||
<a href="tutorial.html" class="active">使用教程</a>
|
||||
<a href="training.html">训练</a>
|
||||
<a href="model-registration.html">开发</a>
|
||||
<a href="../api.html">API</a>
|
||||
</div>
|
||||
<div class="lang-switch">
|
||||
<a href="tutorial.html" title="English">EN</a>
|
||||
<a href="zh/tutorial.html" title="中文">中</a>
|
||||
<a href="ja/tutorial.html" title="日本語">JP</a>
|
||||
</div>
|
||||
<div class="lang-switch">
|
||||
<a href="../tutorial.html" title="English">EN</a>
|
||||
<a href="../zh/tutorial.html" title="中文">中</a>
|
||||
<a href="../ja/tutorial.html" title="日本語">JP</a>
|
||||
</div>
|
||||
<a href="https://github.com/modelscope/FunASR" class="nav-github">GitHub</a>
|
||||
</div></nav>
|
||||
|
||||
<div class="content"><div class="container narrow">
|
||||
<h1>使用教程</h1>
|
||||
<p>From zero to production — learn FunASR in 10 minutes.</p>
|
||||
|
||||
<div class="toc-grid">
|
||||
<a href="#quick">30秒演示</a>
|
||||
<a href="#install">安装</a>
|
||||
<a href="#choose">Choose a Model</a>
|
||||
<a href="#scenarios">使用场景</a>
|
||||
<a href="#offline">离线识别</a>
|
||||
<a href="#streaming">流式识别</a>
|
||||
<a href="#spk">说话人分离</a>
|
||||
<a href="#emotion">情感检测</a>
|
||||
<a href="#vad">VAD</a>
|
||||
<a href="#punc">标点恢复</a>
|
||||
<a href="#export">ONNX导出</a>
|
||||
<a href="#faq">常见问题</a>
|
||||
</div>
|
||||
|
||||
<!-- 30-Second Demo -->
|
||||
<h2 id="quick">30秒演示</h2>
|
||||
<p>Copy-paste this and run. No config needed — FunASR downloads everything automatically:</p>
|
||||
<pre>pip install funasr
|
||||
|
||||
python -c "
|
||||
from funasr import AutoModel
|
||||
model = AutoModel(model='paraformer-zh', vad_model='fsmn-vad', punc_model='ct-punc')
|
||||
res = model.generate(input='https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav')
|
||||
print(res[0]['text'])
|
||||
"</pre>
|
||||
<p><strong>Output:</strong></p>
|
||||
<div class="output-box">欢迎大家来体验达摩院推出的语音识别模型。</div>
|
||||
|
||||
<p>That's it. One model object handles everything: loading audio, VAD segmentation, ASR, and punctuation.</p>
|
||||
|
||||
<!-- Install -->
|
||||
<h2 id="install">安装</h2>
|
||||
<pre># Stable release
|
||||
pip install funasr
|
||||
|
||||
# Latest (recommended — has newest models and bug fixes)
|
||||
pip install git+https://github.com/modelscope/FunASR.git</pre>
|
||||
|
||||
<div class="tip"><strong>China users:</strong> Models download from ModelScope by default (fast in China). International users: add <code>hub="hf"</code> to download from HuggingFace.</div>
|
||||
|
||||
<!-- Choose a Model -->
|
||||
<h2 id="choose">选择哪个模型?</h2>
|
||||
|
||||
<table>
|
||||
<tr><th>Model</th><th>Best For</th><th>Languages</th><th>Speed</th><th>标点恢复</th></tr>
|
||||
<tr><td><strong>Paraformer</strong></td><td>Chinese production ASR</td><td>Chinese, English</td><td>Fast</td><td>Needs punc_model</td></tr>
|
||||
<tr><td><strong>Fun-ASR-Nano</strong></td><td>Multi-language, dialects, lyrics</td><td>31 languages</td><td>Medium</td><td>Built-in ✓</td></tr>
|
||||
<tr><td><strong>SenseVoice</strong></td><td>Emotion + events + ASR</td><td>5 languages</td><td>Ultra-fast (70ms/10s)</td><td>Built-in ✓</td></tr>
|
||||
<tr><td><strong>Qwen3-ASR</strong></td><td>Highest accuracy, context-aware</td><td>52 languages</td><td>Slow (LLM)</td><td>Built-in ✓</td></tr>
|
||||
<tr><td><strong>Paraformer-Streaming</strong></td><td>Real-time transcription</td><td>Chinese</td><td>Real-time</td><td>Needs punc_model</td></tr>
|
||||
</table>
|
||||
|
||||
<div class="note">
|
||||
<strong>Quick recommendation:</strong><br>
|
||||
• Chinese meeting/call → <code>paraformer-zh</code> + <code>fsmn-vad</code> + <code>ct-punc</code> + <code>cam++</code><br>
|
||||
• Multi-language → <code>Fun-ASR-Nano</code><br>
|
||||
• Need emotion/sound events → <code>SenseVoice</code><br>
|
||||
• Real-time subtitles → <code>paraformer-zh-streaming</code><br>
|
||||
• Highest quality, no latency concern → <code>Qwen3-ASR</code>
|
||||
</div>
|
||||
|
||||
<!-- Scenarios -->
|
||||
<h2 id="scenarios">使用场景</h2>
|
||||
|
||||
<div class="scenario-card">
|
||||
<h4>🎤 "I have a meeting recording and want text + who said what"</h4>
|
||||
<p>→ Paraformer + VAD + Punctuation + Speaker Diarization → <a href="#spk">Jump to Speaker Diarization</a></p>
|
||||
</div>
|
||||
|
||||
<div class="scenario-card">
|
||||
<h4>📺 "I want real-time subtitles for a live stream"</h4>
|
||||
<p>→ Paraformer-Streaming, feed audio chunks every 600ms → <a href="#streaming">Jump to Streaming ASR</a></p>
|
||||
</div>
|
||||
|
||||
<div class="scenario-card">
|
||||
<h4>😊 "I want to detect user emotions from voice"</h4>
|
||||
<p>→ SenseVoice (outputs emotion tags: happy, sad, angry, neutral) → <a href="#emotion">Jump to Emotion Detection</a></p>
|
||||
</div>
|
||||
|
||||
<div class="scenario-card">
|
||||
<h4>🌍 "I have audio in Japanese/Korean/Arabic/etc."</h4>
|
||||
<p>→ Fun-ASR-Nano (31 languages) or Qwen3-ASR (52 languages) → <a href="#offline">Jump to Offline ASR</a></p>
|
||||
</div>
|
||||
|
||||
<div class="scenario-card">
|
||||
<h4>✂️ "I want to clip a video by spoken content"</h4>
|
||||
<p>→ Use <a href="https://github.com/modelscope/FunClip">FunClip</a> which integrates FunASR for smart video editing</p>
|
||||
</div>
|
||||
|
||||
<!-- Offline ASR -->
|
||||
<h2 id="offline">离线识别</h2>
|
||||
|
||||
<h3>Paraformer (Chinese)</h3>
|
||||
<pre>from funasr import AutoModel
|
||||
|
||||
model = AutoModel(
|
||||
model="paraformer-zh", # Chinese ASR
|
||||
vad_model="fsmn-vad", # handles any audio length
|
||||
vad_kwargs={"max_single_segment_time": 60000},
|
||||
punc_model="ct-punc", # adds punctuation
|
||||
)
|
||||
res = model.generate(input="meeting.wav", batch_size_s=300, hotword='达摩院 语音识别')
|
||||
print(res[0]["text"]) # "欢迎大家来体验达摩院推出的语音识别模型。"
|
||||
print(res[0]["timestamp"]) # [[880,1120],[1120,1360],...] (ms per character)</pre>
|
||||
|
||||
<h3>Fun-ASR-Nano (31 Languages)</h3>
|
||||
<pre>from funasr import AutoModel
|
||||
|
||||
model = AutoModel(
|
||||
model="FunAudioLLM/Fun-ASR-Nano-2512",
|
||||
trust_remote_code=True,
|
||||
remote_code="./model.py",
|
||||
vad_model="fsmn-vad",
|
||||
vad_kwargs={"max_single_segment_time": 30000},
|
||||
device="cuda:0",
|
||||
hub="hf",
|
||||
)
|
||||
res = model.generate(input=["audio.wav"], cache={}, batch_size=1,
|
||||
hotwords=["keyword"], language="中文")
|
||||
print(res[0]["text"]) # recognized text with punctuation
|
||||
print(res[0]["timestamps"]) # [{"token":"开","start_time":0.42,"end_time":0.48}, ...]</pre>
|
||||
|
||||
<div class="note"><strong>Fun-ASR-Nano outputs punctuation natively</strong> — no need for <code>punc_model</code>.</div>
|
||||
|
||||
<h3>SenseVoice (ASR + Emotion + Events)</h3>
|
||||
<pre>from funasr import AutoModel
|
||||
from funasr.utils.postprocess_utils import rich_transcription_postprocess
|
||||
|
||||
model = AutoModel(
|
||||
model="iic/SenseVoiceSmall",
|
||||
vad_model="fsmn-vad",
|
||||
vad_kwargs={"max_single_segment_time": 30000},
|
||||
device="cuda:0",
|
||||
)
|
||||
res = model.generate(input="audio.wav", cache={}, language="auto",
|
||||
use_itn=True, batch_size_s=60, merge_vad=True, merge_length_s=15)
|
||||
|
||||
# Raw output contains emotion/event tags: <|zh|><|HAPPY|><|Speech|>你好
|
||||
text = rich_transcription_postprocess(res[0]["text"])
|
||||
print(text) # "你好" (clean text)</pre>
|
||||
|
||||
<h3>Qwen3-ASR (52 Languages, Highest Accuracy)</h3>
|
||||
<pre># pip install qwen-asr
|
||||
from funasr import AutoModel
|
||||
|
||||
model = AutoModel(model="Qwen/Qwen3-ASR-1.7B", hub="hf", device="cuda:0")
|
||||
res = model.generate(input="audio.wav")
|
||||
print(res[0]["text"]) # recognized text
|
||||
print(res[0].get("language")) # auto-detected language</pre>
|
||||
|
||||
<!-- Streaming -->
|
||||
<h2 id="streaming">Streaming ASR (Real-time)</h2>
|
||||
<p>Process audio chunk-by-chunk for real-time transcription. Each chunk produces partial text immediately.</p>
|
||||
|
||||
<pre>from funasr import AutoModel
|
||||
import soundfile
|
||||
|
||||
model = AutoModel(model="paraformer-zh-streaming")
|
||||
|
||||
speech, sr = soundfile.read("audio.wav")
|
||||
chunk_size = [0, 10, 5] # 600ms display, 300ms lookahead
|
||||
chunk_stride = chunk_size[1] * 960 # 9600 samples per chunk
|
||||
|
||||
cache = {}
|
||||
total_chunks = int((len(speech) - 1) / chunk_stride + 1)
|
||||
for i in range(total_chunks):
|
||||
chunk = speech[i * chunk_stride:(i + 1) * chunk_stride]
|
||||
is_final = (i == total_chunks - 1)
|
||||
|
||||
res = model.generate(input=chunk, cache=cache, is_final=is_final,
|
||||
chunk_size=chunk_size,
|
||||
encoder_chunk_look_back=4,
|
||||
decoder_chunk_look_back=1)
|
||||
if res[0]["text"]:
|
||||
print(res[0]["text"], end="", flush=True) # incremental output</pre>
|
||||
|
||||
<p><strong>Output (printed incrementally):</strong></p>
|
||||
<div class="output-box">欢迎大 | 家来 | 体验达 | 摩院推 | 出的语 | 音识 | 别模型</div>
|
||||
|
||||
<div class="note">
|
||||
<strong>Key points:</strong><br>
|
||||
• <code>cache={}</code> must persist across all chunks (don't recreate it)<br>
|
||||
• <code>is_final=True</code> on last chunk flushes remaining buffered text<br>
|
||||
• <code>chunk_size=[0,10,5]</code>: first number unused, second=display granularity (×60ms), third=lookahead (×60ms)
|
||||
</div>
|
||||
|
||||
<!-- Speaker Diarization -->
|
||||
<h2 id="spk">Speaker Diarization ("Who Said What")</h2>
|
||||
<p>Works with <strong>all three major models</strong>: Paraformer, Fun-ASR-Nano, and SenseVoice. Add <code>spk_model="cam++"</code> to get speaker labels per sentence.</p>
|
||||
|
||||
<h3>Paraformer + Speaker</h3>
|
||||
<pre>from funasr import AutoModel
|
||||
|
||||
model = AutoModel(
|
||||
model="paraformer-zh",
|
||||
vad_model="fsmn-vad",
|
||||
punc_model="ct-punc", # Paraformer needs this for sentence segmentation
|
||||
spk_model="cam++",
|
||||
)
|
||||
res = model.generate(input="meeting.wav", batch_size_s=300)
|
||||
|
||||
for sent in res[0]["sentence_info"]:
|
||||
print(f"[Speaker {sent['spk']}] [{sent['start']}-{sent['end']}ms] {sent['text']}")</pre>
|
||||
|
||||
<p><strong>Output:</strong></p>
|
||||
<div class="output-box">[Speaker 0] [880-5195ms] 欢迎大家来体验达摩院推出的语音识别模型。</div>
|
||||
|
||||
<h3>Fun-ASR-Nano + Speaker (no punc needed)</h3>
|
||||
<pre>model = AutoModel(
|
||||
model="FunAudioLLM/Fun-ASR-Nano-2512",
|
||||
trust_remote_code=True, remote_code="./model.py",
|
||||
vad_model="fsmn-vad", vad_kwargs={"max_single_segment_time": 30000},
|
||||
spk_model="cam++", # no punc_model needed!
|
||||
device="cuda:0", hub="hf",
|
||||
)
|
||||
res = model.generate(input=["meeting.wav"], cache={}, batch_size=1, language="中文")
|
||||
for sent in res[0]["sentence_info"]:
|
||||
print(f"Speaker {sent['spk']}: {sent.get('text', sent.get('sentence', ''))}")</pre>
|
||||
|
||||
<h3>SenseVoice + Speaker (no punc needed)</h3>
|
||||
<pre>model = AutoModel(
|
||||
model="iic/SenseVoiceSmall",
|
||||
vad_model="fsmn-vad", vad_kwargs={"max_single_segment_time": 30000},
|
||||
spk_model="cam++", # no punc_model needed!
|
||||
device="cuda:0",
|
||||
)
|
||||
res = model.generate(input="meeting.wav", cache={}, language="auto",
|
||||
use_itn=True, batch_size_s=60, merge_vad=True, merge_length_s=15)
|
||||
for sent in res[0]["sentence_info"]:
|
||||
print(f"Speaker {sent['spk']}: {rich_transcription_postprocess(sent['text'])}")</pre>
|
||||
|
||||
<!-- Emotion -->
|
||||
<h2 id="emotion">情感检测</h2>
|
||||
|
||||
<h3>Dedicated Emotion Model (emotion2vec)</h3>
|
||||
<pre>from funasr import AutoModel
|
||||
|
||||
model = AutoModel(model="iic/emotion2vec_plus_large", device="cuda:0")
|
||||
res = model.generate(input="audio.wav", granularity="utterance")
|
||||
|
||||
print(res[0]["labels"]) # ['angry', 'happy', 'neutral', 'sad', ...]
|
||||
print(res[0]["scores"]) # [0.01, 0.03, 0.89, 0.05, ...]
|
||||
# → This audio is "neutral" with 89% confidence</pre>
|
||||
|
||||
<h3>SenseVoice (ASR + Emotion in one shot)</h3>
|
||||
<p>SenseVoice embeds emotion tags directly in the transcription output:</p>
|
||||
<pre># Raw output: "<|zh|><|HAPPY|><|Speech|><|withitn|>今天真是太开心了。"
|
||||
# The <|HAPPY|> tag tells you the emotion
|
||||
# Use rich_transcription_postprocess() to get clean text</pre>
|
||||
|
||||
<!-- VAD -->
|
||||
<h2 id="vad">VAD语音检测</h2>
|
||||
|
||||
<h3>Offline (full audio)</h3>
|
||||
<pre>from funasr import AutoModel
|
||||
|
||||
model = AutoModel(model="fsmn-vad")
|
||||
res = model.generate(input="audio.wav")
|
||||
print(res[0]["value"]) # [[610, 5530], [7200, 12400], ...]
|
||||
# Each pair: [start_ms, end_ms] of speech</pre>
|
||||
|
||||
<h3>Streaming (chunk-by-chunk)</h3>
|
||||
<pre>import soundfile
|
||||
from funasr import AutoModel
|
||||
|
||||
model = AutoModel(model="fsmn-vad")
|
||||
speech, sr = soundfile.read("audio.wav")
|
||||
chunk_stride = int(200 * sr / 1000) # 200ms chunks
|
||||
|
||||
cache = {}
|
||||
for i in range(int((len(speech)-1)/chunk_stride+1)):
|
||||
chunk = speech[i*chunk_stride:(i+1)*chunk_stride]
|
||||
is_final = i == int((len(speech)-1)/chunk_stride)
|
||||
res = model.generate(input=chunk, cache=cache, is_final=is_final, chunk_size=200)
|
||||
if res[0]["value"]:
|
||||
print(res[0]["value"])
|
||||
# [[610, -1]] → speech started at 610ms
|
||||
# [[-1, 5530]] → speech ended at 5530ms
|
||||
# [[610, 5530]] → complete segment</pre>
|
||||
|
||||
<!-- Punctuation -->
|
||||
<h2 id="punc">Punctuation Restoration</h2>
|
||||
<pre>from funasr import AutoModel
|
||||
|
||||
model = AutoModel(model="ct-punc")
|
||||
res = model.generate(input="那今天的会就到这里吧 happy new year 明年见")
|
||||
print(res[0]["text"]) # "那今天的会就到这里吧,happy new year,明年见。"</pre>
|
||||
<div class="note"><strong>When do you need this?</strong> Only with Paraformer (which outputs raw text without punctuation). Fun-ASR-Nano, SenseVoice, and Qwen3-ASR output punctuation natively.</div>
|
||||
|
||||
<!-- Export -->
|
||||
<h2 id="export">ONNX导出</h2>
|
||||
<pre># Export model to ONNX format
|
||||
from funasr import AutoModel
|
||||
model = AutoModel(model="paraformer", device="cpu")
|
||||
model.export(quantize=False) # saves to model cache directory
|
||||
|
||||
# Use ONNX model (faster, no PyTorch needed)
|
||||
# pip install funasr-onnx
|
||||
from funasr_onnx import Paraformer
|
||||
model = Paraformer("damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
|
||||
batch_size=1, quantize=True)
|
||||
result = model(["audio.wav"])
|
||||
print(result)</pre>
|
||||
|
||||
<!-- FAQ -->
|
||||
<h2 id="faq">常见问题</h2>
|
||||
|
||||
<h4>Model download is slow</h4>
|
||||
<p>Set <code>hub="hf"</code> for HuggingFace (faster outside China), or download once and use local path:</p>
|
||||
<pre>model = AutoModel(model="/path/to/local/model", disable_update=True)</pre>
|
||||
|
||||
<h4>Out of Memory (OOM)</h4>
|
||||
<p>Three knobs to reduce memory:</p>
|
||||
<ol>
|
||||
<li>Reduce <code>batch_size_s</code> (e.g., from 300 to 60)</li>
|
||||
<li>Reduce <code>max_single_segment_time</code> in vad_kwargs (e.g., 30000 → 15000)</li>
|
||||
<li>Add <code>batch_size_threshold_s=30</code> to force batch=1 for long segments</li>
|
||||
</ol>
|
||||
|
||||
<h4>Every startup shows "Downloading Model..."</h4>
|
||||
<p>Not actually re-downloading — just verifying cache. To skip entirely:</p>
|
||||
<pre>model = AutoModel(model="/local/path/to/model", disable_update=True)</pre>
|
||||
|
||||
<h4>"ModelName is not registered"</h4>
|
||||
<p>Usually means pypi version is outdated. Install from source:</p>
|
||||
<pre>pip install git+https://github.com/modelscope/FunASR.git</pre>
|
||||
|
||||
<h4>Want to suppress all progress bars and logs?</h4>
|
||||
<pre>model = AutoModel(model="...", disable_update=True, disable_pbar=True, log_level="ERROR")</pre>
|
||||
|
||||
<h4>How to pass pre-loaded numpy audio?</h4>
|
||||
<pre>import soundfile as sf
|
||||
audio, sr = sf.read("audio.wav") # numpy array, 16kHz
|
||||
res = model.generate(input=audio) # pass directly — no file needed</pre>
|
||||
|
||||
<h4>GPU not being used?</h4>
|
||||
<pre># Check:
|
||||
import torch
|
||||
print(torch.cuda.is_available()) # must be True
|
||||
|
||||
# Specify GPU explicitly:
|
||||
model = AutoModel(model="...", device="cuda:0")</pre>
|
||||
|
||||
</div></div>
|
||||
|
||||
<footer>
|
||||
<p>FunASR · Tongyi Lab, Alibaba Group</p>
|
||||
<p><a href="index.html">首页</a> · <a href="tutorial.html">使用教程</a> · <a href="training.html">训练</a> · <a href="model-registration.html">开发</a> · <a href="../api.html">API</a> · <a href="https://github.com/modelscope/FunASR">GitHub</a></p>
|
||||
</footer>
|
||||
</body></html>
|
||||
Reference in New Issue
Block a user