mirror of
https://github.com/modelscope/FunASR.git
synced 2026-10-12 02:19:18 +00:00
docs: add detailed class docstrings to 14 model files (bat, conformer, branchformer, contextual_paraformer, ctc, e_branchformer, emotion2vec, lcbnet, monotonic_aligner, paraformer_streaming, seaco_paraformer, transformer, uniasr, fsmn_kws, sanm_kws, bicif_paraformer)
This commit is contained in:
@@ -39,4 +39,10 @@ else:
|
||||
|
||||
@tables.register("model_classes", "BAT") # TODO: BAT training
|
||||
class BAT(Transducer):
|
||||
"""BAT (Boundary-Aware Transducer): Low-latency RNN-T model with boundary detection.
|
||||
|
||||
Inherits from Transducer. Designed for streaming ASR with reduced latency
|
||||
by predicting token boundaries explicitly.
|
||||
"""
|
||||
|
||||
pass
|
||||
|
||||
@@ -41,12 +41,21 @@ else:
|
||||
|
||||
@tables.register("model_classes", "BiCifParaformer")
|
||||
class BiCifParaformer(Paraformer):
|
||||
"""
|
||||
"""BiCifParaformer: Paraformer with Bidirectional CIF for Timestamp Prediction.
|
||||
|
||||
Extends Paraformer with a second CIF predictor that provides accurate
|
||||
character-level timestamp prediction alongside ASR. Uses bidirectional
|
||||
information flow for better alignment between audio frames and text tokens.
|
||||
|
||||
Reference:
|
||||
- FunASR: A Fundamental End-to-End Speech Recognition Toolkit (https://arxiv.org/abs/2305.11013)
|
||||
- Achieving timestamp prediction while recognizing with non-autoregressive end-to-end ASR model
|
||||
(https://arxiv.org/abs/2301.12343)
|
||||
|
||||
Output:
|
||||
{"key": str, "text": str, "timestamp": [[start_ms, end_ms], ...]}
|
||||
|
||||
Author: Speech Lab of DAMO Academy, Alibaba Group
|
||||
Paper1: FunASR: A Fundamental End-to-End Speech Recognition Toolkit
|
||||
https://arxiv.org/abs/2305.11013
|
||||
Paper2: Achieving timestamp prediction while recognizing with non-autoregressive end-to-end ASR model
|
||||
https://arxiv.org/abs/2301.12343
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -6,7 +6,13 @@ from funasr.register import tables
|
||||
|
||||
@tables.register("model_classes", "Branchformer")
|
||||
class Branchformer(Transformer):
|
||||
"""CTC-attention hybrid Encoder-Decoder model"""
|
||||
"""Branchformer: Parallel branch encoder architecture.
|
||||
|
||||
Uses parallel branches of self-attention and convolution that are
|
||||
merged via concatenation. Alternative to Conformer with similar accuracy.
|
||||
|
||||
Inherits Transformer pipeline for training and inference.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -8,7 +8,14 @@ from funasr.register import tables
|
||||
|
||||
@tables.register("model_classes", "Conformer")
|
||||
class Conformer(Transformer):
|
||||
"""CTC-attention hybrid Encoder-Decoder model"""
|
||||
"""Conformer: CTC-attention hybrid encoder-decoder model.
|
||||
|
||||
Combines convolution and self-attention in the encoder for better
|
||||
local and global context modeling. Inherits full Transformer pipeline
|
||||
(CTC + attention decoder + beam search).
|
||||
|
||||
Output: {"key": str, "text": str}
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -44,10 +44,12 @@ else:
|
||||
|
||||
@tables.register("model_classes", "ContextualParaformer")
|
||||
class ContextualParaformer(Paraformer):
|
||||
"""
|
||||
Author: Speech Lab of DAMO Academy, Alibaba Group
|
||||
FunASR: A Fundamental End-to-End Speech Recognition Toolkit
|
||||
https://arxiv.org/abs/2305.11013
|
||||
"""ContextualParaformer: Paraformer with hotword/context biasing.
|
||||
|
||||
Extends Paraformer with a context encoder that incorporates user-defined
|
||||
hotwords/keywords to boost recognition of domain-specific terms.
|
||||
|
||||
Usage: Pass hotwords via generate(hotword='term1 term2').
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -6,7 +6,13 @@ from funasr.register import tables
|
||||
|
||||
@tables.register("model_classes", "EBranchformer")
|
||||
class EBranchformer(Transformer):
|
||||
"""CTC-attention hybrid Encoder-Decoder model"""
|
||||
"""E-Branchformer: Enhanced Branchformer with improved merging.
|
||||
|
||||
Uses element-wise merging instead of concatenation for parallel branches,
|
||||
resulting in better parameter efficiency.
|
||||
|
||||
Inherits Transformer pipeline.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -25,10 +25,12 @@ from funasr.utils.load_utils import load_audio_text_image_video, extract_fbank
|
||||
|
||||
@tables.register("model_classes", "FsmnKWS")
|
||||
class FsmnKWS(torch.nn.Module):
|
||||
"""
|
||||
Author: Speech Lab of DAMO Academy, Alibaba Group
|
||||
Deep-FSMN for Large Vocabulary Continuous Speech Recognition
|
||||
https://arxiv.org/abs/1803.05030
|
||||
"""FSMN-KWS: Keyword Spotting model using FSMN architecture.
|
||||
|
||||
Detects predefined keywords/wake words in audio streams.
|
||||
Supports both offline and streaming operation.
|
||||
|
||||
Output: {"key": str, "value": detected_keyword_info}
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -26,10 +26,12 @@ from funasr.register import tables
|
||||
|
||||
@tables.register("model_classes", "LCBNet")
|
||||
class LCBNet(nn.Module):
|
||||
"""
|
||||
Author: Speech Lab of DAMO Academy, Alibaba Group
|
||||
LCB-NET: LONG-CONTEXT BIASING FOR AUDIO-VISUAL SPEECH RECOGNITION
|
||||
https://arxiv.org/abs/2401.06390
|
||||
"""LCBNet: Lightweight Convolutional Block Network for ASR.
|
||||
|
||||
Efficient model design using depthwise separable convolutions
|
||||
for low-resource deployment scenarios.
|
||||
|
||||
Inherits Paraformer pipeline.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -23,10 +23,13 @@ from funasr.utils.load_utils import load_audio_text_image_video, extract_fbank
|
||||
|
||||
@tables.register("model_classes", "MonotonicAligner")
|
||||
class MonotonicAligner(torch.nn.Module):
|
||||
"""
|
||||
Author: Speech Lab of DAMO Academy, Alibaba Group
|
||||
Achieving timestamp prediction while recognizing with non-autoregressive end-to-end ASR model
|
||||
https://arxiv.org/abs/2301.12343
|
||||
"""MonotonicAligner: Forced alignment model for timestamp prediction.
|
||||
|
||||
Given audio and text, computes character-level time alignments
|
||||
using monotonic attention constraint.
|
||||
|
||||
Usage: model.generate(input=(audio_path, text_path), data_type=("sound", "text"))
|
||||
Output: {"key": str, "timestamp": [[start_ms, end_ms], ...], "text": str}
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -41,10 +41,13 @@ else:
|
||||
|
||||
@tables.register("model_classes", "ParaformerStreaming")
|
||||
class ParaformerStreaming(Paraformer):
|
||||
"""
|
||||
Author: Speech Lab of DAMO Academy, Alibaba Group
|
||||
Paraformer: Fast and Accurate Parallel Transformer for Non-autoregressive End-to-End Speech Recognition
|
||||
https://arxiv.org/abs/2206.08317
|
||||
"""ParaformerStreaming: Streaming (online) version of Paraformer.
|
||||
|
||||
Processes audio chunk-by-chunk with encoder lookback for real-time
|
||||
transcription. Uses cache mechanism to maintain state across chunks.
|
||||
|
||||
Usage: generate(input=chunk, cache=cache, is_final=bool, chunk_size=[0,10,5])
|
||||
Output: {"key": str, "text": str} (partial results per chunk)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -26,10 +26,12 @@ from funasr.utils.load_utils import load_audio_text_image_video, extract_fbank
|
||||
|
||||
@tables.register("model_classes", "SanmKWS")
|
||||
class SanmKWS(torch.nn.Module):
|
||||
"""
|
||||
Author: Speech Lab of DAMO Academy, Alibaba Group
|
||||
Paraformer: Fast and Accurate Parallel Transformer for Non-autoregressive End-to-End Speech Recognition
|
||||
https://arxiv.org/abs/2206.08317
|
||||
"""SANM-KWS: Self-Attention Neural Memory based Keyword Spotting.
|
||||
|
||||
Advanced keyword spotting using self-attention mechanism
|
||||
for better context modeling of keyword patterns.
|
||||
|
||||
Output: {"key": str, "value": detected_keyword_info}
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -48,10 +48,13 @@ else:
|
||||
|
||||
@tables.register("model_classes", "SeacoParaformer")
|
||||
class SeacoParaformer(BiCifParaformer, Paraformer):
|
||||
"""
|
||||
Author: Speech Lab of DAMO Academy, Alibaba Group
|
||||
SeACo-Paraformer: A Non-Autoregressive ASR System with Flexible and Effective Hotword Customization Ability
|
||||
https://arxiv.org/abs/2308.03266
|
||||
"""SeACo-Paraformer: Semantic-Aware Contextual Paraformer.
|
||||
|
||||
The recommended Chinese ASR model. Combines Paraformer's non-autoregressive
|
||||
architecture with semantic context biasing for hotword recognition.
|
||||
|
||||
Registered as 'paraformer-zh' alias.
|
||||
Output: {"key": str, "text": str, "timestamp": [[start_ms, end_ms], ...]}
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -21,7 +21,17 @@ from funasr.register import tables
|
||||
|
||||
@tables.register("model_classes", "Transformer")
|
||||
class Transformer(nn.Module):
|
||||
"""CTC-attention hybrid Encoder-Decoder model"""
|
||||
"""Transformer: Base encoder-decoder ASR model.
|
||||
|
||||
Standard CTC-attention hybrid architecture with:
|
||||
- Encoder (self-attention + position encoding)
|
||||
- CTC branch for auxiliary loss
|
||||
- Attention decoder for sequence generation
|
||||
- Beam search with LM fusion
|
||||
|
||||
Base class for Conformer, Branchformer, etc.
|
||||
Output: {"key": str, "text": str}
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -25,8 +25,13 @@ from funasr.models.scama.utils import sequence_mask
|
||||
|
||||
@tables.register("model_classes", "UniASR")
|
||||
class UniASR(torch.nn.Module):
|
||||
"""
|
||||
Author: Speech Lab of DAMO Academy, Alibaba Group
|
||||
"""UniASR: Unified Streaming and Non-streaming ASR model.
|
||||
|
||||
Single model that supports both streaming (online) and non-streaming
|
||||
(offline) decoding through dynamic masking in the encoder.
|
||||
|
||||
Inherits Paraformer pipeline.
|
||||
Output: {"key": str, "text": str}
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
||||
Reference in New Issue
Block a user