mirror of
https://github.com/modelscope/FunASR.git
synced 2026-10-12 02:19:18 +00:00
Merge pull request #3751: explicit PCM16 input and VAD source rates
feat(audio): support explicit PCM16 input without container probing
This commit is contained in:
zhifu gao
@@ -5,6 +5,10 @@ on:
|
||||
paths:
|
||||
- "setup.py"
|
||||
- "funasr/__init__.py"
|
||||
- "funasr/auto/auto_model.py"
|
||||
- "funasr/utils/load_utils.py"
|
||||
- "tests/test_pcm_input_format.py"
|
||||
- "tests/test_load_audio_bytes.py"
|
||||
- "funasr/frontends/**"
|
||||
- "funasr/models/emotion2vec/**"
|
||||
- "funasr/models/campplus/**"
|
||||
@@ -21,6 +25,10 @@ on:
|
||||
paths:
|
||||
- "setup.py"
|
||||
- "funasr/__init__.py"
|
||||
- "funasr/auto/auto_model.py"
|
||||
- "funasr/utils/load_utils.py"
|
||||
- "tests/test_pcm_input_format.py"
|
||||
- "tests/test_load_audio_bytes.py"
|
||||
- "funasr/frontends/**"
|
||||
- "funasr/models/emotion2vec/**"
|
||||
- "funasr/models/campplus/**"
|
||||
@@ -58,6 +66,11 @@ jobs:
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: pip
|
||||
- name: Install audio decoder
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install --no-install-recommends -y ffmpeg
|
||||
ffmpeg -version
|
||||
- name: Install CPU runtime and requested NumPy version
|
||||
run: |
|
||||
python -m pip install torch==2.10.0 torchaudio==2.10.0 --index-url https://download.pytorch.org/whl/cpu
|
||||
@@ -76,6 +89,8 @@ jobs:
|
||||
tests/test_eend_feature_compatibility.py \
|
||||
tests/test_pypi_metadata.py \
|
||||
tests/test_sensevoice_word_timestamps.py \
|
||||
tests/test_pcm_input_format.py \
|
||||
tests/test_load_audio_bytes.py \
|
||||
--junitxml=numpy-results.xml
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
|
||||
+9
-1
@@ -104,7 +104,15 @@ for result in results:
|
||||
|
||||
`batch_size=1` processes a list serially through direct inference; raise it only for a model that supports batching and after measuring memory use. `batch_size_s` has a different meaning and only controls the wrapper's VAD path. VAD segmentation is not incremental microphone streaming, and it does not guarantee unlimited recording length or bounded total memory.
|
||||
|
||||
Use a one-dimensional mono float32 waveform with a known sample rate. For models using the common audio loader, `fs` in `generate()` describes the array's source sample rate (fallback 16000); the model frontend supplies the target rate. Do not pass a stereo matrix or a list of numbers and assume it will be treated as one waveform: a Python list is normally a list of inputs. File decoding depends on the installed audio backends. Top-level bytes are handled by `load_bytes`: recognized containers are decoded/resampled to 16 kHz, otherwise bytes are interpreted as int16 PCM without sample-rate metadata. Prefer a decoded array with an explicit rate for ambiguous inputs. See [audio loading](../funasr/utils/load_utils.py) and [byte-input tests](../tests/test_load_audio_bytes.py).
|
||||
Use a one-dimensional mono float32 waveform with a known sample rate. For models using the common audio loader, `fs` in `generate()` describes the array's source sample rate (fallback 16000); the model frontend supplies the target rate. Do not pass a stereo matrix or a list of numbers and assume it will be treated as one waveform: a Python list is normally a list of inputs. File decoding depends on the installed audio backends. With the default `input_format="auto"`, top-level bytes are handled by `load_bytes`: recognized containers are decoded/resampled to 16 kHz, otherwise bytes are interpreted as int16 PCM without sample-rate metadata. Prefer a decoded array with an explicit rate for ambiguous inputs. See [audio loading](../funasr/utils/load_utils.py) and [byte-input tests](../tests/test_load_audio_bytes.py).
|
||||
|
||||
For known **mono signed 16-bit little-endian raw PCM**, explicitly bypass format detection:
|
||||
|
||||
```python
|
||||
result = model.generate(input=pcm_bytes, input_format="pcm_s16le", fs=16000)
|
||||
```
|
||||
|
||||
This option also applies to byte strings in a batch and to the VAD preprocessing path. It normalizes samples to float32 without invoking a container decoder; raw bytes can otherwise accidentally resemble a file header. It does not resample at byte-conversion time: use the actual source sample rate in `fs`. Do not select this format for WAV/MP3 file bytes, float PCM, other bit depths, big-endian samples, or interleaved stereo. A partial two-byte sample raises `ValueError`; assemble complete samples in the receiving layer. Streaming callers must still preserve their per-session `cache`, chunk settings, and final-chunk handling. This opt-in path avoids ambiguous probing, but does not diagnose every source of inference delays.
|
||||
|
||||
`prepare_data_iterator` also accepts manifests. A `.scp` line can be `utterance_id /path/to/audio.wav`; `.jsonl` records use `{"source": "/path/to/audio.wav", "key": "utterance_id"}`. `.json` is not parsed as an arbitrary JSON array by this iterator. Do not expose unrestricted paths or URLs directly to untrusted callers.
|
||||
|
||||
|
||||
@@ -103,7 +103,15 @@ for result in results:
|
||||
|
||||
`batch_size=1` 在直接推理路径中逐条处理列表；只有确认模型支持批量并测量过内存占用后再增大它。`batch_size_s` 的含义不同，只控制包装层的 VAD 路径。VAD 切分不是麦克风增量流式识别，也不保证无限录音长度或固定的总内存占用。
|
||||
|
||||
波形应为采样率已知的一维单声道 float32 数组。使用通用音频加载器的模型，在 `generate()` 中通过 `fs` 指定数组的原始采样率，回退值为 16000；目标采样率由模型前端决定。不要把立体声矩阵或数字列表默认当作单条波形，Python 列表通常被解释为多条输入。文件解码能力取决于已安装的音频后端。顶层 bytes 由 `load_bytes` 处理：识别出的容器格式会被解码并重采样为 16 kHz，否则按没有采样率元数据的 int16 PCM 解释。输入格式有歧义时，优先使用解码后的数组并明确采样率。参见[音频加载代码](../funasr/utils/load_utils.py) 和 [bytes 输入测试](../tests/test_load_audio_bytes.py)。
|
||||
波形应为采样率已知的一维单声道 float32 数组。使用通用音频加载器的模型，在 `generate()` 中通过 `fs` 指定数组的原始采样率，回退值为 16000；目标采样率由模型前端决定。不要把立体声矩阵或数字列表默认当作单条波形，Python 列表通常被解释为多条输入。文件解码能力取决于已安装的音频后端。默认 `input_format="auto"` 时，顶层 bytes 由 `load_bytes` 处理：识别出的容器格式会被解码并重采样为 16 kHz，否则按没有采样率元数据的 int16 PCM 解释。输入格式有歧义时，优先使用解码后的数组并明确采样率。参见[音频加载代码](../funasr/utils/load_utils.py) 和 [bytes 输入测试](../tests/test_load_audio_bytes.py)。
|
||||
|
||||
对于格式已知的**单声道、有符号 16 位、小端原始 PCM**，可以显式跳过格式探测：
|
||||
|
||||
```python
|
||||
result = model.generate(input=pcm_bytes, input_format="pcm_s16le", fs=16000)
|
||||
```
|
||||
|
||||
该选项也适用于批量输入中的 bytes 和 VAD 预处理路径。它直接把采样值归一化为 float32，不调用容器解码器，避免原始字节偶然匹配文件头。字节转换本身不做重采样，请通过 `fs` 传入真实原始采样率。不要用于 WAV/MP3 文件字节、浮点 PCM、其他位深、大端数据或交错排列的立体声。若字节数不足以组成完整的双字节采样点，会抛出 `ValueError`，收包层应先拼接完整采样。流式调用仍须保留每会话的 `cache`、分块配置和末帧处理。这个显式入口避免了探测歧义，但不代表已经定位所有推理卡顿的原因。
|
||||
|
||||
`prepare_data_iterator` 也接受清单文件。`.scp` 的一行可以是 `utterance_id /path/to/audio.wav`；`.jsonl` 使用 `{"source": "/path/to/audio.wav", "key": "utterance_id"}`。该迭代器不会把 `.json` 文件当作任意 JSON 数组解析。不要向不可信调用方直接开放任意路径或 URL 输入。
|
||||
|
||||
|
||||
+31
-10
@@ -448,8 +448,10 @@ except:
|
||||
pass
|
||||
|
||||
|
||||
def prepare_data_iterator(data_in, input_len=None, data_type=None, key=None):
|
||||
""" """
|
||||
def prepare_data_iterator(
|
||||
data_in, input_len=None, data_type=None, key=None, input_format="auto"
|
||||
):
|
||||
"""Prepare inputs, optionally interpreting audio bytes as explicit PCM16."""
|
||||
data_list = []
|
||||
key_list = []
|
||||
filelist = [".scp", ".txt", ".json", ".jsonl", ".text"]
|
||||
@@ -490,7 +492,7 @@ def prepare_data_iterator(data_in, input_len=None, data_type=None, key=None):
|
||||
data_list_tmp = []
|
||||
for data_in_i, data_type_i in zip(data_in, data_type):
|
||||
key_list, data_list_i = prepare_data_iterator(
|
||||
data_in=data_in_i, data_type=data_type_i
|
||||
data_in=data_in_i, data_type=data_type_i, input_format=input_format
|
||||
)
|
||||
data_list_tmp.append(data_list_i)
|
||||
data_list = []
|
||||
@@ -499,6 +501,12 @@ def prepare_data_iterator(data_in, input_len=None, data_type=None, key=None):
|
||||
else:
|
||||
# [audio sample point, fbank, text]
|
||||
data_list = data_in
|
||||
if input_format != "auto":
|
||||
data_list = [
|
||||
load_bytes(item, input_format=input_format)
|
||||
if isinstance(item, bytes) else item
|
||||
for item in data_in
|
||||
]
|
||||
key_list = []
|
||||
for data_i in data_in:
|
||||
if isinstance(data_i, str) and os.path.exists(data_i):
|
||||
@@ -510,7 +518,7 @@ def prepare_data_iterator(data_in, input_len=None, data_type=None, key=None):
|
||||
|
||||
else: # raw text; audio sample point, fbank; bytes
|
||||
if isinstance(data_in, bytes): # audio bytes
|
||||
data_in = load_bytes(data_in)
|
||||
data_in = load_bytes(data_in, input_format=input_format)
|
||||
if key is None:
|
||||
key = "rand_key_" + "".join(random.choice(chars) for _ in range(13))
|
||||
data_list = [data_in]
|
||||
@@ -833,6 +841,9 @@ class AutoModel:
|
||||
- device: Placement is selected at construction. Runtime device overrides
|
||||
are ignored; create another AutoModel to use a different device.
|
||||
- cache (dict): State cache for streaming mode. Pass {} for first call.
|
||||
- input_format (str): "auto" (default) or "pcm_s16le" to bypass
|
||||
container detection for mono little-endian PCM16 byte inputs.
|
||||
Use fs for the source sample rate.
|
||||
- hotword (str/list): Keywords to boost recognition accuracy.
|
||||
- postprocess_hotwords (str/list/dict): Text-level hotword correction after
|
||||
decoding. Unlike model-level ``hotword``, this runs on the final text.
|
||||
@@ -915,7 +926,8 @@ class AutoModel:
|
||||
# batch_size = 1
|
||||
|
||||
key_list, data_list = prepare_data_iterator(
|
||||
input, input_len=input_len, data_type=kwargs.get("data_type", None), key=key
|
||||
input, input_len=input_len, data_type=kwargs.get("data_type", None), key=key,
|
||||
input_format=kwargs.get("input_format", "auto"),
|
||||
)
|
||||
|
||||
speed_stats = {}
|
||||
@@ -1002,8 +1014,13 @@ class AutoModel:
|
||||
kwargs = self.kwargs
|
||||
# step.1: compute the vad model
|
||||
beg_vad = time.time()
|
||||
vad_cfg = {
|
||||
"input_format": kwargs.get("input_format", "auto"),
|
||||
"fs": kwargs.get("fs", 16000),
|
||||
**cfg,
|
||||
}
|
||||
res = self.inference(
|
||||
input, input_len=input_len, model=self.vad_model, kwargs=self.vad_kwargs, **cfg
|
||||
input, input_len=input_len, model=self.vad_model, kwargs=self.vad_kwargs, **vad_cfg
|
||||
)
|
||||
end_vad = time.time()
|
||||
|
||||
@@ -1022,9 +1039,12 @@ class AutoModel:
|
||||
kwargs["batch_size"] = batch_size
|
||||
|
||||
key_list, data_list = prepare_data_iterator(
|
||||
input, input_len=input_len, data_type=kwargs.get("data_type", None)
|
||||
input, input_len=input_len, data_type=kwargs.get("data_type", None),
|
||||
input_format=kwargs.get("input_format", "auto"),
|
||||
)
|
||||
results_ret_list = []
|
||||
# ASR updates kwargs with the resampled rate; retain the original for each input.
|
||||
input_fs = kwargs.get("fs", 16000)
|
||||
time_speech_total_all_samples = 1e-6
|
||||
|
||||
beg_total = time.time()
|
||||
@@ -1038,7 +1058,7 @@ class AutoModel:
|
||||
vadsegments = res[i]["value"]
|
||||
input_i = data_list[i]
|
||||
fs = kwargs["frontend"].fs if hasattr(kwargs["frontend"], "fs") else 16000
|
||||
speech = load_audio_text_image_video(input_i, fs=fs, audio_fs=kwargs.get("fs", 16000))
|
||||
speech = load_audio_text_image_video(input_i, fs=fs, audio_fs=input_fs)
|
||||
speech_lengths = len(speech)
|
||||
n = len(vadsegments)
|
||||
data_with_index = [(vadsegments[i], i) for i in range(n)]
|
||||
@@ -1084,7 +1104,7 @@ class AutoModel:
|
||||
speech, speech_lengths, sorted_data[beg_idx:end_idx]
|
||||
)
|
||||
results = self.inference(
|
||||
speech_j, input_len=None, model=model, kwargs=kwargs, **cfg
|
||||
speech_j, input_len=None, model=model, kwargs=kwargs, **{**cfg, "fs": fs}
|
||||
)
|
||||
if self.spk_model is not None:
|
||||
# compose vad segments: [[start_time_sec, end_time_sec, speech], [...]]
|
||||
@@ -1435,7 +1455,8 @@ class AutoModel:
|
||||
type = kwargs.get("type", "onnx")
|
||||
|
||||
key_list, data_list = prepare_data_iterator(
|
||||
input, input_len=None, data_type=kwargs.get("data_type", None), key=None
|
||||
input, input_len=None, data_type=kwargs.get("data_type", None), key=None,
|
||||
input_format=kwargs.get("input_format", "auto"),
|
||||
)
|
||||
|
||||
with torch.no_grad():
|
||||
|
||||
@@ -312,15 +312,25 @@ def _is_audio_container(data: bytes) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def load_bytes(input):
|
||||
def load_bytes(input, input_format="auto"):
|
||||
"""Convert raw PCM or container-formatted audio bytes to a waveform.
|
||||
|
||||
Args:
|
||||
input (bytes): Raw int16 PCM or encoded audio-file bytes.
|
||||
input_format (str): "auto" preserves format detection. "pcm_s16le"
|
||||
reads mono signed little-endian PCM16 without probing or decoding.
|
||||
|
||||
Returns:
|
||||
numpy.ndarray: Mono float32 samples at 16 kHz.
|
||||
numpy.ndarray: Normalized float32 samples. Containers are resampled to
|
||||
16 kHz; raw PCM keeps its source rate and is not resampled here.
|
||||
"""
|
||||
if input_format not in ("auto", "pcm_s16le"):
|
||||
raise ValueError("input_format must be 'auto' or 'pcm_s16le'")
|
||||
if input_format == "pcm_s16le":
|
||||
if len(input) % 2:
|
||||
raise ValueError("PCM16 input must contain complete 2-byte samples")
|
||||
return np.frombuffer(input, dtype="<i2").astype(np.float32) / 32768.0
|
||||
|
||||
if _is_audio_container(input):
|
||||
try:
|
||||
waveform = load_audio_text_image_video(BytesIO(input), fs=16000)
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
"""Explicit PCM routing tests; no model weights or inference are required."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from funasr.auto.auto_model import AutoModel, prepare_data_iterator
|
||||
from funasr.utils import load_utils
|
||||
|
||||
|
||||
def _ambiguous_pcm():
|
||||
data = bytearray(6400)
|
||||
for offset in (0, 417):
|
||||
data[offset : offset + 4] = b"\xff\xfb\x90\x00"
|
||||
return bytes(data)
|
||||
|
||||
|
||||
def _expected(data):
|
||||
return np.frombuffer(data, dtype="<i2").astype(np.float32) / 32768.0
|
||||
|
||||
|
||||
def _forbid_decode(*args, **kwargs):
|
||||
raise AssertionError("container decoding must not run for explicit PCM")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"data",
|
||||
[
|
||||
b"",
|
||||
b"\x00\x80\xff\xff\x00\x00\x01\x00\xff\x7f",
|
||||
b"RIFF\x00\x00\x00\x00WAVE\x00\x00",
|
||||
b"ID3\x00",
|
||||
_ambiguous_pcm(),
|
||||
],
|
||||
ids=["empty", "limits", "wav-like", "id3-like", "mpeg-like"],
|
||||
)
|
||||
def test_explicit_pcm_never_probes_or_decodes(data, monkeypatch):
|
||||
monkeypatch.setattr(load_utils, "_is_audio_container", _forbid_decode)
|
||||
monkeypatch.setattr(load_utils, "load_audio_text_image_video", _forbid_decode)
|
||||
actual = load_utils.load_bytes(data, input_format="pcm_s16le")
|
||||
assert actual.dtype == np.float32
|
||||
np.testing.assert_array_equal(actual, _expected(data))
|
||||
|
||||
|
||||
def test_ambiguous_pcm_really_matches_automatic_container_detection(monkeypatch):
|
||||
data = _ambiguous_pcm()
|
||||
assert load_utils._is_audio_container(data)
|
||||
decoded = np.array([0.25, -0.25], dtype=np.float32)
|
||||
monkeypatch.setattr(
|
||||
load_utils, "load_audio_text_image_video", lambda *a, **kw: decoded
|
||||
)
|
||||
np.testing.assert_array_equal(load_utils.load_bytes(data), decoded)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["pcm", "wav", None])
|
||||
def test_invalid_byte_format_is_rejected(value):
|
||||
with pytest.raises(ValueError, match="input_format"):
|
||||
load_utils.load_bytes(b"\x00\x00", input_format=value)
|
||||
|
||||
|
||||
def test_partial_pcm_sample_is_rejected_without_probing(monkeypatch):
|
||||
monkeypatch.setattr(load_utils, "_is_audio_container", _forbid_decode)
|
||||
with pytest.raises(ValueError, match="complete 2-byte samples"):
|
||||
load_utils.load_bytes(b"\x00", input_format="pcm_s16le")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch", [False, True])
|
||||
def test_iterator_decodes_explicit_pcm_and_preserves_keys(batch, monkeypatch):
|
||||
monkeypatch.setattr(load_utils, "load_audio_text_image_video", _forbid_decode)
|
||||
data = _ambiguous_pcm()
|
||||
value = [data, data] if batch else data
|
||||
keys, samples = prepare_data_iterator(
|
||||
value, input_format="pcm_s16le", key="session"
|
||||
)
|
||||
assert keys == ["session"] * (2 if batch else 1)
|
||||
for sample in samples:
|
||||
np.testing.assert_array_equal(sample, _expected(data))
|
||||
|
||||
|
||||
def test_multimodal_iterator_forwards_byte_format(monkeypatch):
|
||||
monkeypatch.setattr(load_utils, "load_audio_text_image_video", _forbid_decode)
|
||||
data = _ambiguous_pcm()
|
||||
keys, values = prepare_data_iterator(
|
||||
([data], ["context"]), data_type=("sound", "text"),
|
||||
input_format="pcm_s16le"
|
||||
)
|
||||
assert len(keys) == len(values) == 1
|
||||
np.testing.assert_array_equal(values[0][0], _expected(data))
|
||||
assert values[0][1] == "context"
|
||||
|
||||
|
||||
def test_default_batch_behavior_is_unchanged():
|
||||
data = [_ambiguous_pcm()]
|
||||
_, values = prepare_data_iterator(data)
|
||||
assert values is data
|
||||
|
||||
|
||||
class _RecordingModel(torch.nn.Module):
|
||||
def __init__(self, vad=False):
|
||||
super().__init__()
|
||||
self.anchor = torch.nn.Parameter(torch.zeros(1))
|
||||
self.vad = vad
|
||||
self.segments = []
|
||||
self.calls = []
|
||||
|
||||
def inference(self, data_in, key, **kwargs):
|
||||
self.calls.append((data_in, kwargs))
|
||||
if self.vad:
|
||||
results = [{"key": k, "value": self.segments} for k in key]
|
||||
else:
|
||||
results = [{"key": k, "text": "ok"} for k in key]
|
||||
return results, {"batch_data_time": 0.2}
|
||||
|
||||
|
||||
def _wrapper(vad=False, default_format=None):
|
||||
wrapper = AutoModel.__new__(AutoModel)
|
||||
wrapper.model = _RecordingModel()
|
||||
wrapper.vad_model = _RecordingModel(vad=True) if vad else None
|
||||
wrapper.punc_model = None
|
||||
wrapper.spk_model = None
|
||||
wrapper.kwargs = {
|
||||
"device": "cpu", "disable_pbar": True, "model": "pcm-routing-probe",
|
||||
"frontend": SimpleNamespace(fs=16000), "ncpu": torch.get_num_threads(),
|
||||
}
|
||||
if default_format is not None:
|
||||
wrapper.kwargs["input_format"] = default_format
|
||||
wrapper.vad_kwargs = {"device": "cpu", "disable_pbar": True}
|
||||
wrapper._store_base_configs()
|
||||
return wrapper
|
||||
|
||||
|
||||
@pytest.mark.parametrize("vad", [False, True])
|
||||
@pytest.mark.parametrize("at_construction", [False, True])
|
||||
def test_generate_forwards_pcm_format_without_model_download(
|
||||
vad, at_construction, monkeypatch
|
||||
):
|
||||
monkeypatch.setattr(load_utils, "load_audio_text_image_video", _forbid_decode)
|
||||
wrapper = _wrapper(vad, "pcm_s16le" if at_construction else None)
|
||||
options = {} if at_construction else {"input_format": "pcm_s16le"}
|
||||
data = _ambiguous_pcm()
|
||||
result = wrapper.generate(input=data, fs=16000, **options)
|
||||
recorder = wrapper.vad_model if vad else wrapper.model
|
||||
np.testing.assert_array_equal(recorder.calls[0][0][0], _expected(data))
|
||||
assert result[0]["text"] == ("" if vad else "ok")
|
||||
|
||||
|
||||
def test_generate_pcm_flag_does_not_leak_to_next_call(monkeypatch):
|
||||
monkeypatch.setattr(load_utils, "load_audio_text_image_video", _forbid_decode)
|
||||
wrapper = _wrapper()
|
||||
data = _ambiguous_pcm()
|
||||
wrapper.generate(input=data, input_format="pcm_s16le")
|
||||
with pytest.raises(RuntimeError, match="container-formatted"):
|
||||
wrapper.generate(input=data)
|
||||
|
||||
|
||||
def test_generate_vad_segments_reach_asr_with_same_samples(monkeypatch):
|
||||
monkeypatch.setattr(load_utils, "load_audio_text_image_video", _forbid_decode)
|
||||
wrapper = _wrapper(vad=True)
|
||||
wrapper.vad_model.segments = [[0, 200]]
|
||||
data = _ambiguous_pcm()
|
||||
result = wrapper.generate(input=data, input_format="pcm_s16le", fs=16000)
|
||||
assert result[0]["text"] == "ok"
|
||||
np.testing.assert_array_equal(wrapper.model.calls[0][0][0], _expected(data))
|
||||
|
||||
|
||||
def test_streaming_options_are_preserved(monkeypatch):
|
||||
monkeypatch.setattr(load_utils, "load_audio_text_image_video", _forbid_decode)
|
||||
wrapper = _wrapper()
|
||||
wrapper.generate(
|
||||
input=_ambiguous_pcm(), input_format="pcm_s16le",
|
||||
fs=8000, cache={"marker": 7}, is_final=True, chunk_size=200,
|
||||
)
|
||||
samples, options = wrapper.model.calls[0]
|
||||
assert len(samples[0]) == 3200
|
||||
assert options["fs"] == 8000
|
||||
assert options["cache"] == {"marker": 7}
|
||||
assert options["is_final"] is True
|
||||
assert options["chunk_size"] == 200
|
||||
|
||||
|
||||
@pytest.mark.parametrize("at_construction", [False, True])
|
||||
def test_export_preparation_forwards_pcm_format(at_construction, monkeypatch):
|
||||
import importlib
|
||||
|
||||
auto_module = importlib.import_module("funasr.auto.auto_model")
|
||||
monkeypatch.setattr(load_utils, "load_audio_text_image_video", _forbid_decode)
|
||||
wrapper = _wrapper(default_format="pcm_s16le" if at_construction else None)
|
||||
data = _ambiguous_pcm()
|
||||
|
||||
def capture_export(model, data_in, **kwargs):
|
||||
np.testing.assert_array_equal(data_in[0], _expected(data))
|
||||
return "prepared"
|
||||
|
||||
monkeypatch.setattr(auto_module.export_utils, "export", capture_export)
|
||||
options = {} if at_construction else {"input_format": "pcm_s16le"}
|
||||
assert wrapper.export(input=data, **options) == "prepared"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source_rate", [8000, 48000])
|
||||
@pytest.mark.parametrize("batch", [False, True])
|
||||
@pytest.mark.parametrize("at_construction", [False, True])
|
||||
@pytest.mark.parametrize("input_kind", ["array", "pcm"])
|
||||
def test_vad_source_rates_are_not_reapplied_to_resampled_segments(
|
||||
source_rate, batch, at_construction, input_kind, monkeypatch
|
||||
):
|
||||
count = source_rate // 5
|
||||
pcm = np.round(
|
||||
np.sin(2 * np.pi * 440 * np.arange(count) / source_rate) * 12000
|
||||
).astype("<i2").tobytes()
|
||||
waveform = _expected(pcm)
|
||||
expected = load_utils.load_audio_text_image_video(
|
||||
waveform, fs=16000, audio_fs=source_rate
|
||||
)
|
||||
monkeypatch.setattr(load_utils, "_is_audio_container", _forbid_decode)
|
||||
wrapper = _wrapper(vad=True)
|
||||
wrapper.vad_model.segments = [[0, 200]]
|
||||
if at_construction:
|
||||
wrapper.kwargs["fs"] = source_rate
|
||||
wrapper._store_base_configs()
|
||||
options = {} if at_construction else {"fs": source_rate}
|
||||
if input_kind == "pcm":
|
||||
options["input_format"] = "pcm_s16le"
|
||||
value = pcm if input_kind == "pcm" else waveform
|
||||
results = wrapper.generate(input=[value, value] if batch else value, **options)
|
||||
assert len(results) == (2 if batch else 1)
|
||||
for _, vad_options in wrapper.vad_model.calls:
|
||||
assert vad_options["fs"] == source_rate
|
||||
for samples, asr_options in wrapper.model.calls:
|
||||
assert asr_options["fs"] == 16000
|
||||
np.testing.assert_allclose(samples[0], expected, atol=1e-6, rtol=0)
|
||||
loaded = load_utils.load_audio_text_image_video(
|
||||
samples[0], fs=16000, audio_fs=asr_options["fs"]
|
||||
)
|
||||
assert len(loaded) == 3200
|
||||
np.testing.assert_allclose(loaded, expected, atol=1e-6, rtol=0)
|
||||
@@ -46,7 +46,7 @@ class PythonApiDocsContract(unittest.TestCase):
|
||||
"funasr/models/paraformer_streaming/model.py")))
|
||||
for path in DOCS:
|
||||
blocks = python_blocks(path)
|
||||
self.assertEqual(len(blocks), 5, path.name)
|
||||
self.assertEqual(len(blocks), 6, path.name)
|
||||
for index, block in enumerate(blocks):
|
||||
with self.subTest(doc=path.name, block=index):
|
||||
compile(block, str(path), "exec")
|
||||
|
||||
Reference in New Issue
Block a user