From 9d611293f44b2d2ba335b336d0ebf8c9a69b6a3a Mon Sep 17 00:00:00 2001 From: LauraGPT <18321252+LauraGPT@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:17:23 +0000 Subject: [PATCH] feat(audio): support explicit PCM16 input without container probing Signed-off-by: LauraGPT <18321252+LauraGPT@users.noreply.github.com> --- .../workflows/test-numpy-compatibility.yml | 10 + docs/python_api.md | 10 +- docs/python_api_zh.md | 10 +- funasr/auto/auto_model.py | 41 ++- funasr/utils/load_utils.py | 14 +- tests/test_pcm_input_format.py | 237 ++++++++++++++++++ tests/test_python_api_docs_contract.py | 2 +- 7 files changed, 309 insertions(+), 15 deletions(-) create mode 100644 tests/test_pcm_input_format.py diff --git a/.github/workflows/test-numpy-compatibility.yml b/.github/workflows/test-numpy-compatibility.yml index d23e0c8ff..d4606f8d3 100644 --- a/.github/workflows/test-numpy-compatibility.yml +++ b/.github/workflows/test-numpy-compatibility.yml @@ -5,6 +5,10 @@ on: paths: - "setup.py" - "funasr/__init__.py" + - "funasr/auto/auto_model.py" + - "funasr/utils/load_utils.py" + - "tests/test_pcm_input_format.py" + - "tests/test_load_audio_bytes.py" - "funasr/frontends/**" - "funasr/models/emotion2vec/**" - "funasr/models/campplus/**" @@ -21,6 +25,10 @@ on: paths: - "setup.py" - "funasr/__init__.py" + - "funasr/auto/auto_model.py" + - "funasr/utils/load_utils.py" + - "tests/test_pcm_input_format.py" + - "tests/test_load_audio_bytes.py" - "funasr/frontends/**" - "funasr/models/emotion2vec/**" - "funasr/models/campplus/**" @@ -76,6 +84,8 @@ jobs: tests/test_eend_feature_compatibility.py \ tests/test_pypi_metadata.py \ tests/test_sensevoice_word_timestamps.py \ + tests/test_pcm_input_format.py \ + tests/test_load_audio_bytes.py \ --junitxml=numpy-results.xml - uses: actions/upload-artifact@v4 if: always() diff --git a/docs/python_api.md b/docs/python_api.md index db6caef08..2cadd9633 100644 --- a/docs/python_api.md +++ b/docs/python_api.md @@ -104,7 +104,15 @@ for result in results: `batch_size=1` processes a list serially through direct inference; raise it only for a model that supports batching and after measuring memory use. `batch_size_s` has a different meaning and only controls the wrapper's VAD path. VAD segmentation is not incremental microphone streaming, and it does not guarantee unlimited recording length or bounded total memory. -Use a one-dimensional mono float32 waveform with a known sample rate. For models using the common audio loader, `fs` in `generate()` describes the array's source sample rate (fallback 16000); the model frontend supplies the target rate. Do not pass a stereo matrix or a list of numbers and assume it will be treated as one waveform: a Python list is normally a list of inputs. File decoding depends on the installed audio backends. Top-level bytes are handled by `load_bytes`: recognized containers are decoded/resampled to 16 kHz, otherwise bytes are interpreted as int16 PCM without sample-rate metadata. Prefer a decoded array with an explicit rate for ambiguous inputs. See [audio loading](../funasr/utils/load_utils.py) and [byte-input tests](../tests/test_load_audio_bytes.py). +Use a one-dimensional mono float32 waveform with a known sample rate. For models using the common audio loader, `fs` in `generate()` describes the array's source sample rate (fallback 16000); the model frontend supplies the target rate. Do not pass a stereo matrix or a list of numbers and assume it will be treated as one waveform: a Python list is normally a list of inputs. File decoding depends on the installed audio backends. With the default `input_format="auto"`, top-level bytes are handled by `load_bytes`: recognized containers are decoded/resampled to 16 kHz, otherwise bytes are interpreted as int16 PCM without sample-rate metadata. Prefer a decoded array with an explicit rate for ambiguous inputs. See [audio loading](../funasr/utils/load_utils.py) and [byte-input tests](../tests/test_load_audio_bytes.py). + +For known **mono signed 16-bit little-endian raw PCM**, explicitly bypass format detection: + +```python +result = model.generate(input=pcm_bytes, input_format="pcm_s16le", fs=16000) +``` + +This option also applies to byte strings in a batch and to the VAD preprocessing path. It normalizes samples to float32 without invoking a container decoder; raw bytes can otherwise accidentally resemble a file header. It does not resample at byte-conversion time: use the actual source sample rate in `fs`. Do not select this format for WAV/MP3 file bytes, float PCM, other bit depths, big-endian samples, or interleaved stereo. A partial two-byte sample raises `ValueError`; assemble complete samples in the receiving layer. Streaming callers must still preserve their per-session `cache`, chunk settings, and final-chunk handling. This opt-in path avoids ambiguous probing, but does not diagnose every source of inference delays. `prepare_data_iterator` also accepts manifests. A `.scp` line can be `utterance_id /path/to/audio.wav`; `.jsonl` records use `{"source": "/path/to/audio.wav", "key": "utterance_id"}`. `.json` is not parsed as an arbitrary JSON array by this iterator. Do not expose unrestricted paths or URLs directly to untrusted callers. diff --git a/docs/python_api_zh.md b/docs/python_api_zh.md index 4b9864b4e..71eae9811 100644 --- a/docs/python_api_zh.md +++ b/docs/python_api_zh.md @@ -103,7 +103,15 @@ for result in results: `batch_size=1` 在直接推理路径中逐条处理列表;只有确认模型支持批量并测量过内存占用后再增大它。`batch_size_s` 的含义不同,只控制包装层的 VAD 路径。VAD 切分不是麦克风增量流式识别,也不保证无限录音长度或固定的总内存占用。 -波形应为采样率已知的一维单声道 float32 数组。使用通用音频加载器的模型,在 `generate()` 中通过 `fs` 指定数组的原始采样率,回退值为 16000;目标采样率由模型前端决定。不要把立体声矩阵或数字列表默认当作单条波形,Python 列表通常被解释为多条输入。文件解码能力取决于已安装的音频后端。顶层 bytes 由 `load_bytes` 处理:识别出的容器格式会被解码并重采样为 16 kHz,否则按没有采样率元数据的 int16 PCM 解释。输入格式有歧义时,优先使用解码后的数组并明确采样率。参见[音频加载代码](../funasr/utils/load_utils.py) 和 [bytes 输入测试](../tests/test_load_audio_bytes.py)。 +波形应为采样率已知的一维单声道 float32 数组。使用通用音频加载器的模型,在 `generate()` 中通过 `fs` 指定数组的原始采样率,回退值为 16000;目标采样率由模型前端决定。不要把立体声矩阵或数字列表默认当作单条波形,Python 列表通常被解释为多条输入。文件解码能力取决于已安装的音频后端。默认 `input_format="auto"` 时,顶层 bytes 由 `load_bytes` 处理:识别出的容器格式会被解码并重采样为 16 kHz,否则按没有采样率元数据的 int16 PCM 解释。输入格式有歧义时,优先使用解码后的数组并明确采样率。参见[音频加载代码](../funasr/utils/load_utils.py) 和 [bytes 输入测试](../tests/test_load_audio_bytes.py)。 + +对于格式已知的**单声道、有符号 16 位、小端原始 PCM**,可以显式跳过格式探测: + +```python +result = model.generate(input=pcm_bytes, input_format="pcm_s16le", fs=16000) +``` + +该选项也适用于批量输入中的 bytes 和 VAD 预处理路径。它直接把采样值归一化为 float32,不调用容器解码器,避免原始字节偶然匹配文件头。字节转换本身不做重采样,请通过 `fs` 传入真实原始采样率。不要用于 WAV/MP3 文件字节、浮点 PCM、其他位深、大端数据或交错排列的立体声。若字节数不足以组成完整的双字节采样点,会抛出 `ValueError`,收包层应先拼接完整采样。流式调用仍须保留每会话的 `cache`、分块配置和末帧处理。这个显式入口避免了探测歧义,但不代表已经定位所有推理卡顿的原因。 `prepare_data_iterator` 也接受清单文件。`.scp` 的一行可以是 `utterance_id /path/to/audio.wav`;`.jsonl` 使用 `{"source": "/path/to/audio.wav", "key": "utterance_id"}`。该迭代器不会把 `.json` 文件当作任意 JSON 数组解析。不要向不可信调用方直接开放任意路径或 URL 输入。 diff --git a/funasr/auto/auto_model.py b/funasr/auto/auto_model.py index 49adfea21..e814cba84 100644 --- a/funasr/auto/auto_model.py +++ b/funasr/auto/auto_model.py @@ -448,8 +448,10 @@ except: pass -def prepare_data_iterator(data_in, input_len=None, data_type=None, key=None): - """ """ +def prepare_data_iterator( + data_in, input_len=None, data_type=None, key=None, input_format="auto" +): + """Prepare inputs, optionally interpreting audio bytes as explicit PCM16.""" data_list = [] key_list = [] filelist = [".scp", ".txt", ".json", ".jsonl", ".text"] @@ -490,7 +492,7 @@ def prepare_data_iterator(data_in, input_len=None, data_type=None, key=None): data_list_tmp = [] for data_in_i, data_type_i in zip(data_in, data_type): key_list, data_list_i = prepare_data_iterator( - data_in=data_in_i, data_type=data_type_i + data_in=data_in_i, data_type=data_type_i, input_format=input_format ) data_list_tmp.append(data_list_i) data_list = [] @@ -499,6 +501,12 @@ def prepare_data_iterator(data_in, input_len=None, data_type=None, key=None): else: # [audio sample point, fbank, text] data_list = data_in + if input_format != "auto": + data_list = [ + load_bytes(item, input_format=input_format) + if isinstance(item, bytes) else item + for item in data_in + ] key_list = [] for data_i in data_in: if isinstance(data_i, str) and os.path.exists(data_i): @@ -510,7 +518,7 @@ def prepare_data_iterator(data_in, input_len=None, data_type=None, key=None): else: # raw text; audio sample point, fbank; bytes if isinstance(data_in, bytes): # audio bytes - data_in = load_bytes(data_in) + data_in = load_bytes(data_in, input_format=input_format) if key is None: key = "rand_key_" + "".join(random.choice(chars) for _ in range(13)) data_list = [data_in] @@ -833,6 +841,9 @@ class AutoModel: - device: Placement is selected at construction. Runtime device overrides are ignored; create another AutoModel to use a different device. - cache (dict): State cache for streaming mode. Pass {} for first call. + - input_format (str): "auto" (default) or "pcm_s16le" to bypass + container detection for mono little-endian PCM16 byte inputs. + Use fs for the source sample rate. - hotword (str/list): Keywords to boost recognition accuracy. - postprocess_hotwords (str/list/dict): Text-level hotword correction after decoding. Unlike model-level ``hotword``, this runs on the final text. @@ -915,7 +926,8 @@ class AutoModel: # batch_size = 1 key_list, data_list = prepare_data_iterator( - input, input_len=input_len, data_type=kwargs.get("data_type", None), key=key + input, input_len=input_len, data_type=kwargs.get("data_type", None), key=key, + input_format=kwargs.get("input_format", "auto"), ) speed_stats = {} @@ -1002,8 +1014,13 @@ class AutoModel: kwargs = self.kwargs # step.1: compute the vad model beg_vad = time.time() + vad_cfg = { + "input_format": kwargs.get("input_format", "auto"), + "fs": kwargs.get("fs", 16000), + **cfg, + } res = self.inference( - input, input_len=input_len, model=self.vad_model, kwargs=self.vad_kwargs, **cfg + input, input_len=input_len, model=self.vad_model, kwargs=self.vad_kwargs, **vad_cfg ) end_vad = time.time() @@ -1022,9 +1039,12 @@ class AutoModel: kwargs["batch_size"] = batch_size key_list, data_list = prepare_data_iterator( - input, input_len=input_len, data_type=kwargs.get("data_type", None) + input, input_len=input_len, data_type=kwargs.get("data_type", None), + input_format=kwargs.get("input_format", "auto"), ) results_ret_list = [] + # ASR updates kwargs with the resampled rate; retain the original for each input. + input_fs = kwargs.get("fs", 16000) time_speech_total_all_samples = 1e-6 beg_total = time.time() @@ -1038,7 +1058,7 @@ class AutoModel: vadsegments = res[i]["value"] input_i = data_list[i] fs = kwargs["frontend"].fs if hasattr(kwargs["frontend"], "fs") else 16000 - speech = load_audio_text_image_video(input_i, fs=fs, audio_fs=kwargs.get("fs", 16000)) + speech = load_audio_text_image_video(input_i, fs=fs, audio_fs=input_fs) speech_lengths = len(speech) n = len(vadsegments) data_with_index = [(vadsegments[i], i) for i in range(n)] @@ -1084,7 +1104,7 @@ class AutoModel: speech, speech_lengths, sorted_data[beg_idx:end_idx] ) results = self.inference( - speech_j, input_len=None, model=model, kwargs=kwargs, **cfg + speech_j, input_len=None, model=model, kwargs=kwargs, **{**cfg, "fs": fs} ) if self.spk_model is not None: # compose vad segments: [[start_time_sec, end_time_sec, speech], [...]] @@ -1435,7 +1455,8 @@ class AutoModel: type = kwargs.get("type", "onnx") key_list, data_list = prepare_data_iterator( - input, input_len=None, data_type=kwargs.get("data_type", None), key=None + input, input_len=None, data_type=kwargs.get("data_type", None), key=None, + input_format=kwargs.get("input_format", "auto"), ) with torch.no_grad(): diff --git a/funasr/utils/load_utils.py b/funasr/utils/load_utils.py index 35c341c87..b689d65ec 100644 --- a/funasr/utils/load_utils.py +++ b/funasr/utils/load_utils.py @@ -312,15 +312,25 @@ def _is_audio_container(data: bytes) -> bool: return False -def load_bytes(input): +def load_bytes(input, input_format="auto"): """Convert raw PCM or container-formatted audio bytes to a waveform. Args: input (bytes): Raw int16 PCM or encoded audio-file bytes. + input_format (str): "auto" preserves format detection. "pcm_s16le" + reads mono signed little-endian PCM16 without probing or decoding. Returns: - numpy.ndarray: Mono float32 samples at 16 kHz. + numpy.ndarray: Normalized float32 samples. Containers are resampled to + 16 kHz; raw PCM keeps its source rate and is not resampled here. """ + if input_format not in ("auto", "pcm_s16le"): + raise ValueError("input_format must be 'auto' or 'pcm_s16le'") + if input_format == "pcm_s16le": + if len(input) % 2: + raise ValueError("PCM16 input must contain complete 2-byte samples") + return np.frombuffer(input, dtype="