diff --git a/.github/workflows/product-site.yml b/.github/workflows/product-site.yml index 93bda3250..b7edf8e7d 100644 --- a/.github/workflows/product-site.yml +++ b/.github/workflows/product-site.yml @@ -91,6 +91,7 @@ jobs: tests/test_python_api_docs_contract.py tests/test_streaming_vad_docs.py tests/test_kws_docs.py + tests/test_speaker_emotion_docs.py tests/test_service_docs_contract.py tests/test_generated_api_docs.py tests/test_api_signatures.py diff --git a/docs/README.md b/docs/README.md index e4412aa08..c672eb65b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -30,6 +30,7 @@ | Compare CPU, GPU, realtime and API runtimes | [Deployment matrix](deployment_matrix.md) / [中文](deployment_matrix_zh.md) | | Handle streaming speech boundaries and finalization | [Streaming VAD](streaming_vad.md) / [中文](streaming_vad_zh.md) | | Detect keywords and manage utterance-end KWS results | [Keyword spotting](keyword_spotting.md) / [中文](keyword_spotting_zh.md) | +| Distinguish speaker vectors, anonymous labels and emotion tags | [Speakers and emotion tags](speaker_emotion.md) / [中文](speaker_emotion_zh.md) | | Transcribe and diarize with third-party MOSS | [MOSS](moss_transcribe_diarize.md) / [中文](moss_transcribe_diarize_zh.md) | | Accelerate with the FunASR vLLM split engine | [vLLM](vllm_guide.md) / [中文](vllm_guide_zh.md) | | Evaluate native vLLM serving | [Validation record](vllm_native_funasr_validation.md) | diff --git a/docs/index.rst b/docs/index.rst index 3e00f16b6..d89e2a15b 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -44,6 +44,8 @@ entry to these guides. Source Markdown remains in this repository. streaming_vad_zh keyword_spotting keyword_spotting_zh + speaker_emotion + speaker_emotion_zh deployment_matrix deployment_matrix_zh ./moss_transcribe_diarize.md diff --git a/docs/speaker_emotion.md b/docs/speaker_emotion.md new file mode 100644 index 000000000..2c199328f --- /dev/null +++ b/docs/speaker_emotion.md @@ -0,0 +1,205 @@ +[简体中文](speaker_emotion_zh.md) | English + +# Speakers and Emotion Tags + +Choose the result you need before choosing the model. A speaker embedding, +an anonymous cluster label, an enrolled person's identity and an emotion tag +are different outputs. None can be substituted for another. + +## Choose a Task + +| Task | Path | +| --- | --- | +| Extract one speaker vector from selected speech | CAMPPlus or ERes2NetV2; example below | +| Attribute transcript segments to anonymous speakers | [ASR/VAD/speaker pipeline](python_api.md#vad-timestamps-and-speakers) | +| Preserve transcription, emotion and event tags | SenseVoiceSmall; example below | +| Joint transcription and anonymous diarization | Third-party [OpenMOSS MOSS guide](moss_transcribe_diarize.md) | + +CAMPPlus and ERes2NetV2 return `spk_embedding`, a Tensor, not a person's name, +match decision or general ASR `text` field. A single input normally has one row; +its embedding dimension comes from the checkpoint configuration, not a universal +192-element API guarantee. Use `batch_size=1` here: some batching paths return +multiple embedding rows in one dictionary, not one keyed result per input. + +Embeddings alone do not establish identity. Enrollment, matching, threshold +calibration, consent and evaluation on the intended population are separate +application work. A clustering label such as `spk=0` or MOSS's `S01` is anonymous +within a recording, not a stable person ID across recordings. A requested +`spk_embedding_center` is a cluster mean, not completed enrollment. + +## Prepare a Local Checkpoint + +Complete the [installation checks](installation/installation.md) and use a +complete local snapshot with its configuration, frontend, tokenizer where +applicable, and weights. Review each model's license and record the resolved +revision or file hashes, SDK version, imported module path and source commit. +This guide follows the implementations linked below; it does not claim that +every older wheel, export or model variant has the same interface. + +- **CAMPPlus (`embedding`):** `iic/speech_campplus_sv_zh-cn_16k-common`. + Its ModelScope alias is `cam++`. +- **ERes2NetV2 (`embedding`):** `iic/speech_eres2netv2_sv_zh-cn_16k-common`. +- **SenseVoice (`sensevoice`):** `iic/SenseVoiceSmall`. + +The `embedding` and `sensevoice` choices belong to this example program, not new +FunASR model aliases or CLI subcommands. Do not point `embedding` at an ASR +checkpoint or generalize ERes2NetV2 behavior to every ERes2Net variant. +Check [Model Zoo](../model_zoo/readme.md) for other model paths. + +Use nonempty mono 16 kHz WAV audio. The program reads normalized `float32` +samples and rejects stereo, empty or differently sampled audio instead of +silently transforming it. For embeddings, select a single person's speech; +mixtures, silence and extremely short clips are not reliable identity evidence. +The input checks below are format checks, not a speech-quality detector. + +## Save the SDK Result Without Losing Tags + +This standalone program takes `task`, `model_dir`, `audio` and a new JSON output +path. For example, run it with `embedding /models/campplus speaker.wav vector.json` +or `sensevoice /models/sensevoice utterance.wav tags.json` after the script name. +Both modes process a complete clip on CPU, without VAD, punctuation, a companion +speaker model or streaming cache. There are no model downloads in the program. + +```python +import argparse +import json +import os +from pathlib import Path +import soundfile as sf +from funasr import AutoModel +from funasr.utils.postprocess_utils import rich_transcription_postprocess + + +def embedding_record(results): + if not results: + raise ValueError("No result from the speaker model") + if len(results) != 1: + raise ValueError("Expected one speaker result for one input") + vector = results[0]["spk_embedding"] + if vector.ndim != 2 or vector.shape[0] != 1 or vector.shape[1] == 0: + raise ValueError("Expected a nonempty single-row speaker embedding") + return {"spk_embedding": vector.detach().cpu().tolist()} + + +def tagged_records(results): + if not results: + raise ValueError("No result from SenseVoice") + records = [] + for item in results: + raw = item["text"] + if not isinstance(raw, str): + raise ValueError("Expected SenseVoice text with its original tags") + records.append({ + "key": item.get("key"), "raw_tagged_text": raw, + "display_text": rich_transcription_postprocess(raw), + }) + return records + + +def write_result(path, record): + payload = json.dumps(record, ensure_ascii=False, allow_nan=False, indent=2) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as stream: + stream.write(payload + "\n") + + +parser = argparse.ArgumentParser() +parser.add_argument("task", choices=["embedding", "sensevoice"]) +parser.add_argument("model_dir") +parser.add_argument("audio") +parser.add_argument("output") +args = parser.parse_args() +model_dir = Path(args.model_dir).expanduser().resolve(strict=True) +if not model_dir.is_dir(): + raise ValueError("Expected a complete local model directory") +speech, sample_rate = sf.read(args.audio, dtype="float32") +if sample_rate != 16000 or speech.ndim != 1 or len(speech) == 0: + raise ValueError("Expected nonempty mono 16 kHz audio") +model = AutoModel( + model=str(model_dir), device="cpu", ncpu=1, disable_update=True, + trust_remote_code=False, vad_model=None, punc_model=None, spk_model=None, +) +if args.task == "embedding": + results = model.generate(input=speech, fs=sample_rate, batch_size=1) + result = embedding_record(results) +else: + results = model.generate( + input=speech, fs=sample_rate, batch_size=1, + language="auto", use_itn=True, output_timestamp=False, + ) + result = tagged_records(results) +write_result(args.output, { + "task": args.task, "model_dir": str(model_dir), + "sample_rate": sample_rate, "result": result, +}) +``` + +The JSON envelope, `raw_tagged_text` and `display_text` are application-owned +fields created by this example, not new SDK response fields. The vector is +explicitly detached, moved to CPU and converted to a nested list for JSON. +Nonfinite numeric values are rejected before creating a file. Existing output +files are never overwritten; select a new path for each run. On POSIX the file +is created with owner-only mode; use appropriate directory permissions and +platform ACLs too. Voice-derived vectors and transcripts can be sensitive: +retain only what the application needs and do not post private samples publicly. + +## Read SenseVoice Tags Correctly + +The raw tagged string is normally in `result["text"]`, not a guaranteed +`raw_text`, `emotion` or `emotion_score` field. Tags are case-sensitive, for +example `<|zh|>`, `<|HAPPY|>`, `<|Speech|>` and `<|withitn|>`. Their presence and +meaning depend on the checkpoint. A presentation mapping table is not a promise +that every model emits every mapped tag. + +`rich_transcription_postprocess` is lossy display processing: it removes tags, +chooses display emotion by occurrence counts and merges repeated display +markers. It also performs text replacements. It is neither a structured tag +parser nor a probability calculator; preserve the original string first. +Do not infer confidence, psychological state or diagnosis from a display symbol +or model label. Silence and short/noisy clips are not reliable emotional evidence. +`emotion2vec` is a separate model/interface, not an alias for SenseVoice tags. + +The example chooses `language="auto"`, `use_itn=True` and +`output_timestamp=False` explicitly; these are not all mandatory parameters. +SenseVoice language hints use `auto/zh/en/yue/ja/ko`. `use_itn` controls inverse +text normalization, not emotion recognition, and explicit `text_norm` takes +precedence. Repeat request options on each call. This is whole-clip inference, +not the KWS EOS protocol or a live emotional-state monitor. + +## VAD, Diarization and Service Boundaries + +Standalone embedding extraction and short-clip SenseVoice inference do not +require VAD. Generic AutoModel speaker clustering instead lives in its VAD +pipeline: setting only `spk_model` or `return_spk_res=True` does not add +diarization to direct inference. VAD boundaries are not guaranteed speaker +changes. Long-audio segmentation also changes the context used for tags. +Follow the separate [SDK pipeline](python_api.md#vad-timestamps-and-speakers) +for compatible ASR/VAD/punctuation/speaker models. `sentence_info[].spk` is +anonymous and its SDK `start/end` fields use milliseconds; NumPy cluster-center +arrays need explicit serialization just like tensors. + +Third-party OpenMOSS MOSS-Transcribe-Diarize supplies its own joint transcription +and diarization path without an external `vad_model` or `spk_model`. Reuse the +[MOSS guide](moss_transcribe_diarize.md) for backend, memory and response details; +do not attach a second clustering pipeline or advertise known-person identity. + +The current built-in [HTTP transcription service](../examples/openai_api/README.md) +configures SenseVoice with VAD and its fallback strips rich tags from top-level +and segment text. It does not add emotion score fields. Its `spk=true` path is +a service-specific pipeline, and HTTP segment `start/end` use seconds. Moving +this SDK example to `/v1/audio/transcriptions` is not a promise to preserve its +tags or parameters. Check the [deployment matrix](deployment_matrix.md) rather +than assuming vLLM, llama.cpp, ONNX or WebSocket exports share Python outputs. + +## Source and Validation + +Contracts: [CAMPPlus](../funasr/models/campplus/model.py), +[ERes2NetV2](../funasr/models/eres2net/model.py), +[SenseVoice](../funasr/models/sense_voice/model.py), +[postprocessing](../funasr/utils/postprocess_utils.py), +[AutoModel](../funasr/auto/auto_model.py), and +[HTTP adaptation](../funasr/bin/_server_app.py). +The [guide tests](../tests/test_speaker_emotion_docs.py) execute the published +program with recording SDK doubles and the real presentation helper. They check +field preservation, serialization and invalid input handling, not acoustic +quality, identity matching, demographic performance or emotion accuracy. diff --git a/docs/speaker_emotion_zh.md b/docs/speaker_emotion_zh.md new file mode 100644 index 000000000..bcba32d02 --- /dev/null +++ b/docs/speaker_emotion_zh.md @@ -0,0 +1,179 @@ +简体中文 | [English](speaker_emotion.md) + +# 说话人与情感标签 + +先确定需要的结果,再选择模型。声纹向量、匿名聚类编号、已注册人员身份与情感标签 +是四类不同输出,不能互相替代。 + +## 按任务选择 + +| 任务 | 路径 | +| --- | --- | +| 从选定语音提取一个说话人向量 | CAMPPlus 或 ERes2NetV2;见下方示例 | +| 为转写片段分配匿名说话人 | [ASR/VAD/说话人流水线](python_api_zh.md#vad时间戳与说话人) | +| 保留转写、情感和音频事件标签 | SenseVoiceSmall;见下方示例 | +| 联合转写与匿名说话人分离 | 第三方 [OpenMOSS MOSS 指南](moss_transcribe_diarize_zh.md) | + +CAMPPlus 与 ERes2NetV2 返回 `spk_embedding` Tensor,不返回人的姓名、匹配结论 +或通用 ASR `text` 字段。单条输入通常对应一行,向量维度由检查点配置决定, +不是接口保证永远只有 192 维。本例使用 `batch_size=1`;部分批处理路径会在一个 +字典内返回多行向量,而非每个输入对应一个带 `key` 的结果。 + +向量本身不等于身份确认。注册、匹配、阈值校准、授权及面向目标人群的评测需要 +应用单独设计。`spk=0` 或 MOSS 的 `S01` 是录音内的匿名标签,不是跨录音稳定的人员 ID。 +`spk_embedding_center` 是聚类均值,不表示已经完成身份注册。 + +## 准备本地检查点 + +完成[安装检查](installation/installation_zh.md),准备包含配置、前端、适用时的 +tokenizer 及权重的完整本地快照。阅读各模型许可证,记录解析后的 revision 或文件哈希、 +SDK 版本、实际导入模块路径与源码 commit。本指南对应文末链接的实现,不保证所有旧版 +wheel、导出后端或模型变体接口一致。 + +- **CAMPPlus(`embedding`):** `iic/speech_campplus_sv_zh-cn_16k-common`。 + ModelScope 别名为 `cam++`。 +- **ERes2NetV2(`embedding`):** `iic/speech_eres2netv2_sv_zh-cn_16k-common`。 +- **SenseVoice(`sensevoice`):** `iic/SenseVoiceSmall`。 + +`embedding` 和 `sensevoice` 选项只属于下方示例程序,不是新增的 FunASR 模型别名或 CLI 子命令。 +不要把 ASR 检查点传给 `embedding`,也不要将 ERes2NetV2 的行为推广到所有 ERes2Net 变体。 +其他模型请查看 [Model Zoo](../model_zoo/readme_zh.md)。 + +音频使用非空、单声道、16 kHz WAV。程序读取归一化 `float32` 波形,拒绝立体声、空音频 +或其他采样率,不隐式转换。提取声纹时使用选定的单人有效语音;混合说话人、静音或极短 +片段不能提供可靠身份依据。下方校验仅检查格式,不是语音质量检测器。 + +## 保存结果并保留原始标签 + +独立程序接收 `task`、`model_dir`、`audio` 和新的 JSON 输出路径。例如在脚本名后传入 +`embedding /models/campplus speaker.wav vector.json`,或 +`sensevoice /models/sensevoice utterance.wav tags.json`。 +两个模式都在 CPU 上处理完整片段,不添加 VAD、标点、配套说话人模型或流式缓存, +程序内部不下载模型。 + +```python +import argparse +import json +import os +from pathlib import Path +import soundfile as sf +from funasr import AutoModel +from funasr.utils.postprocess_utils import rich_transcription_postprocess + + +def embedding_record(results): + if not results: + raise ValueError("No result from the speaker model") + if len(results) != 1: + raise ValueError("Expected one speaker result for one input") + vector = results[0]["spk_embedding"] + if vector.ndim != 2 or vector.shape[0] != 1 or vector.shape[1] == 0: + raise ValueError("Expected a nonempty single-row speaker embedding") + return {"spk_embedding": vector.detach().cpu().tolist()} + + +def tagged_records(results): + if not results: + raise ValueError("No result from SenseVoice") + records = [] + for item in results: + raw = item["text"] + if not isinstance(raw, str): + raise ValueError("Expected SenseVoice text with its original tags") + records.append({ + "key": item.get("key"), "raw_tagged_text": raw, + "display_text": rich_transcription_postprocess(raw), + }) + return records + + +def write_result(path, record): + payload = json.dumps(record, ensure_ascii=False, allow_nan=False, indent=2) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as stream: + stream.write(payload + "\n") + + +parser = argparse.ArgumentParser() +parser.add_argument("task", choices=["embedding", "sensevoice"]) +parser.add_argument("model_dir") +parser.add_argument("audio") +parser.add_argument("output") +args = parser.parse_args() +model_dir = Path(args.model_dir).expanduser().resolve(strict=True) +if not model_dir.is_dir(): + raise ValueError("Expected a complete local model directory") +speech, sample_rate = sf.read(args.audio, dtype="float32") +if sample_rate != 16000 or speech.ndim != 1 or len(speech) == 0: + raise ValueError("Expected nonempty mono 16 kHz audio") +model = AutoModel( + model=str(model_dir), device="cpu", ncpu=1, disable_update=True, + trust_remote_code=False, vad_model=None, punc_model=None, spk_model=None, +) +if args.task == "embedding": + results = model.generate(input=speech, fs=sample_rate, batch_size=1) + result = embedding_record(results) +else: + results = model.generate( + input=speech, fs=sample_rate, batch_size=1, + language="auto", use_itn=True, output_timestamp=False, + ) + result = tagged_records(results) +write_result(args.output, { + "task": args.task, "model_dir": str(model_dir), + "sample_rate": sample_rate, "result": result, +}) +``` + +外层 JSON、`raw_tagged_text` 和 `display_text` 是示例应用创建的字段,不是 SDK 新增返回字段。 +向量先显式 detach、移到 CPU,再转换为嵌套列表。非有限数值会在创建文件前被拒绝。 +程序不会覆盖已有输出,每次运行使用新路径。POSIX 上以仅所有者可访问的模式创建文件; +仍需配置目录权限及对应平台 ACL。声纹向量和转写可能涉及敏感信息,应按需保留, +不要公开私人音频或结果。 + +## 正确读取 SenseVoice 标签 + +原始带标签字符串通常在 `result["text"]`,不能假设存在 `raw_text`、`emotion` 或 +`emotion_score`。标签区分大小写,例如 `<|zh|>`、`<|HAPPY|>`、`<|Speech|>`、 +`<|withitn|>`;是否出现及其含义取决于检查点。展示映射表不保证所有模型都会输出其中全部标签。 + +`rich_transcription_postprocess` 是有损展示函数:删除标签、按出现次数选择展示情感、 +合并重复展示标记,还会做文本替换。它不是结构化标签解析器或概率计算器,必须先保留原文。 +不要从展示符号或模型标签推导置信概率、真实心理状态或诊断结论。静音、短片段和噪声音频 +不能提供可靠情感依据。`emotion2vec` 是另一类模型及接口,不是 SenseVoice 标签的别名。 + +本例显式选择 `language="auto"`、`use_itn=True`、`output_timestamp=False`,并非这些参数 +都必填。SenseVoice 语言提示使用 `auto/zh/en/yue/ja/ko`。 +`use_itn` 控制逆文本规范化,不是情感识别开关,显式 `text_norm` 的优先级更高。 +每次调用重复传入需要固定的选项。本例是完整片段推理,不是 KWS 的 EOS 协议或实时情绪监控。 + +## VAD、说话人分离与服务边界 + +独立声纹提取和短片段 SenseVoice 推理不要求 VAD。通用 AutoModel 说话人聚类则位于 +VAD 流水线中,只设置 `spk_model` 或 `return_spk_res=True` 不会给直接推理增加说话人分离。 +VAD 边界不保证对应说话人切换;长音频分段也会改变情感标签所依据的上下文。 +组合兼容的 ASR/VAD/标点/说话人模型时,遵循独立的 +[SDK 流水线](python_api_zh.md#vad时间戳与说话人)。 +`sentence_info[].spk` 是匿名编号,SDK `start/end` 单位为毫秒; +NumPy 聚类中心数组也需要像 Tensor 一样显式序列化。 + +第三方 OpenMOSS MOSS-Transcribe-Diarize 有自己的联合转写与分离路径,不需要外部 +`vad_model` 或 `spk_model`。后端、显存和返回格式复用现有 +[MOSS 指南](moss_transcribe_diarize_zh.md),不要叠加第二套聚类或宣称已识别已知人员身份。 + +当前内置 [HTTP 转写服务](../examples/openai_api/README_zh.md) 的 SenseVoice 配置附带 VAD, +fallback 会剥离顶层及分段文本中的富标签,不会补建情感分数字段。 +其 `spk=true` 属于服务自己的流水线,HTTP 分段 `start/end` 单位为秒。 +把此 SDK 示例换成 `/v1/audio/transcriptions` 不代表仍能读取同样的标签或使用同样的参数。 +请查阅[部署矩阵](deployment_matrix_zh.md),不要假设 vLLM、llama.cpp、ONNX 或 WebSocket +导出路径都保留 Python 返回内容。 + +## 源码与验证 + +契约来源:[CAMPPlus](../funasr/models/campplus/model.py)、 +[ERes2NetV2](../funasr/models/eres2net/model.py)、 +[SenseVoice](../funasr/models/sense_voice/model.py)、 +[后处理](../funasr/utils/postprocess_utils.py)、 +[AutoModel](../funasr/auto/auto_model.py) 与 [HTTP 适配](../funasr/bin/_server_app.py)。 +[指南测试](../tests/test_speaker_emotion_docs.py) 使用记录调用的 SDK 替身和真实展示函数执行 +公开程序,验证字段保留、序列化和错误输入,不代表声学质量、身份匹配、不同人群表现或情感准确率评测。 diff --git a/model_zoo/readme.md b/model_zoo/readme.md index b42dad748..3da96f4d7 100644 --- a/model_zoo/readme.md +++ b/model_zoo/readme.md @@ -57,6 +57,10 @@ interchangeable ASR checkpoint. ## Pipeline components +For speaker vectors and SenseVoice raw/display tags, use the +[speaker and emotion guide](../docs/speaker_emotion.md). For utterance-end KWS, +use the separate [keyword spotting guide](../docs/keyword_spotting.md). + | Component | Alias | Model cards | What it does not do | | --- | --- | --- | --- | | Voice activity detection | `fsmn-vad` | [ModelScope](https://modelscope.cn/models/iic/speech_fsmn_vad_zh-cn-16k-common-pytorch/summary) / [HF](https://huggingface.co/funasr/fsmn-vad) | Does not transcribe speech or identify a speaker. | diff --git a/model_zoo/readme_zh.md b/model_zoo/readme_zh.md index 9cc02c776..744f3114f 100644 --- a/model_zoo/readme_zh.md +++ b/model_zoo/readme_zh.md @@ -54,6 +54,9 @@ print(result[0]["text"]) ## 流水线组件 +声纹向量及 SenseVoice 原始/展示标签见[说话人与情感指南](../docs/speaker_emotion_zh.md)。 +句末关键词检测见独立的 [KWS 指南](../docs/keyword_spotting_zh.md)。 + | 组件 | 别名 | 模型卡 | 不包含的能力 | | --- | --- | --- | --- | | 语音活动检测 | `fsmn-vad` | [ModelScope](https://modelscope.cn/models/iic/speech_fsmn_vad_zh-cn-16k-common-pytorch/summary) / [HF](https://huggingface.co/funasr/fsmn-vad) | 不转写语音,也不识别人名。 | diff --git a/tests/test_speaker_emotion_docs.py b/tests/test_speaker_emotion_docs.py new file mode 100644 index 000000000..20ff4ea3b --- /dev/null +++ b/tests/test_speaker_emotion_docs.py @@ -0,0 +1,159 @@ +"""Execute the bilingual recipe and the real rich-text presentation helper.""" + +import ast +import json +import os +import runpy +import sys +import types +from pathlib import Path + +import pytest +from bs4 import BeautifulSoup +from markdown import markdown + +ROOT = Path(__file__).resolve().parents[1] +GUIDES = ["speaker_emotion.md", "speaker_emotion_zh.md"] + + +def test_bilingual_task_guides_exist(): + for name in GUIDES: + assert (ROOT / "docs" / name).is_file() + + +@pytest.fixture(params=GUIDES) +def recipe(request): + path = ROOT / "docs" / request.param + soup = BeautifulSoup(markdown(path.read_text(), extensions=["fenced_code"]), "html.parser") + blocks = soup.select("code.language-python") + assert len(blocks) == 1 + ast.parse(blocks[0].get_text()) + return compile(blocks[0].get_text(), str(path), "exec") + + +class Waveform: + ndim = 1 + + def __init__(self, length=48000, ndim=1): + self.length, self.ndim = length, ndim + + def __len__(self): + return self.length + + +class Embedding: + ndim = 2 + + def __init__(self, rows=None): + self.rows = [[0.25, -0.5, 0.75]] if rows is None else rows + self.shape = (len(self.rows), len(self.rows[0]) if self.rows else 0) + self.detached = self.on_cpu = False + + def detach(self): + self.detached = True + return self + + def cpu(self): + self.on_cpu = True + return self + + def tolist(self): + assert self.detached and self.on_cpu + return self.rows + + +def run_recipe(recipe, monkeypatch, tmp_path, task="embedding", results=None, + sample_rate=16000, length=48000, ndim=1, output=None): + model_dir = tmp_path / "checkpoint" + model_dir.mkdir(exist_ok=True) + constructors, calls = [], [] + if results is None: + results = [{"spk_embedding": Embedding()}] if task == "embedding" else [ + {"key": "sample", "text": "<|zh|><|HAPPY|><|Speech|><|withitn|>你好"} + ] + + class Model: + def __init__(self, **kwargs): + constructors.append(kwargs) + + def generate(self, **kwargs): + calls.append(kwargs) + return results + + helper = runpy.run_path(str(ROOT / "funasr/utils/postprocess_utils.py")) + postprocess = types.ModuleType("funasr.utils.postprocess_utils") + postprocess.rich_transcription_postprocess = helper["rich_transcription_postprocess"] + monkeypatch.setitem(sys.modules, "funasr", types.SimpleNamespace(AutoModel=Model)) + monkeypatch.setitem(sys.modules, "funasr.utils", types.ModuleType("funasr.utils")) + monkeypatch.setitem(sys.modules, "funasr.utils.postprocess_utils", postprocess) + monkeypatch.setitem(sys.modules, "soundfile", types.SimpleNamespace( + read=lambda *a, **k: (Waveform(length, ndim), sample_rate))) + output = output or tmp_path / "result.json" + monkeypatch.setattr(sys, "argv", ["attributes.py", task, str(model_dir), "speech.wav", str(output)]) + scope = {"__name__": "__main__"} + exec(recipe, scope) + return json.loads(output.read_text()), calls, constructors + + +def test_embedding_is_a_finite_cpu_serializable_vector_not_identity(recipe, monkeypatch, tmp_path): + data, calls, constructors = run_recipe(recipe, monkeypatch, tmp_path) + assert data["result"] == {"spk_embedding": [[0.25, -0.5, 0.75]]} + assert "name" not in data["result"] and "text" not in data["result"] + assert calls[0]["batch_size"] == 1 and calls[0]["fs"] == 16000 + assert "cache" not in calls[0] and "language" not in calls[0] + assert constructors[0]["device"] == "cpu" + assert constructors[0]["trust_remote_code"] is False + assert all(constructors[0][p] is None for p in ("vad_model", "punc_model", "spk_model")) + if os.name == "posix": + assert (tmp_path / "result.json").stat().st_mode & 0o777 == 0o600 + + +def test_raw_tags_survive_real_lossy_display_processing(recipe, monkeypatch, tmp_path): + raw = "<|zh|><|HAPPY|><|Speech|><|withitn|>你好" + data, calls, _ = run_recipe(recipe, monkeypatch, tmp_path, task="sensevoice", results=[{"key": "sample", "text": raw}]) + assert data["result"] == [{"key": "sample", "raw_tagged_text": raw, "display_text": "你好😊"}] + assert calls[0]["language"] == "auto" + assert calls[0]["use_itn"] is True and calls[0]["output_timestamp"] is False + assert "return_raw_text" not in calls[0] + + +def test_multiple_sensevoice_results_are_preserved_in_order(recipe, monkeypatch, tmp_path): + raw = ["<|en|><|NEUTRAL|><|Speech|><|withitn|>Hello", "<|zh|><|SAD|><|Speech|><|withitn|>再见"] + data, _, _ = run_recipe(recipe, monkeypatch, tmp_path, task="sensevoice", results=[{"text": text} for text in raw]) + assert [item["raw_tagged_text"] for item in data["result"]] == raw + assert all(item["key"] is None for item in data["result"]) + + +@pytest.mark.parametrize("rate,length,ndim", [(8000, 48000, 1), (16000, 0, 1), (16000, 48000, 2)]) +def test_invalid_audio_rejected(recipe, monkeypatch, tmp_path, rate, length, ndim): + with pytest.raises(ValueError, match="nonempty mono 16 kHz"): + run_recipe(recipe, monkeypatch, tmp_path, sample_rate=rate, length=length, ndim=ndim) + assert not (tmp_path / "result.json").exists() + + +@pytest.mark.parametrize("rows", [[], [[]], [[], []], [[1.0], [2.0]], [[float("nan")]]]) +def test_invalid_embedding_is_not_published_as_json(recipe, monkeypatch, tmp_path, rows): + with pytest.raises(ValueError): + run_recipe(recipe, monkeypatch, tmp_path, results=[{"spk_embedding": Embedding(rows)}]) + assert not (tmp_path / "result.json").exists() + + +@pytest.mark.parametrize("task", ["embedding", "sensevoice"]) +def test_empty_result_is_not_invented(recipe, monkeypatch, tmp_path, task): + with pytest.raises(ValueError, match="No result"): + run_recipe(recipe, monkeypatch, tmp_path, task=task, results=[]) + assert not (tmp_path / "result.json").exists() + + +def test_existing_output_is_not_overwritten(recipe, monkeypatch, tmp_path): + path = tmp_path / "prior.json" + path.write_text("existing result") + with pytest.raises(FileExistsError): + run_recipe(recipe, monkeypatch, tmp_path, output=path) + assert path.read_text() == "existing result" + + +def test_malformed_rich_text_is_not_silently_discarded(recipe, monkeypatch, tmp_path): + with pytest.raises(ValueError, match="Expected SenseVoice text"): + run_recipe(recipe, monkeypatch, tmp_path, task="sensevoice", results=[{"text": 123}]) + assert not (tmp_path / "result.json").exists() diff --git a/web-pages/product-site/data/documentation.json b/web-pages/product-site/data/documentation.json index e3ad24e2a..f20c63a90 100644 --- a/web-pages/product-site/data/documentation.json +++ b/web-pages/product-site/data/documentation.json @@ -19,6 +19,7 @@ {"slug": "model-zoo", "group": "models", "zh": "Model Zoo", "en": "Model Zoo", "source_zh": "model_zoo/readme_zh.md", "source_en": "model_zoo/readme.md"}, {"slug": "streaming-vad", "group": "models", "zh": "流式语音活动检测", "en": "Streaming voice activity detection", "source_zh": "docs/streaming_vad_zh.md", "source_en": "docs/streaming_vad.md"}, {"slug": "keyword-spotting", "group": "models", "zh": "关键词检测", "en": "Keyword spotting", "source_zh": "docs/keyword_spotting_zh.md", "source_en": "docs/keyword_spotting.md"}, + {"slug": "speaker-emotion", "group": "models", "zh": "说话人与情感标签", "en": "Speakers and emotion tags", "source_zh": "docs/speaker_emotion_zh.md", "source_en": "docs/speaker_emotion.md"}, {"slug": "training", "group": "train", "zh": "微调与评测", "en": "Fine-tuning & evaluation", "source_zh": "docs/training_zh.md", "source_en": "docs/training.md"}, {"slug": "model-registration", "group": "train", "zh": "自定义模型注册", "en": "Custom model registration", "source_zh": "docs/model_registration_zh.md", "source_en": "docs/model_registration.md"}, {"slug": "docker", "group": "deploy", "zh": "容器环境与镜像选择", "en": "Containers & image selection", "source_zh": "docs/installation/docker_zh.md", "source_en": "docs/installation/docker.md"}, diff --git a/web-pages/product-site/tests/browser/speaker-emotion.spec.ts b/web-pages/product-site/tests/browser/speaker-emotion.spec.ts new file mode 100644 index 000000000..d54ef35a3 --- /dev/null +++ b/web-pages/product-site/tests/browser/speaker-emotion.spec.ts @@ -0,0 +1,35 @@ +import { expect, test } from '@playwright/test'; + +for (const width of [390, 1440]) { + for (const language of ['zh', 'en']) { + test(`Speaker and emotion guide: ${language} ${width}px`, async ({ page, context }, testInfo) => { + const prefix = language === 'en' ? '/en' : ''; + const title = language === 'en' ? 'Speakers and emotion tags' : '说话人与情感标签'; + const errors: string[] = []; + page.on('pageerror', error => errors.push(error.message)); + await context.grantPermissions(['clipboard-read', 'clipboard-write']); + await page.setViewportSize({ width, height: 900 }); + await page.goto(`${prefix}/docs/`); + await page.locator('[data-doc-search] input').fill(title); + await page.locator(`.search-results a[href="${prefix}/docs/speaker-emotion.html"]`).click(); + await expect(page.locator('.docs-title h1')).toHaveText(title); + expect(await page.evaluate(() => document.documentElement.scrollWidth - innerWidth)).toBeLessThanOrEqual(1); + await page.screenshot({ path: testInfo.outputPath('initial.png') }); + const checkpoints = page.locator('.docs-article ul').first(); + await expect(checkpoints.locator('li')).toHaveCount(3); + await checkpoints.scrollIntoViewIfNeeded(); + await page.screenshot({ path: testInfo.outputPath('checkpoints.png') }); + const block = page.locator('.docs-article pre').filter({ hasText: 'def embedding_record' }); + await expect(block).toHaveCount(1); + const expected = await block.locator('code').innerText(); + await block.locator('button').click(); + expect((await page.evaluate(() => navigator.clipboard.readText())).trim()).toBe(expected.trim()); + await page.screenshot({ path: testInfo.outputPath('code.png') }); + await page.goto(`${prefix}/docs/model-zoo.html`); + await page.locator(`.docs-article a[href="${prefix}/docs/speaker-emotion.html"]`).click(); + await expect(page).toHaveURL(new RegExp(`${prefix}/docs/speaker-emotion.html$`)); + expect(await page.evaluate(() => document.documentElement.scrollWidth - innerWidth)).toBeLessThanOrEqual(1); + expect(errors).toEqual([]); + }); + } +}