From c75338c995106e03c49d2308e19abe725eb92e69 Mon Sep 17 00:00:00 2001 From: zhifu gao Date: Sun, 6 Sep 2026 14:59:39 +0000 Subject: [PATCH] docs: resolve repository links in legacy Sphinx output Signed-off-by: zhifu gao --- .github/workflows/product-site.yml | 28 ++++ docs/_ext/repository_links.py | 146 ++++++++++++++++++++ docs/conf.py | 6 + docs/reference/FQA.md | 2 +- tests/test_sphinx_links.py | 208 +++++++++++++++++++++++++++++ 5 files changed, 389 insertions(+), 1 deletion(-) create mode 100644 docs/_ext/repository_links.py create mode 100644 tests/test_sphinx_links.py diff --git a/.github/workflows/product-site.yml b/.github/workflows/product-site.yml index 70fbc2681..d9688126d 100644 --- a/.github/workflows/product-site.yml +++ b/.github/workflows/product-site.yml @@ -5,6 +5,9 @@ on: paths: - 'README*.md' - 'docs/**/*.md' + - 'docs/conf.py' + - 'docs/_ext/**' + - 'tests/test_sphinx_links.py' - 'model_zoo/**/*.md' - 'runtime/**/*.md' - 'examples/openai_api/**/*.md' @@ -20,6 +23,9 @@ on: paths: - 'README*.md' - 'docs/**/*.md' + - 'docs/conf.py' + - 'docs/_ext/**' + - 'tests/test_sphinx_links.py' - 'model_zoo/**/*.md' - 'runtime/**/*.md' - 'examples/openai_api/**/*.md' @@ -34,6 +40,28 @@ permissions: contents: read jobs: + legacy-docs-links: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Install documentation dependencies + run: >- + python -m pip install -r web-pages/product-site/requirements-site.txt + Sphinx==9.1.0 docutils==0.22.4 recommonmark==0.7.1 + nbsphinx==0.9.8 sphinx-rtd-theme==3.1.0 sphinx-markdown-tables==0.0.17 + - name: Check legacy document links without skips + run: | + python -m pytest tests/test_sphinx_links.py -q --junitxml=/tmp/sphinx-links.xml + python - <<'PY' + import xml.etree.ElementTree as ET + report = ET.parse('/tmp/sphinx-links.xml').getroot() + assert list(report.iter('testcase')), 'No Sphinx link tests ran' + assert not list(report.iter('skipped')), 'Sphinx link tests must not be skipped' + PY + build-and-validate: runs-on: ubuntu-latest steps: diff --git a/docs/_ext/repository_links.py b/docs/_ext/repository_links.py new file mode 100644 index 000000000..95e52a0db --- /dev/null +++ b/docs/_ext/repository_links.py @@ -0,0 +1,146 @@ +"""Keep repository Markdown links usable in the legacy Sphinx output.""" + +from html import escape +from html.parser import HTMLParser +from pathlib import Path +from urllib.parse import quote, unquote, urlsplit, urlunsplit + +from docutils import nodes +from recommonmark.parser import CommonMarkParser +from sphinx.util import logging + +LOGGER = logging.getLogger(__name__) +GITHUB = 'https://github.com/modelscope/FunASR' + + +def resolve_uri(app, source, uri, *, raw=False, validate=False): + """Resolve only existing repository targets; None leaves Sphinx in charge.""" + link = urlsplit(uri) + if link.scheme or link.netloc or link.path.startswith('/'): + return uri + if not link.path and not (raw or link.query): + return None + root = Path(app.srcdir).resolve().parent + target = (source.parent / unquote(link.path)).resolve() if link.path else source.resolve() + if not target.is_relative_to(root) or not target.exists(): + if raw: + LOGGER.warning('Unresolved repository link: %s', uri, + location=str(source), type='repository_links', subtype='missing') + return None + target_doc = app.env.path2doc(str(target)) + if target_doc in app.env.found_docs: + if not raw and not link.query: + # Preserve Sphinx's own document/fragment resolution and warnings. + return None + current_doc = app.env.path2doc(str(source)) + destination = app.builder.get_relative_uri(current_doc, target_doc) + if validate and link.fragment: + validate_fragment(app, source, target_doc, uri, unquote(link.fragment)) + else: + kind = 'tree' if target.is_dir() else 'blob' + destination = f'{GITHUB}/{kind}/main/{quote(target.relative_to(root).as_posix(), safe="/")}' + parts = urlsplit(destination) + return urlunsplit((parts.scheme, parts.netloc, parts.path, link.query, link.fragment)) + + +def validate_fragment(app, source, target_doc, uri, fragment): + """Called during writing, when every target doctree has been read.""" + target = app.env.get_doctree(target_doc) + ids = set(target.ids) + for node in target.findall(nodes.raw): + if 'html' in node.get('format', '').split(): + ids.update(RawLinks(node.astext(), lambda uri: None).ids) + if fragment not in ids: + LOGGER.warning('Unresolved repository fragment: %s', uri, + location=str(source), type='repository_links', subtype='fragment') + + +class RepositoryMarkdownParser(CommonMarkParser): + def visit_link(self, mdnode): + source = Path(self.document['source']) + uri = resolve_uri(self.document.settings.env.app, source, mdnode.destination) + if uri is None: + return super().visit_link(mdnode) + # Resolve before recommonmark strips .md or treats //host as a doc ref. + reference = nodes.reference(refuri=uri) + reference['repository_links_uri'] = mdnode.destination + reference.line = self._get_line(mdnode) + if mdnode.title: + reference['title'] = mdnode.title + self.current_node.append(reference) + self.current_node = reference + + +class RawLinks(HTMLParser): + """Replace changed opening tags only, preserving all other raw HTML bytes.""" + + def __init__(self, text, resolve): + super().__init__(convert_charrefs=False) + self.text = text + self.resolve = resolve + self.replacements = [] + self.ids = set() + self.offsets = [0] + for line in text.splitlines(keepends=True): + self.offsets.append(self.offsets[-1] + len(line)) + self.feed(text) + self.close() + + def handle_starttag(self, tag, attrs): + attributes = dict(attrs) + if attributes.get('id'): + self.ids.add(attributes['id']) + if tag == 'a' and attributes.get('name'): + self.ids.add(attributes['name']) + if tag != 'a': + return + uri = attributes.get('href') + if uri is None: + return + destination = self.resolve(uri) + if destination is None or destination == uri: + return + original = self.get_starttag_text() + rewritten = '<' + tag + for key, value in attrs: + if key == 'href': + value = destination + rewritten += ' ' + key + if value is not None: + rewritten += '="' + escape(value, quote=True) + '"' + rewritten += '/>' if original.endswith('/>') else '>' + line, column = self.getpos() + start = self.offsets[line - 1] + column + self.replacements.append((start, start + len(original), rewritten)) + + def rendered(self): + result = self.text + for start, end, replacement in reversed(self.replacements): + result = result[:start] + replacement + result[end:] + return result + + +def resolve_links(app, doctree, docname): + if app.builder.format != 'html': + return + source = Path(doctree['source']) + for node in doctree.findall(nodes.reference): + original = node.attributes.pop('repository_links_uri', None) + if original is not None: + resolve_uri(app, source, original, validate=True) + for node in doctree.findall(nodes.raw): + if 'html' not in node.get('format', '').split(): + continue + original = node.astext() + rewritten = RawLinks( + original, lambda uri: resolve_uri(app, source, uri, raw=True, validate=True) + ).rendered() + if rewritten != original: + node.children[:] = [] + node += nodes.Text(rewritten) + + +def setup(app): + app.add_source_parser(RepositoryMarkdownParser, override=True) + app.connect('doctree-resolved', resolve_links) + return {'version': '1.0', 'parallel_read_safe': True} diff --git a/docs/conf.py b/docs/conf.py index a1bdcd7c4..230b1712e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,6 +14,11 @@ # import sys # sys.path.insert(0, os.path.abspath('.')) +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent / '_ext')) + # -- Project information ----------------------------------------------------- @@ -38,6 +43,7 @@ extensions = [ "sphinx_markdown_tables", "recommonmark", "sphinx_rtd_theme", + "repository_links", ] # Add any paths that contain templates here, relative to this directory. diff --git a/docs/reference/FQA.md b/docs/reference/FQA.md index 4dcb397ca..b4c24dc2c 100644 --- a/docs/reference/FQA.md +++ b/docs/reference/FQA.md @@ -199,7 +199,7 @@ result = model.generate(input="meeting.wav") For a fully custom speaker model, make sure the model can be loaded by FunASR `AutoModel` and that its `inference()` method returns `spk_embedding`; the user-facing call still goes through `AutoModel.generate()`. The diarization post-processing and clustering steps use those embeddings to assign speaker labels. If your diarization model emits speaker segments directly instead of embeddings, add an adapter layer that converts its output to the current speaker-label structure. -The tutorial has a longer ERes2NetV2 example: [Speaker Verification / Diarization](../tutorial/README.md#speaker-verification--diarization-eres2netv2). +For the supported pipeline and result fields, see [VAD, timestamps, and speakers](../python_api.md#vad-timestamps-and-speakers). ## The same command works on CPU but fails on CUDA diff --git a/tests/test_sphinx_links.py b/tests/test_sphinx_links.py new file mode 100644 index 000000000..efd4af82c --- /dev/null +++ b/tests/test_sphinx_links.py @@ -0,0 +1,208 @@ +"""Small, model-free builds using the real Sphinx configuration and link adapter. + +Run with Sphinx and the extensions listed in docs/conf.py installed. +""" + +import os +from pathlib import Path +import shutil +import subprocess +import sys + +from bs4 import BeautifulSoup +import pytest + +pytest.importorskip('sphinx') +ROOT = Path(__file__).resolve().parents[1] + +GUIDE = '''# Link fixture + +[Body code](../../examples/demo.py) +[Body model guide](../../examples/guide.md) +[Body directory](../../examples/) +[Body license](../../LICENSE) +[Body encoded](../../examples/folder%20name/guide.md?view=1&lang=zh#usage) +[Body Unicode](../../examples/中文/guide.md) +[Body internal](../training.md) +[Body query](../training.md?view=1#details) +[Body query without fragment](../training.md?view=2) +[Body query raw anchor](../training.md?view=1#%72aw-anchor) +[Body query missing](../training.md?view=1#body-query-ABSENT) +[Body later target](../zz_later.md?view=1#details) +[Body external](https://docs.example.test/guide.md?view=1#usage) +[Body cross-host](//docs.example.test/guide.md) +[Body root](../../../outside.txt) +[Body missing](../../examples/missing.py) +[Body missing fragment](../training.md#absent-section) + +| Case | Destination | +| --- | --- | +| code | [Table code](../../examples/demo.py) | +| markdown | [Table model guide](../../examples/guide.md) | +| directory | [Table directory](../../examples/) | +| encoded | [Table encoded](../../examples/folder%20name/guide.md?view=1&lang=zh#usage) | +| Unicode | [Table Unicode](../../examples/中文/guide.md) | +| internal | [Table internal](../training.md?view=1#details) | +| query | [Table query without fragment](../training.md?view=2) | +| raw anchor | [Table raw anchor](../training.md?view=1#%72aw-anchor) | +| missing fragment | [Table missing fragment](../training.md#table-ABSENT) | +| missing query fragment | [Table missing query fragment](../training.md?view=1#table-query-ABSENT) | +| later target | [Table later target](../zz_later.md?view=1#details) | +| rst | [Table RST](../reference.rst) | +| external | [Table external](https://docs.example.test/guide.md?view=1#usage) | +| cross-host | [Table cross-host](//docs.example.test/guide.md) | +| missing | [Table missing](../../examples/table-missing.py) | +| escape | [Table escape](../../../outside.txt) | +| symlink | [Table symlink](../../examples/escape.txt) | + +Raw cross-host +Root-relative +Mail +Local fragment + +Raw missing fragment +Raw missing self fragment + +```python +print("../examples/demo.py is code, not a link") +``` +''' + + +@pytest.fixture(scope='module') +def built(tmp_path_factory): + work = tmp_path_factory.mktemp('sphinx-links') + repo = work / 'repo' + docs = repo / 'docs' + (docs / 'tutorial').mkdir(parents=True) + shutil.copy2(ROOT / 'docs/conf.py', docs / 'conf.py') + if (ROOT / 'docs/_ext').is_dir(): + shutil.copytree(ROOT / 'docs/_ext', docs / '_ext') + (docs / 'index.rst').write_text( + 'Fixture\n=======\n\n.. toctree::\n\n tutorial/guide\n tutorial/guide_zh\n training\n reference\n zz_later\n') + for name in ('guide.md', 'guide_zh.md'): + (docs / 'tutorial' / name).write_text(GUIDE, encoding='utf-8') + (docs / 'training.md').write_text( + '# Training\n\n## Details\n\nReal destination.\n\n\n') + (docs / 'reference.rst').write_text('Reference\n=========\n\nExisting RST page.\n') + # Sphinx reads this target after tutorial/*, so early doctree access fails. + (docs / 'zz_later.md').write_text('# Later target\n\n## Details\n\nLate-read destination.\n') + for name in ('examples/demo.py', 'examples/guide.md', 'LICENSE', + 'examples/folder name/guide.md', 'examples/中文/guide.md'): + path = repo / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('fixture\n', encoding='utf-8') + (work / 'outside.txt').write_text('outside repository\n') + (repo / 'examples/escape.txt').symlink_to(work / 'outside.txt') + original = {p: p.read_bytes() for p in docs.rglob('*') if p.is_file()} + output = work / 'html' + command = [sys.executable, '-m', 'sphinx', '-b', 'html', '-n', '-E', + '-d', str(work / 'doctrees'), str(docs), str(output)] + result = subprocess.run(command, text=True, capture_output=True, timeout=90, + env={**os.environ, 'PYTHONDONTWRITEBYTECODE': '1'}) + assert result.returncode == 0, result.stdout + result.stderr + assert all(p.read_bytes() == text for p, text in original.items()) + pages = {name: BeautifulSoup((output / f'tutorial/{name}.html').read_text(), 'html.parser') + for name in ('guide', 'guide_zh')} + return output, pages, result.stdout + result.stderr + + +@pytest.mark.parametrize('language', ('guide', 'guide_zh')) +@pytest.mark.parametrize('label,path', ( + ('Body code', 'blob/main/examples/demo.py'), + ('Body model guide', 'blob/main/examples/guide.md'), + ('Body directory', 'tree/main/examples'), + ('Body license', 'blob/main/LICENSE'), + ('Body encoded', 'blob/main/examples/folder%20name/guide.md?view=1&lang=zh#usage'), + ('Body Unicode', 'blob/main/examples/%E4%B8%AD%E6%96%87/guide.md'), + ('Table code', 'blob/main/examples/demo.py'), + ('Table model guide', 'blob/main/examples/guide.md'), + ('Table directory', 'tree/main/examples'), + ('Table encoded', 'blob/main/examples/folder%20name/guide.md?view=1&lang=zh#usage'), + ('Table Unicode', 'blob/main/examples/%E4%B8%AD%E6%96%87/guide.md'), +)) +def test_existing_repository_targets_use_github(built, language, label, path): + _, pages, _ = built + link = pages[language].find('a', string=label) + assert link is not None + assert link['href'] == 'https://github.com/modelscope/FunASR/' + path + + +@pytest.mark.parametrize('language', ('guide', 'guide_zh')) +def test_internal_table_links_use_builder_routes(built, language): + output, pages, _ = built + page = pages[language] + assert page.find('a', string='Body internal')['href'] == '../training.html' + assert page.find('a', string='Table internal')['href'] == '../training.html?view=1#details' + assert page.find('a', string='Table RST')['href'] == '../reference.html' + assert (output / 'training.html').is_file() + assert BeautifulSoup((output / 'training.html').read_text(), 'html.parser').find(id='details') + + +@pytest.mark.parametrize('language', ('guide', 'guide_zh')) +def test_body_and_table_queries_resolve_identically(built, language): + _, pages, log = built + page = pages[language] + for body, table, uri in ( + ('Body query', 'Table internal', '../training.html?view=1#details'), + ('Body query without fragment', 'Table query without fragment', '../training.html?view=2'), + ('Body query raw anchor', 'Table raw anchor', '../training.html?view=1#%72aw-anchor'), + ('Body later target', 'Table later target', '../zz_later.html?view=1#details'), + ): + assert page.find('a', string=body)['href'] == uri + assert page.find('a', string=table)['href'] == uri + warnings = [line for line in log.splitlines() if 'WARNING:' in line] + assert not any('raw-anchor' in line or '%72aw-anchor' in line or '#details' in line + or 'docs.example.test' in line or '#local-anchor' in line for line in warnings) + + +@pytest.mark.parametrize('fragment', ( + 'body-query-ABSENT', 'table-ABSENT', 'table-query-ABSENT', 'raw-ABSENT', 'self-ABSENT', +)) +def test_unresolved_query_and_raw_fragments_warn(built, fragment): + _, _, log = built + warnings = [line for line in log.splitlines() if 'WARNING:' in line] + assert any(fragment in line and '[repository_links.fragment]' in line for line in warnings) + + +@pytest.mark.parametrize('language', ('guide', 'guide_zh')) +def test_external_hosts_root_urls_and_code_are_untouched(built, language): + _, pages, _ = built + page = pages[language] + for label, uri in ( + ('Body external', 'https://docs.example.test/guide.md?view=1#usage'), + ('Table external', 'https://docs.example.test/guide.md?view=1#usage'), + ('Body cross-host', '//docs.example.test/guide.md'), + ('Table cross-host', '//docs.example.test/guide.md'), + ('Raw cross-host', '//docs.example.test/raw.md'), + ('Root-relative', '/other-site/guide.md'), + ('Mail', 'mailto:help@example.test'), + ('Local fragment', '#local-anchor'), + ): + assert page.find('a', string=label)['href'] == uri + assert page.find('a', string='Raw cross-host')['data-label'] == 'do not alter' + assert any('print("../examples/demo.py is code, not a link")' in pre.get_text() + for pre in page.select('pre')) + + +def test_missing_targets_and_fragments_still_warn(built): + _, pages, log = built + warnings = [line for line in log.splitlines() if 'WARNING:' in line] + for marker in ('missing.py', 'table-missing.py', 'outside.txt', 'escape.txt', 'absent-section'): + assert any(marker in line for line in warnings), (marker, warnings) + assert not any('examples/demo.py' in line or 'examples/guide' in line for line in warnings) + for page in pages.values(): + assert page.find('a', string='Body missing')['href'] == '../../examples/missing.py' + assert page.find('a', string='Table missing')['href'] == '../../examples/table-missing.py' + assert page.find('a', string='Table escape')['href'] == '../../../outside.txt' + assert page.find('a', string='Table symlink')['href'] == '../../examples/escape.txt' + + +def test_faq_points_to_the_existing_speaker_contract(): + source = (ROOT / 'docs/reference/FQA.md').read_text() + assert 'speaker-verification--diarization-eres2netv2' not in source + assert '../python_api.md#vad-timestamps-and-speakers' in source + from markdown import markdown + page = BeautifulSoup(markdown((ROOT / 'docs/python_api.md').read_text(), + extensions=['extra', 'toc']), 'html.parser') + assert page.find(id='vad-timestamps-and-speakers')