diff --git a/.github/workflows/product-site.yml b/.github/workflows/product-site.yml index f463aa189..5ee0cb5fa 100644 --- a/.github/workflows/product-site.yml +++ b/.github/workflows/product-site.yml @@ -15,6 +15,7 @@ on: - 'examples/openai_api/**/*.md' - 'scripts/gen_api_docs.py' - 'tests/test_*docs*.py' + - 'tests/test_api_signatures.py' - 'gh-pages-output/**' - 'web-pages/product-site/**' - 'web-pages/scripts/**' @@ -35,6 +36,7 @@ on: - 'examples/openai_api/**/*.md' - 'scripts/gen_api_docs.py' - 'tests/test_*docs*.py' + - 'tests/test_api_signatures.py' - 'gh-pages-output/**' - 'web-pages/product-site/**' - 'web-pages/scripts/**' @@ -89,6 +91,7 @@ jobs: tests/test_python_api_docs_contract.py tests/test_service_docs_contract.py tests/test_generated_api_docs.py + tests/test_api_signatures.py tests/test_docs_funasr_install_commands.py tests/test_moss_transcribe_diarize_docs.py -k 'not no_download_toy_against_checkout and not browser_navigation_and_responsive_overflow and not skip_link_preserves_selection_playwright' -q diff --git a/scripts/gen_api_docs.py b/scripts/gen_api_docs.py index c01abe971..9b8aa80f8 100644 --- a/scripts/gen_api_docs.py +++ b/scripts/gen_api_docs.py @@ -26,7 +26,10 @@ STYLESHEET = REPO_ROOT / "gh-pages-output" / "api-reference.css" os.chdir(REPO_ROOT) tree_structure = { - "funasr.auto": {"auto_model": "funasr/auto/auto_model.py"}, + "funasr.auto": { + "auto_model": "funasr/auto/auto_model.py", + "auto_model_vllm": "funasr/auto/auto_model_vllm.py", + }, "funasr.register": {"register": "funasr/register.py"}, "funasr.models": {}, "funasr.frontends": {}, @@ -115,6 +118,43 @@ def get_source_lines(filepath, node): def github_url(filepath, lineno): return f"{REPO_URL}/blob/{BRANCH}/{filepath}#L{lineno}" +def callable_signature(node, source, bound=False, constructor=False): + """Render source expressions without importing code or evaluating defaults.""" + def expression(value): + return ast.get_source_segment(source, value).strip() + + def argument(value, default=None, prefix=""): + result = prefix + value.arg + if value.annotation is not None: + result += ": " + expression(value.annotation) + if default is not None: + result += (" = " if value.annotation is not None else "=") + expression(default) + return result + + args = node.args + positional = args.posonlyargs + args.args + defaults = [None] * (len(positional) - len(args.defaults)) + list(args.defaults) + static = any(isinstance(d, ast.Name) and d.id == "staticmethod" + for d in node.decorator_list) + start = 1 if bound and not static and positional else 0 + parts = [] + for index in range(start, len(positional)): + parts.append(argument(positional[index], defaults[index])) + if index + 1 == len(args.posonlyargs): + parts.append("/") + if args.vararg is not None: + parts.append(argument(args.vararg, prefix="*")) + elif args.kwonlyargs: + parts.append("*") + parts.extend(argument(value, default) for value, default + in zip(args.kwonlyargs, args.kw_defaults)) + if args.kwarg is not None: + parts.append(argument(args.kwarg, prefix="**")) + signature = "(" + ", ".join(parts) + ")" + if node.returns is not None and not constructor: + signature += " -> " + expression(node.returns) + return signature + # Extract all data all_data = {} for top_level, sub_modules in tree_structure.items(): @@ -136,27 +176,30 @@ for top_level, sub_modules in tree_structure.items(): class_doc = ast.get_docstring(node) or "" src, lineno, trunc = get_source_lines(filepath, node) methods = [] + constructor = None for item in node.body: if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)): + if item.name == "__init__": + constructor = { + "signature": callable_signature(item, source, bound=True, constructor=True), + "doc": ast.get_docstring(item) or "", + "lineno": item.lineno, + } if item.name.startswith("_"): continue mdoc = ast.get_docstring(item) or "" - args = [a.arg for a in item.args.args if a.arg != "self"] - if item.args.vararg: args.append("*" + item.args.vararg.arg) - if item.args.kwarg: args.append("**" + item.args.kwarg.arg) msrc, mline, mtrunc = get_source_lines(filepath, item) - methods.append({"name": item.name, "args": args, "doc": mdoc, + methods.append({"name": item.name, + "signature": callable_signature(item, source, bound=True), "doc": mdoc, "source": msrc, "lineno": mline, "truncated": mtrunc}) entries.append({"type": "class", "name": node.name, "doc": class_doc, - "methods": methods, "source": src, "lineno": lineno, + "methods": methods, "constructor": constructor, "source": src, "lineno": lineno, "truncated": trunc, "filepath": filepath}) elif isinstance(node, ast.FunctionDef) and not node.name.startswith("_"): fdoc = ast.get_docstring(node) or "" - args = [a.arg for a in node.args.args] - if node.args.vararg: args.append("*" + node.args.vararg.arg) - if node.args.kwarg: args.append("**" + node.args.kwarg.arg) fsrc, fline, ftrunc = get_source_lines(filepath, node) - entries.append({"type": "function", "name": node.name, "args": args, + entries.append({"type": "function", "name": node.name, + "signature": callable_signature(node, source), "doc": fdoc, "source": fsrc, "lineno": fline, "truncated": ftrunc, "filepath": filepath}) if entries: @@ -169,7 +212,20 @@ print(f"Extracted: {len(all_data)} modules, {total_entries} entries") sidebar = "" content = "" overview_links = "" +# Allocate newly documented vLLM entries after legacy entries, even though their +# navigation belongs beside AutoModel. Existing numeric deep links stay stable. +new_module = "funasr.auto.auto_model_vllm" +anchor_modules = [key for key in all_data if key != new_module] +if new_module in all_data: + anchor_modules.append(new_module) eid = 0 +for key in anchor_modules: + for entry in all_data[key]: + eid += 1 + entry["anchor"] = f"e{eid}" + for method in entry.get("methods", []): + eid += 1 + method["anchor"] = f"e{eid}" for group_index, top_level in enumerate(tree_structure): sub_modules = tree_structure[top_level] @@ -182,23 +238,30 @@ for group_index, top_level in enumerate(tree_structure): module_id = f"{group_id}-module-{module_index}" sidebar += f'
\n' for entry in all_data[key]: - eid += 1 - eid_str = f"e{eid}" + eid_str = entry["anchor"] filepath = entry.get("filepath", "") if entry["type"] == "class": - if entry["name"] in {"AutoModel", "RegisterTables"}: + if entry["name"] in {"AutoModel", "AutoModelVLLM", "RegisterTables"}: overview_links += f'{esc(entry["name"])} ' sidebar += f'{esc(entry["name"])}\n' gh_link = github_url(filepath, entry["lineno"]) - content += f'
class

{esc(entry["name"])}

{key} · View on GitHub ↗
{format_doc(entry["doc"])}
' + title = esc(entry["name"]) + if entry["constructor"] is not None: + title = '' + esc(entry["name"] + entry["constructor"]["signature"]) + '' + content += f'
class

{title}

{key} · View on GitHub ↗
{format_doc(entry["doc"])}
' content += f'
📄 Source code
{esc(entry["source"])}
' if entry["truncated"]: content += f'View full source on GitHub →' content += '
' + if entry["constructor"] is not None: + constructor = entry["constructor"] + constructor_url = github_url(filepath, constructor["lineno"]) + content += f'

Constructor

L{constructor["lineno"]}' + content += format_doc(constructor["doc"]) + '
' if entry["methods"]: content += '

Methods

' for m in entry["methods"]: - sig = f'.{m["name"]}({", ".join(m["args"])})' + sig = f'.{m["name"]}{m["signature"]}' mgh = github_url(filepath, m["lineno"]) content += f'
{esc(sig)} L{m["lineno"]}
{format_doc(m["doc"])}
' content += f'
📄 Source
{esc(m["source"])}
' @@ -207,10 +270,9 @@ for group_index, top_level in enumerate(tree_structure): content += '
' content += '
\n' for m in entry["methods"]: - eid += 1 - mid = f"e{eid}" + mid = m["anchor"] sidebar += f'.{esc(m["name"])}()\n' - sig = f'{entry["name"]}.{m["name"]}({", ".join(m["args"])})' + sig = f'{entry["name"]}.{m["name"]}{m["signature"]}' mgh = github_url(filepath, m["lineno"]) content += f'
method

{esc(sig)}

{key}.{entry["name"]} · View on GitHub ↗
{format_doc(m["doc"])}
' content += f'
📄 Source code
{esc(m["source"])}
' @@ -218,10 +280,10 @@ for group_index, top_level in enumerate(tree_structure): content += f'View full source on GitHub →' content += '
\n' else: - args_str = ", ".join(entry.get("args", [])) + signature = entry["name"] + entry["signature"] gh_link = github_url(filepath, entry["lineno"]) sidebar += f'{esc(entry["name"])}\n' - content += f'
function

{esc(entry["name"])}({esc(args_str)})

{key} · View on GitHub ↗
{format_doc(entry["doc"])}
' + content += f'
function

{esc(signature)}

{key} · View on GitHub ↗
{format_doc(entry["doc"])}
' content += f'
📄 Source code
{esc(entry["source"])}
' if entry["truncated"]: content += f'View full source on GitHub →' diff --git a/tests/test_api_signatures.py b/tests/test_api_signatures.py new file mode 100644 index 000000000..d03037b7b --- /dev/null +++ b/tests/test_api_signatures.py @@ -0,0 +1,118 @@ +"""Source-only API signatures must describe the callable without importing it.""" + +import html +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] +FIXTURE = '''raise RuntimeError("Documentation must never import this module") + +class Example: + def __init__(self, model: str, *, device: str = "cpu"): + """Construct without loading weights in the documentation builder.""" + + def generate(self, input, input_len=None, progress_callback=None, **cfg): + pass + + def typed(this, value: "Tensor", /, limit: int = 3, *items: str, + required: bool, label: str = "", **options: int) -> "Result": + pass + + @staticmethod + def static(self: int = 5, *, enabled=False): + pass + + @classmethod + def factory(cls, value=None): + pass + +def helper(value=explode(), *, required, optional=None): + pass +''' + + +def headings(page): + return [html.unescape(re.sub(r"<[^>]*>", "", item)) + for item in re.findall(r"

(.*?)

", page, re.S)] + + +class ApiSignatureTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.temp = tempfile.TemporaryDirectory(prefix="funasr-signatures-") + cls.repo = Path(cls.temp.name) + for directory in ("scripts", "funasr/auto", "gh-pages-output"): + (cls.repo / directory).mkdir(parents=True, exist_ok=True) + shutil.copy2(ROOT / "scripts/gen_api_docs.py", cls.repo / "scripts") + shutil.copy2(ROOT / "gh-pages-output/api-reference.css", cls.repo / "gh-pages-output") + (cls.repo / "funasr/auto/auto_model.py").write_text(FIXTURE) + (cls.repo / "funasr/register.py").write_text("def register(value=None): pass\n") + shutil.copy2(ROOT / "funasr/auto/auto_model_vllm.py", cls.repo / "funasr/auto") + result = subprocess.run([sys.executable, str(cls.repo / "scripts/gen_api_docs.py")], + capture_output=True, text=True, timeout=60) + if result.returncode: + raise AssertionError(result.stdout + result.stderr) + cls.pages = [(cls.repo / "gh-pages-output" / name).read_text() + for name in ("api.html",)] + + @classmethod + def tearDownClass(cls): + cls.temp.cleanup() + + def test_defaults_are_visible_without_evaluation(self): + for page in self.pages: + self.assertIn("Example.generate(input, input_len=None, progress_callback=None, **cfg)", headings(page)) + self.assertIn("helper(value=explode(), *, required, optional=None)", headings(page)) + + def test_all_argument_kinds_annotations_and_return(self): + expected = 'Example.typed(value: "Tensor", /, limit: int = 3, *items: str, required: bool, label: str = "", **options: int) -> "Result"' + for page in self.pages: + self.assertIn(expected, headings(page)) + self.assertIn('<tag>', page) + self.assertNotIn('', page) + + def test_static_and_classmethod_receivers(self): + for page in self.pages: + self.assertIn("Example.static(self: int = 5, *, enabled=False)", headings(page)) + self.assertIn("Example.factory(value=None)", headings(page)) + + def test_constructor_is_public_without_a_new_numeric_anchor(self): + for page in self.pages: + self.assertIn('Example(model: str, *, device: str = "cpu")', headings(page)) + self.assertIn('Construct without loading weights', page) + # Adding constructor details and a vLLM module must not move old links. + old = re.findall(r'class="api-detail"[^>]*id="(e\d+)"[^>]*>.*?

(.*?)

', page, re.S) + by_id = {key: html.unescape(re.sub(r'<[^>]*>', '', title)) for key, title in old} + self.assertTrue(by_id['e1'].startswith('Example(')) + self.assertTrue(by_id['e2'].startswith('Example.generate(')) + self.assertTrue(by_id['e6'].startswith('helper(')) + self.assertTrue(by_id['e7'].startswith('register(')) + + def test_real_vllm_constructor_and_discovery(self): + for page in self.pages: + matches = [h for h in headings(page) if h.startswith('AutoModelVLLM(')] + self.assertEqual(len(matches), 1) + self.assertIn('model: str, hub: str = "ms", device: str = "cuda:0"', matches[0]) + self.assertIn('tensor_parallel_size: int = 1', matches[0]) + self.assertIn('enforce_eager: bool = False, **kwargs)', matches[0]) + self.assertRegex(page, r'class="entry-link">AutoModelVLLM ') + self.assertIn('funasr/auto/auto_model_vllm.py#L', page) + + def test_actual_auto_model_generate_defaults(self): + with tempfile.TemporaryDirectory(prefix="funasr-signature-real-") as output: + result = subprocess.run([sys.executable, str(ROOT / 'scripts/gen_api_docs.py'), + '--output-dir', output], capture_output=True, text=True, timeout=60) + self.assertEqual(result.returncode, 0, result.stderr) + for name in ('api.html',): + self.assertIn('AutoModel.generate(input, input_len=None, progress_callback=None, **cfg)', + headings((Path(output) / name).read_text())) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_generated_api_docs.py b/tests/test_generated_api_docs.py index 7ab9a70a4..bee52c8a1 100644 --- a/tests/test_generated_api_docs.py +++ b/tests/test_generated_api_docs.py @@ -194,7 +194,9 @@ console.log('hash routing passed'); page = Page((output / "api.html").read_text()) entries = page.with_class("api-detail") self.assertGreater(len(entries), 100) - self.assertEqual([attrs["id"] for _, attrs in entries], + # New modules may be grouped beside old modules in navigation while + # receiving IDs after legacy entries to preserve existing deep links. + self.assertEqual(sorted((attrs["id"] for _, attrs in entries), key=lambda value: int(value[1:])), [f"e{i}" for i in range(1, len(entries) + 1)]) def test_skip_link_preserves_selection_playwright(self):