mirror of
https://github.com/codewhale-hq/Codewhale.git
synced 2026-11-07 10:36:05 -10:00
CodeWhale Bot
e34f207d21
Move the Engine band, extension host, fixtures and shared test support into codewhale-engine; wire TUI/CLI to that owner. Preserve human-decision construction boundaries and lower reusable test helpers. Remove unused copied dependencies and repair path-based guards, scripts and crate maps. Correct the blocking scanner external-module scope leak; raised counts equal the corrected pre-move source. Refresh embedded built-in hashes after path-only bundle regeneration. Restore the provider-free prompt budget to the exact pre-b984e31290 macOS bytes and representative identities; tool catalogs are unchanged. Portable prompt-fixture pinning remains follow-up work. Local validation: Engine/TUI/CLI clippy --tests --no-deps -D warnings passed with the existing allow-list; production CLI check passed; fmt passed. Focused native run: 705 passed / 83 failed initially; after hash/path repairs, 81 affected cases plus 2 existing build-relocation cases passed (83/83). Thus 786/788 original cases pass; 2 compiled-release-image cases lack input and are not qualified. Frozen input inventories unchanged. macOS test-link unwind-table warnings remain. Four metric tests pass; all 55 budgets pass. Host Node: 413 pass, 7 skip; Bun: 414 pass, 6 skip. Existing scanner checks: 34 + 33 pass. Compiled-delivery synthetic fixtures: 7 pass. Web: 42 pass. Import, blocking, dead-code, coarse-map, version and facts checks pass. CW-7701. No hosted CI, provider call, release build or publication. Stage A part 2 and the old App Server/ratatui dependencies remain separate.
98 lines
2.6 KiB
Python
98 lines
2.6 KiB
Python
#!/usr/bin/env python3
|
|
"""Measure the provider-free model-facing runtime contract.
|
|
|
|
Combines the serialized tool catalog and the rendered system prompt into a
|
|
single reproducible receipt. No API keys or live providers are required.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
|
|
|
|
METRIC_TEST_MODULE = "core::engine::tests"
|
|
|
|
|
|
def metric_test_name(test_name: str) -> str:
|
|
return f"{METRIC_TEST_MODULE}::{test_name}"
|
|
|
|
|
|
def metric_command(test_name: str) -> list[str]:
|
|
exact_test_name = metric_test_name(test_name)
|
|
return [
|
|
"cargo",
|
|
"test",
|
|
"--locked",
|
|
"-p",
|
|
"codewhale-engine",
|
|
"--lib",
|
|
exact_test_name,
|
|
"--",
|
|
"--ignored",
|
|
"--exact",
|
|
"--nocapture",
|
|
"--test-threads=1",
|
|
]
|
|
|
|
|
|
def run_metric(test_name: str, marker: str) -> dict:
|
|
cmd = metric_command(test_name)
|
|
proc = subprocess.run(cmd, text=True, capture_output=True, check=False)
|
|
sys.stderr.write(proc.stderr)
|
|
if proc.returncode != 0:
|
|
sys.stdout.write(proc.stdout)
|
|
proc.check_returncode()
|
|
|
|
combined = proc.stdout.splitlines() + proc.stderr.splitlines()
|
|
if re.search(r"\brunning\s+0\s+tests?\b", "\n".join(combined)):
|
|
sys.stdout.write(proc.stdout)
|
|
raise RuntimeError(
|
|
f"exact library metric test {metric_test_name(test_name)} ran zero tests"
|
|
)
|
|
for line in combined:
|
|
if marker in line:
|
|
return json.loads(line.split(marker, 1)[1])
|
|
|
|
sys.stdout.write(proc.stdout)
|
|
raise RuntimeError(
|
|
f"missing {marker.rstrip()} marker from exact library metric test "
|
|
f"{metric_test_name(test_name)}"
|
|
)
|
|
|
|
|
|
def main() -> int:
|
|
tool_metrics = run_metric(
|
|
"print_mode_tool_catalog_metrics",
|
|
"TOOL_CATALOG_METRICS ",
|
|
)
|
|
prompt_metrics = run_metric(
|
|
"print_mode_runtime_contract_metrics",
|
|
"RUNTIME_CONTRACT_METRICS ",
|
|
)
|
|
representative_context_metrics = run_metric(
|
|
"print_representative_runtime_context_metrics",
|
|
"REPRESENTATIVE_CONTEXT_METRICS ",
|
|
)
|
|
skill_discovery_metrics = run_metric(
|
|
"print_skill_discovery_turn_metrics",
|
|
"SKILL_DISCOVERY_METRICS ",
|
|
)
|
|
|
|
receipt = {
|
|
"document_kind": "codewhale.runtime_contract_receipt",
|
|
"schema_version": 1,
|
|
"representative_context": representative_context_metrics,
|
|
"skill_discovery": skill_discovery_metrics,
|
|
"tool_catalog": tool_metrics,
|
|
"system_prompt": prompt_metrics,
|
|
}
|
|
print(json.dumps(receipt, indent=2, sort_keys=True))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|