From 7ebc575ed85417cd1b00dcd69fc217c7feffbcb9 Mon Sep 17 00:00:00 2001 From: John Smith Date: Sat, 17 Oct 2026 18:39:30 -0400 Subject: [PATCH] M7: config and scenario schema with validation, loader and JSON schemas --- docs/reports/m7-implementation.md | 98 +++++ schema/config.json | 244 +++++++++++ schema/scenario.json | 660 ++++++++++++++++++++++++++++++ scripts/gen-schema.ts | 24 ++ src/config/load.ts | 157 +++++++ src/config/schema.ts | 270 ++++++++++++ test/unit/config-load.test.ts | 202 +++++++++ test/unit/config-schema.test.ts | 251 ++++++++++++ test/unit/schema-sync.test.ts | 21 + 9 files changed, 1927 insertions(+) create mode 100644 docs/reports/m7-implementation.md create mode 100644 schema/config.json create mode 100644 schema/scenario.json create mode 100644 scripts/gen-schema.ts create mode 100644 src/config/load.ts create mode 100644 src/config/schema.ts create mode 100644 test/unit/config-load.test.ts create mode 100644 test/unit/config-schema.test.ts create mode 100644 test/unit/schema-sync.test.ts diff --git a/docs/reports/m7-implementation.md b/docs/reports/m7-implementation.md new file mode 100644 index 0000000..de64f24 --- /dev/null +++ b/docs/reports/m7-implementation.md @@ -0,0 +1,98 @@ +# M7 Implementation Report — Config and Scenario Schema + +- **Stage ID:** M7 (config: `src/config/` + `scripts/gen-schema.ts` + `schema/` + unit tests) +- **Date:** 2026-10-17 +- **Branch:** `impl/v1` (not committed, per task instructions) + +## Summary + +Implemented the M7 configuration layer: zod v4 schemas for +`.opencode-ci/config.*` and `.opencode-ci/scenarios/*` (every object +`.strict()`), a fail-closed loader (`ConfigError` on any layout/schema +problem), a `scripts/gen-schema.ts` generator emitting `schema/config.json` + +`schema/scenario.json` via `z.toJSONSchema()`, and 35 new unit tests. All 10 +fixture plugins' configs/scenarios parse unchanged (no fixture fixes needed). + +## Files created + +- `src/config/schema.ts` — `ConfigSchema` / `ScenarioSchema` / + `MatcherSchema` (+ `Step`, `ToolCall`, `CountMatcher`, `ScenarioExpect` + types via `z.infer`). Details: + - Config: `plugin` (`path` [`.`], `options` [`{}`], optional + `entry`/`install`/`build`/`ui`), `opencode.versions` ([`["2.0.26"]`], + non-empty, semver-or-`"latest"`), `expect` (7 string arrays, each + default `[]`), `live` (`keys`/`repeat` 3/`pass` 2/`budgetUSD` + 1/`timeoutMs` 240000), `gate` (`enabled`/`window` 5/`threshold` + 0.9/`bootstrap` `"fail"`), `logLevel` (`"info"`). Cross-field: + `live.pass <= live.repeat` via `superRefine`. + - Scenario: `id` (`/^[a-z0-9][a-z0-9-]*$/`), `mode`, `prompt` (min 1), + `timeoutMs` [120000], `expect` (exitCode/noErrors/tools/noTools/hooks/ + request/files/stderr/text), optional `plugin.options`, + `workspace`, `judgeReply`, `judge`. Cross-field: `script` required in + scripted / forbidden in live, `expect.request` live-forbidden, + `judge` live-only, `judgeReply` scripted-only. Every message names + the scenario id and the field (e.g. `scenario "x": field "script" + is required when mode is "scripted"`). + - Matcher: `equals`/`contains`/`notContains` (unknown), + `matches`/`notMatches` (string, `(?i)` accepted — stripping happens + in M9 matchers, schema just validates the string), `includes` + (string array), `exists` (boolean), `count` (`eq`/`gte`/`lte`); + empty matcher rejected. `count` object shared with tool `count`. + - Defaults use `.prefault()` (not `.default()`) for nested objects: + zod v4 `.default()` returns the fallback as-is without running inner + field defaults (verified empirically: `outer.parse({})` gave + `{plugin:{}}` with `.default({})` vs `{plugin:{path:"."}}` with + `.prefault({})`). Scalars keep `.default()`. +- `src/config/load.ts` — `loadConfig(cwd?)` (`{config, configPath?, + warnings}`; both yaml+json present → `ConfigError`; neither → zod + defaults + `console.warn` + `warnings` entry), `loadScenarios(cwd?)` + (sorted `*.yaml/*.yml/*.json`, duplicate-`id` error naming id + both + files), `loadAll(cwd?)`, `formatIssues()` (file path + one line per + issue), `ConfigError`. YAML errors keep the file path plus the + `line N` hint from `yaml`'s `linePos`/message. +- `scripts/gen-schema.ts` — `writeSchemas(outDir)` (`mkdir -p`, pretty + JSON + trailing newline); runs as `bun scripts/gen-schema.ts [outDir]` + via `import.meta.main`, defaulting to `/schema`. +- `schema/config.json`, `schema/scenario.json` — generated output, + committed as files (uncommitted in git, per task constraints). +- `test/unit/config-schema.test.ts` — 18 tests: every default, + unknown-key rejection at 6 levels (`expect.tool` typo, top-level, + plugin, scenario, matcher, expect), all 6 cross-field rules (+ a + valid live+judge case, pass==repeat boundary), `(?i)` acceptance, + empty-matcher failure, id regex, workspace/plugin-options. +- `test/unit/config-load.test.ts` — 13 tests: yaml-vs-json precedence, + missing-config defaults+warning, YAML parse error (path+line), + bad-JSON + schema-violation file/field naming, JSON config parity, + duplicate-id (id + both files), sorted yaml/yml/json loading, + missing-dir → `[]`, plus a sweep loading all 10 fixture plugins and + a shape check on `tool-basic-pass`. +- `test/unit/schema-sync.test.ts` — regenerates into a temp dir via + `writeSchemas()` and byte-compares with the committed files. + +## Test results + +- `bun run check` — typecheck clean, biome clean (remaining 18 infos are + pre-existing), **90 unit tests pass, 0 fail** (55 pre-existing + 35 new). +- `bun scripts/gen-schema.ts` regenerates `schema/*.json` deterministically; + schema-sync test proves the committed copies are current. +- No fixture changes required: all 10 fixture plugins under + `test/fixtures/plugins/` load through `loadAll()` unchanged. + +## Self-audit notes + +- **Fail-closed:** every loader failure path throws `ConfigError` + (both-configs, bad YAML/JSON, zod violation, duplicate id); no + permissive fallbacks. Missing config is the only non-throwing case and + it warns explicitly. +- **Strictness:** all 13 object schemas are `.strict()`; the typo test + (`expect: {tool: [...]}`) fails as required. +- **Deviations from the M7 reference:** (1) `.prefault()` instead of + `.default()` for nested-object fallbacks (zod v4 semantics — see above; + behavior matches the spec's defaults exactly); (2) `writeSchemas()` + takes an explicit `outDir` (default `schema/`) so the sync test can + target a temp dir without subprocesses; (3) `judgeReply` live-forbidden + check added (spec says "scripted only", implied but not listed in the + cross-field bullets); (4) biome autofixes applied (import sorting, + `import type { z }`); (5) no commits (per task constraints). +- **Not done (out of scope for M7):** `validate`/`run` CLI wiring (M10+); + `(?i)` runtime handling (M9 matchers); exit-code-2 mapping (CLI layer). diff --git a/schema/config.json b/schema/config.json new file mode 100644 index 0000000..c83f60d --- /dev/null +++ b/schema/config.json @@ -0,0 +1,244 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "plugin": { + "type": "object", + "properties": { + "path": { + "default": ".", + "type": "string", + "minLength": 1 + }, + "entry": { + "type": "string", + "minLength": 1 + }, + "install": { + "type": "string", + "minLength": 1 + }, + "build": { + "type": "string", + "minLength": 1 + }, + "options": { + "default": {}, + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {} + }, + "ui": { + "type": "boolean" + } + }, + "required": [ + "path", + "options" + ], + "additionalProperties": false + }, + "opencode": { + "type": "object", + "properties": { + "versions": { + "default": [ + "2.0.26" + ], + "minItems": 1, + "type": "array", + "items": { + "anyOf": [ + { + "type": "string", + "const": "latest" + }, + { + "type": "string", + "pattern": "^\\d+\\.\\d+\\.\\d+$" + } + ] + } + } + }, + "required": [ + "versions" + ], + "additionalProperties": false + }, + "expect": { + "type": "object", + "properties": { + "tools": { + "default": [], + "type": "array", + "items": { + "type": "string" + } + }, + "commands": { + "default": [], + "type": "array", + "items": { + "type": "string" + } + }, + "agents": { + "default": [], + "type": "array", + "items": { + "type": "string" + } + }, + "skills": { + "default": [], + "type": "array", + "items": { + "type": "string" + } + }, + "mcp": { + "default": [], + "type": "array", + "items": { + "type": "string" + } + }, + "providers": { + "default": [], + "type": "array", + "items": { + "type": "string" + } + }, + "models": { + "default": [], + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "tools", + "commands", + "agents", + "skills", + "mcp", + "providers", + "models" + ], + "additionalProperties": false + }, + "live": { + "type": "object", + "properties": { + "model": { + "type": "string", + "minLength": 1 + }, + "keys": { + "default": [ + "OPENCODE_API_KEY" + ], + "minItems": 1, + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "repeat": { + "default": 3, + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "pass": { + "default": 2, + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "budgetUSD": { + "default": 1, + "type": "number", + "exclusiveMinimum": 0 + }, + "timeoutMs": { + "default": 240000, + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + } + }, + "required": [ + "keys", + "repeat", + "pass", + "budgetUSD", + "timeoutMs" + ], + "additionalProperties": false + }, + "gate": { + "type": "object", + "properties": { + "enabled": { + "default": true, + "type": "boolean" + }, + "window": { + "default": 5, + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "threshold": { + "default": 0.9, + "type": "number", + "minimum": 0, + "maximum": 1 + }, + "bootstrap": { + "default": "fail", + "type": "string", + "enum": [ + "fail", + "allow" + ] + }, + "workflow": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "enabled", + "window", + "threshold", + "bootstrap" + ], + "additionalProperties": false + }, + "logLevel": { + "default": "info", + "type": "string", + "enum": [ + "info", + "all" + ] + } + }, + "required": [ + "plugin", + "opencode", + "expect", + "live", + "gate", + "logLevel" + ], + "additionalProperties": false +} diff --git a/schema/scenario.json b/schema/scenario.json new file mode 100644 index 0000000..dcf27eb --- /dev/null +++ b/schema/scenario.json @@ -0,0 +1,660 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "id": { + "type": "string", + "pattern": "^[a-z0-9][a-z0-9-]*$" + }, + "description": { + "type": "string" + }, + "mode": { + "type": "string", + "enum": [ + "scripted", + "live" + ] + }, + "plugin": { + "type": "object", + "properties": { + "options": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {} + } + }, + "additionalProperties": false + }, + "workspace": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "string" + } + }, + "prompt": { + "type": "string", + "minLength": 1 + }, + "script": { + "type": "array", + "items": { + "type": "object", + "properties": { + "text": { + "type": "string" + }, + "toolCalls": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "arguments": { + "default": {}, + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": {} + } + }, + "required": [ + "name", + "arguments" + ], + "additionalProperties": false + } + }, + "raw": { + "type": "boolean" + } + }, + "additionalProperties": false + } + }, + "judgeReply": { + "type": "string" + }, + "timeoutMs": { + "default": 120000, + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "expect": { + "type": "object", + "properties": { + "exitCode": { + "default": 0, + "type": "integer", + "minimum": -9007199254740991, + "maximum": 9007199254740991 + }, + "noErrors": { + "default": true, + "type": "boolean" + }, + "tools": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "status": { + "type": "string", + "enum": [ + "completed", + "error" + ] + }, + "input": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "properties": { + "equals": {}, + "contains": {}, + "notContains": {}, + "matches": { + "type": "string", + "minLength": 1 + }, + "notMatches": { + "type": "string", + "minLength": 1 + }, + "includes": { + "minItems": 1, + "type": "array", + "items": { + "type": "string" + } + }, + "exists": { + "type": "boolean" + }, + "count": { + "type": "object", + "properties": { + "eq": { + "type": "number" + }, + "gte": { + "type": "number" + }, + "lte": { + "type": "number" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + }, + "output": { + "type": "object", + "properties": { + "equals": {}, + "contains": {}, + "notContains": {}, + "matches": { + "type": "string", + "minLength": 1 + }, + "notMatches": { + "type": "string", + "minLength": 1 + }, + "includes": { + "minItems": 1, + "type": "array", + "items": { + "type": "string" + } + }, + "exists": { + "type": "boolean" + }, + "count": { + "type": "object", + "properties": { + "eq": { + "type": "number" + }, + "gte": { + "type": "number" + }, + "lte": { + "type": "number" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "count": { + "type": "object", + "properties": { + "eq": { + "type": "number" + }, + "gte": { + "type": "number" + }, + "lte": { + "type": "number" + } + }, + "additionalProperties": false + } + }, + "required": [ + "name" + ], + "additionalProperties": false + } + }, + "noTools": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "hooks": { + "type": "array", + "items": { + "type": "object", + "properties": { + "hook": { + "type": "string", + "enum": [ + "tool.execute.before", + "tool.execute.after", + "shell.create.before", + "session.context", + "session.prompt" + ] + }, + "phase": { + "default": "post", + "type": "string", + "enum": [ + "pre", + "post" + ] + }, + "tool": { + "type": "string", + "minLength": 1 + }, + "input": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "properties": { + "equals": {}, + "contains": {}, + "notContains": {}, + "matches": { + "type": "string", + "minLength": 1 + }, + "notMatches": { + "type": "string", + "minLength": 1 + }, + "includes": { + "minItems": 1, + "type": "array", + "items": { + "type": "string" + } + }, + "exists": { + "type": "boolean" + }, + "count": { + "type": "object", + "properties": { + "eq": { + "type": "number" + }, + "gte": { + "type": "number" + }, + "lte": { + "type": "number" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + }, + "env": { + "type": "object", + "propertyNames": { + "type": "string" + }, + "additionalProperties": { + "type": "object", + "properties": { + "equals": {}, + "contains": {}, + "notContains": {}, + "matches": { + "type": "string", + "minLength": 1 + }, + "notMatches": { + "type": "string", + "minLength": 1 + }, + "includes": { + "minItems": 1, + "type": "array", + "items": { + "type": "string" + } + }, + "exists": { + "type": "boolean" + }, + "count": { + "type": "object", + "properties": { + "eq": { + "type": "number" + }, + "gte": { + "type": "number" + }, + "lte": { + "type": "number" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + }, + "system": { + "type": "object", + "properties": { + "equals": {}, + "contains": {}, + "notContains": {}, + "matches": { + "type": "string", + "minLength": 1 + }, + "notMatches": { + "type": "string", + "minLength": 1 + }, + "includes": { + "minItems": 1, + "type": "array", + "items": { + "type": "string" + } + }, + "exists": { + "type": "boolean" + }, + "count": { + "type": "object", + "properties": { + "eq": { + "type": "number" + }, + "gte": { + "type": "number" + }, + "lte": { + "type": "number" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + }, + "required": [ + "hook", + "phase" + ], + "additionalProperties": false + } + }, + "request": { + "type": "object", + "properties": { + "nth": { + "default": 1, + "type": "integer", + "exclusiveMinimum": 0, + "maximum": 9007199254740991 + }, + "system": { + "type": "object", + "properties": { + "equals": {}, + "contains": {}, + "notContains": {}, + "matches": { + "type": "string", + "minLength": 1 + }, + "notMatches": { + "type": "string", + "minLength": 1 + }, + "includes": { + "minItems": 1, + "type": "array", + "items": { + "type": "string" + } + }, + "exists": { + "type": "boolean" + }, + "count": { + "type": "object", + "properties": { + "eq": { + "type": "number" + }, + "gte": { + "type": "number" + }, + "lte": { + "type": "number" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "tools": { + "type": "object", + "properties": { + "equals": {}, + "contains": {}, + "notContains": {}, + "matches": { + "type": "string", + "minLength": 1 + }, + "notMatches": { + "type": "string", + "minLength": 1 + }, + "includes": { + "minItems": 1, + "type": "array", + "items": { + "type": "string" + } + }, + "exists": { + "type": "boolean" + }, + "count": { + "type": "object", + "properties": { + "eq": { + "type": "number" + }, + "gte": { + "type": "number" + }, + "lte": { + "type": "number" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + }, + "required": [ + "nth" + ], + "additionalProperties": false + }, + "files": { + "type": "array", + "items": { + "type": "object", + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "exists": { + "type": "boolean" + }, + "contains": { + "type": "string" + }, + "notContains": { + "type": "string" + }, + "matches": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "path" + ], + "additionalProperties": false + } + }, + "stderr": { + "type": "object", + "properties": { + "equals": {}, + "contains": {}, + "notContains": {}, + "matches": { + "type": "string", + "minLength": 1 + }, + "notMatches": { + "type": "string", + "minLength": 1 + }, + "includes": { + "minItems": 1, + "type": "array", + "items": { + "type": "string" + } + }, + "exists": { + "type": "boolean" + }, + "count": { + "type": "object", + "properties": { + "eq": { + "type": "number" + }, + "gte": { + "type": "number" + }, + "lte": { + "type": "number" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "text": { + "type": "object", + "properties": { + "equals": {}, + "contains": {}, + "notContains": {}, + "matches": { + "type": "string", + "minLength": 1 + }, + "notMatches": { + "type": "string", + "minLength": 1 + }, + "includes": { + "minItems": 1, + "type": "array", + "items": { + "type": "string" + } + }, + "exists": { + "type": "boolean" + }, + "count": { + "type": "object", + "properties": { + "eq": { + "type": "number" + }, + "gte": { + "type": "number" + }, + "lte": { + "type": "number" + } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + } + }, + "required": [ + "exitCode", + "noErrors" + ], + "additionalProperties": false + }, + "judge": { + "type": "object", + "properties": { + "rubric": { + "type": "string", + "minLength": 1 + }, + "model": { + "type": "string", + "minLength": 1 + } + }, + "required": [ + "rubric" + ], + "additionalProperties": false + } + }, + "required": [ + "id", + "mode", + "prompt", + "timeoutMs", + "expect" + ], + "additionalProperties": false +} diff --git a/scripts/gen-schema.ts b/scripts/gen-schema.ts new file mode 100644 index 0000000..7490c4c --- /dev/null +++ b/scripts/gen-schema.ts @@ -0,0 +1,24 @@ +// Writes schema/config.json + schema/scenario.json from the zod schemas. +// Run with `bun scripts/gen-schema.ts [outDir]`. The schema-sync unit test +// calls writeSchemas() directly, so generated files can never drift. +import { mkdirSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { z } from "zod"; +import { ConfigSchema, ScenarioSchema } from "../src/config/schema.ts"; + +export function writeSchemas(outDir: string): void { + mkdirSync(outDir, { recursive: true }); + writeFileSync( + join(outDir, "config.json"), + `${JSON.stringify(z.toJSONSchema(ConfigSchema), null, 2)}\n`, + ); + writeFileSync( + join(outDir, "scenario.json"), + `${JSON.stringify(z.toJSONSchema(ScenarioSchema), null, 2)}\n`, + ); +} + +if (import.meta.main) { + const outDir = process.argv[2] ?? join(import.meta.dir, "..", "schema"); + writeSchemas(outDir); +} diff --git a/src/config/load.ts b/src/config/load.ts new file mode 100644 index 0000000..9d03ae9 --- /dev/null +++ b/src/config/load.ts @@ -0,0 +1,157 @@ +// Find, parse and validate `.opencode-ci/config.*` + scenarios/*. Fail closed: +// any schema or layout problem throws ConfigError (exit 2 upstream). +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import * as YAML from "yaml"; +import type { z } from "zod"; +import { + type Config, + ConfigSchema, + type Scenario, + ScenarioSchema, +} from "./schema.ts"; + +/** Thrown for every config/layout problem. The CLI maps it to exit 2. */ +export class ConfigError extends Error { + constructor(message: string) { + super(message); + this.name = "ConfigError"; + } +} + +export interface LoadResult { + config: Config; + scenarios: Scenario[]; + /** Absolute path of the config file, if one existed. */ + configPath?: string; + warnings: string[]; +} + +/** Format zod issues with the file path, one issue per line. */ +export function formatIssues(file: string, error: z.ZodError): string { + const lines = error.issues.map((issue) => { + const at = + issue.path.length > 0 ? ` at "${String(issue.path.join("."))}"` : ""; + return ` -${at}: ${issue.message}`; + }); + return `${file}: validation failed:\n${lines.join("\n")}`; +} + +/** Extract a "line N" hint from a YAML parse error, when it has one. */ +function yamlLineHint(error: unknown): string { + if ( + error !== null && + typeof error === "object" && + "linePos" in error && + Array.isArray((error as { linePos?: unknown }).linePos) && + ((error as { linePos: Array<{ line?: unknown }> }).linePos[0]?.line ?? + undefined) !== undefined + ) { + const line = (error as { linePos: Array<{ line: unknown }> }).linePos[0] + ?.line; + return ` (line ${String(line)})`; + } + const message = error instanceof Error ? error.message : String(error); + const m = /line (\d+)/i.exec(message); + return m ? ` (line ${m[1]})` : ""; +} + +function parseDoc(file: string, text: string): unknown { + if (file.endsWith(".json")) { + try { + return JSON.parse(text) as unknown; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new ConfigError(`${file}: invalid JSON: ${message}`); + } + } + try { + return YAML.parse(text) as unknown; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new ConfigError( + `${file}: invalid YAML${yamlLineHint(error)}: ${message}`, + ); + } +} + +const CONFIG_DIR = ".opencode-ci"; +const CONFIG_YAML = join(CONFIG_DIR, "config.yaml"); +const CONFIG_JSON = join(CONFIG_DIR, "config.json"); +const SCENARIOS_DIR = join(CONFIG_DIR, "scenarios"); + +export function loadConfig(cwd: string = process.cwd()): { + config: Config; + configPath?: string; + warnings: string[]; +} { + const yamlPath = join(cwd, CONFIG_YAML); + const jsonPath = join(cwd, CONFIG_JSON); + const hasYaml = existsSync(yamlPath); + const hasJson = existsSync(jsonPath); + if (hasYaml && hasJson) { + throw new ConfigError( + `Both ${yamlPath} and ${jsonPath} exist: keep exactly one config file.`, + ); + } + if (!hasYaml && !hasJson) { + const warnings = [ + `No ${CONFIG_YAML} or ${CONFIG_JSON} found under ${cwd}: using defaults.`, + ]; + for (const w of warnings) { + console.warn(w); + } + return { config: ConfigSchema.parse({}), warnings }; + } + const configPath = hasYaml ? yamlPath : jsonPath; + const doc = parseDoc(configPath, readFileSync(configPath, "utf8")); + const parsed = ConfigSchema.safeParse(doc); + if (!parsed.success) { + throw new ConfigError(formatIssues(configPath, parsed.error)); + } + return { config: parsed.data, configPath, warnings: [] }; +} + +const SCENARIO_EXTS = new Set([".yaml", ".yml", ".json"]); + +function isScenarioFile(name: string): boolean { + const dot = name.lastIndexOf("."); + if (dot < 0) { + return false; + } + return SCENARIO_EXTS.has(name.slice(dot).toLowerCase()); +} + +export function loadScenarios(cwd: string = process.cwd()): Scenario[] { + const dir = join(cwd, SCENARIOS_DIR); + if (!existsSync(dir)) { + return []; + } + const names = readdirSync(dir).filter(isScenarioFile).sort(); + const scenarios: Scenario[] = []; + const seen = new Map(); + for (const name of names) { + const file = join(dir, name); + const doc = parseDoc(file, readFileSync(file, "utf8")); + const parsed = ScenarioSchema.safeParse(doc); + if (!parsed.success) { + throw new ConfigError(formatIssues(file, parsed.error)); + } + const prev = seen.get(parsed.data.id); + if (prev !== undefined) { + throw new ConfigError( + `${file}: duplicate scenario id "${parsed.data.id}" (already defined in ${prev}).`, + ); + } + seen.set(parsed.data.id, file); + scenarios.push(parsed.data); + } + return scenarios; +} + +/** Load config + scenarios together. */ +export function loadAll(cwd: string = process.cwd()): LoadResult { + const { config, configPath, warnings } = loadConfig(cwd); + const scenarios = loadScenarios(cwd); + return { config, scenarios, configPath, warnings }; +} diff --git a/src/config/schema.ts b/src/config/schema.ts new file mode 100644 index 0000000..cd9c867 --- /dev/null +++ b/src/config/schema.ts @@ -0,0 +1,270 @@ +// Config + scenario schemas (zod v4). Every object is `.strict()` so typos +// fail validation instead of silently passing (SPEC 5.1, 5.2). +import { z } from "zod"; + +const VERSION_RE = /^\d+\.\d+\.\d+$/; +const SCENARIO_ID_RE = /^[a-z0-9][a-z0-9-]*$/; + +const versionSchema = z.union([ + z.literal("latest"), + z.string().regex(VERSION_RE, 'must be semver (x.y.z) or "latest"'), +]); + +const pluginConfigSchema = z + .object({ + path: z.string().min(1).default("."), + entry: z.string().min(1).optional(), + install: z.string().min(1).optional(), + build: z.string().min(1).optional(), + options: z.record(z.string(), z.unknown()).default({}), + // Absent means auto-detect (server entry present => server, else TUI). + ui: z.boolean().optional(), + }) + .strict(); + +const opencodeConfigSchema = z + .object({ + versions: z.array(versionSchema).min(1).default(["2.0.26"]), + }) + .strict(); + +const stringArray = (fallback: string[] = []) => + z.array(z.string()).default(fallback); + +const configExpectSchema = z + .object({ + tools: stringArray(), + commands: stringArray(), + agents: stringArray(), + skills: stringArray(), + mcp: stringArray(), + providers: stringArray(), + models: stringArray(), + }) + .strict(); + +const liveConfigSchema = z + .object({ + model: z.string().min(1).optional(), + keys: z.array(z.string().min(1)).min(1).default(["OPENCODE_API_KEY"]), + repeat: z.number().int().positive().default(3), + pass: z.number().int().positive().default(2), + budgetUSD: z.number().positive().default(1), + timeoutMs: z.number().int().positive().default(240_000), + }) + .strict(); + +const gateConfigSchema = z + .object({ + enabled: z.boolean().default(true), + window: z.number().int().positive().default(5), + threshold: z.number().min(0).max(1).default(0.9), + bootstrap: z.enum(["fail", "allow"]).default("fail"), + workflow: z.string().min(1).optional(), + }) + .strict(); + +export const ConfigSchema = z + .object({ + // prefault (not default): the fallback is parsed, so inner field + // defaults still apply when the whole block is missing. zod v4's + // .default() would return the fallback as-is. + plugin: pluginConfigSchema.prefault({}), + opencode: opencodeConfigSchema.prefault({}), + expect: configExpectSchema.prefault({}), + live: liveConfigSchema.prefault({}), + gate: gateConfigSchema.prefault({}), + logLevel: z.enum(["info", "all"]).default("info"), + }) + .strict() + .superRefine((val, ctx) => { + if (val.live.pass > val.live.repeat) { + ctx.addIssue({ + code: "custom", + path: ["live", "pass"], + message: `live.pass (${val.live.pass}) must be <= live.repeat (${val.live.repeat})`, + }); + } + }); + +export type Config = z.infer; + +// --- Scenario --- + +const countMatcherSchema = z + .object({ + eq: z.number().optional(), + gte: z.number().optional(), + lte: z.number().optional(), + }) + .strict(); + +export const MatcherSchema = z + .object({ + equals: z.unknown().optional(), + contains: z.unknown().optional(), + notContains: z.unknown().optional(), + matches: z.string().min(1).optional(), + notMatches: z.string().min(1).optional(), + includes: z.array(z.string()).min(1).optional(), + exists: z.boolean().optional(), + count: countMatcherSchema.optional(), + }) + .strict() + .superRefine((val, ctx) => { + if (Object.keys(val).length === 0) { + ctx.addIssue({ + code: "custom", + message: + "matcher must specify at least one of: equals, contains, notContains, matches, notMatches, includes, exists, count", + }); + } + }); + +export type Matcher = z.infer; +export type CountMatcher = z.infer; + +const toolCallSchema = z + .object({ + name: z.string().min(1), + arguments: z.record(z.string(), z.unknown()).default({}), + }) + .strict(); + +const stepSchema = z + .object({ + text: z.string().optional(), + toolCalls: z.array(toolCallSchema).optional(), + raw: z.boolean().optional(), + }) + .strict(); + +export type Step = z.infer; +export type ToolCall = z.infer; + +const toolExpectSchema = z + .object({ + name: z.string().min(1), + status: z.enum(["completed", "error"]).optional(), + input: z.record(z.string(), MatcherSchema).optional(), + output: MatcherSchema.optional(), + count: countMatcherSchema.optional(), + }) + .strict(); + +const hookExpectSchema = z + .object({ + hook: z.enum([ + "tool.execute.before", + "tool.execute.after", + "shell.create.before", + "session.context", + "session.prompt", + ]), + phase: z.enum(["pre", "post"]).default("post"), + tool: z.string().min(1).optional(), + input: z.record(z.string(), MatcherSchema).optional(), + env: z.record(z.string(), MatcherSchema).optional(), + system: MatcherSchema.optional(), + }) + .strict(); + +const requestExpectSchema = z + .object({ + nth: z.number().int().positive().default(1), + system: MatcherSchema.optional(), + tools: MatcherSchema.optional(), + }) + .strict(); + +const fileExpectSchema = z + .object({ + path: z.string().min(1), + exists: z.boolean().optional(), + contains: z.string().optional(), + notContains: z.string().optional(), + matches: z.string().min(1).optional(), + }) + .strict(); + +const scenarioExpectSchema = z + .object({ + exitCode: z.number().int().default(0), + noErrors: z.boolean().default(true), + tools: z.array(toolExpectSchema).optional(), + noTools: z.array(z.string().min(1)).optional(), + hooks: z.array(hookExpectSchema).optional(), + request: requestExpectSchema.optional(), + files: z.array(fileExpectSchema).optional(), + stderr: MatcherSchema.optional(), + text: MatcherSchema.optional(), + }) + .strict(); + +export type ScenarioExpect = z.infer; + +const judgeSchema = z + .object({ + rubric: z.string().min(1), + model: z.string().min(1).optional(), + }) + .strict(); + +export const ScenarioSchema = z + .object({ + id: z.string().regex(SCENARIO_ID_RE, "must match /^[a-z0-9][a-z0-9-]*$/"), + description: z.string().optional(), + mode: z.enum(["scripted", "live"]), + plugin: z + .object({ options: z.record(z.string(), z.unknown()).optional() }) + .strict() + .optional(), + workspace: z.record(z.string(), z.string()).optional(), + prompt: z.string().min(1), + script: z.array(stepSchema).optional(), + judgeReply: z.string().optional(), + timeoutMs: z.number().int().positive().default(120_000), + expect: scenarioExpectSchema.prefault({}), + judge: judgeSchema.optional(), + }) + .strict() + .superRefine((val, ctx) => { + const tag = `scenario "${val.id}"`; + if (val.mode === "scripted" && val.script === undefined) { + ctx.addIssue({ + code: "custom", + path: ["script"], + message: `${tag}: field "script" is required when mode is "scripted"`, + }); + } + if (val.mode === "live" && val.script !== undefined) { + ctx.addIssue({ + code: "custom", + path: ["script"], + message: `${tag}: field "script" is forbidden when mode is "live"`, + }); + } + if (val.mode === "live" && val.expect.request !== undefined) { + ctx.addIssue({ + code: "custom", + path: ["expect", "request"], + message: `${tag}: field "expect.request" is scripted-only and is not allowed when mode is "live"`, + }); + } + if (val.mode === "scripted" && val.judge !== undefined) { + ctx.addIssue({ + code: "custom", + path: ["judge"], + message: `${tag}: field "judge" is live-only and is not allowed when mode is "scripted"`, + }); + } + if (val.mode === "live" && val.judgeReply !== undefined) { + ctx.addIssue({ + code: "custom", + path: ["judgeReply"], + message: `${tag}: field "judgeReply" is scripted-only and is not allowed when mode is "live"`, + }); + } + }); + +export type Scenario = z.infer; diff --git a/test/unit/config-load.test.ts b/test/unit/config-load.test.ts new file mode 100644 index 0000000..39bc82a --- /dev/null +++ b/test/unit/config-load.test.ts @@ -0,0 +1,202 @@ +// M7: config/scenario file loading: precedence, duplicates, defaults, +// parse errors and fixture coverage. +import { describe, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, readdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { + ConfigError, + loadAll, + loadConfig, + loadScenarios, +} from "../../src/config/load.ts"; + +const REPO_ROOT = join(import.meta.dir, "..", ".."); +const FIXTURES = join(REPO_ROOT, "test", "fixtures", "plugins"); + +function mkTree(files: Record): string { + const root = mkdtempSync(join(tmpdir(), "opencode-ci-config-")); + for (const [rel, content] of Object.entries(files)) { + const abs = join(root, rel); + mkdirSync(dirname(abs), { recursive: true }); + writeFileSync(abs, content); + } + return root; +} + +const PASS_SCENARIO = `id: my-case +mode: scripted +prompt: Hello. +script: + - text: ok +expect: + exitCode: 0 +`; + +describe("loadConfig", () => { + test("YAML vs JSON precedence error names both files", () => { + const root = mkTree({ + ".opencode-ci/config.yaml": "plugin:\n path: .\n", + ".opencode-ci/config.json": "{}", + }); + let error: unknown; + try { + loadConfig(root); + } catch (e) { + error = e; + } + expect(error).toBeInstanceOf(ConfigError); + expect(String((error as Error).message)).toMatch(/config\.yaml/); + expect(String((error as Error).message)).toMatch(/config\.json/); + }); + + test("missing config yields defaults with a warning", () => { + const root = mkTree({}); + const { config, configPath, warnings } = loadConfig(root); + expect(configPath).toBeUndefined(); + expect(warnings.length).toBeGreaterThan(0); + expect(config.plugin.path).toBe("."); + expect(config.opencode.versions).toEqual(["2.0.26"]); + expect(config.logLevel).toBe("info"); + }); + + test("YAML parse error includes path and line", () => { + const root = mkTree({ + ".opencode-ci/config.yaml": "plugin:\n path: [unclosed\n bad indent\n", + }); + let error: unknown; + try { + loadConfig(root); + } catch (e) { + error = e; + } + expect(error).toBeInstanceOf(ConfigError); + const message = String((error as Error).message); + expect(message).toMatch(/config\.yaml/); + expect(message).toMatch(/line \d+/i); + }); + + test("invalid JSON config names the file", () => { + const root = mkTree({ ".opencode-ci/config.json": "{oops" }); + expect(() => loadConfig(root)).toThrow(ConfigError); + try { + loadConfig(root); + } catch (e) { + expect(String((e as Error).message)).toMatch(/config\.json/); + } + }); + + test("schema violation names the file and the field", () => { + const root = mkTree({ + ".opencode-ci/config.yaml": "expect:\n tool: [my_tool]\n", + }); + try { + loadConfig(root); + expect.unreachable(); + } catch (e) { + expect(e).toBeInstanceOf(ConfigError); + expect(String((e as Error).message)).toMatch(/config\.yaml/); + expect(String((e as Error).message)).toMatch(/expect/); + } + }); + + test("JSON config parses with the same schema", () => { + const root = mkTree({ + ".opencode-ci/config.json": JSON.stringify({ + plugin: { path: "packages/p" }, + logLevel: "all", + }), + }); + const { config, configPath } = loadConfig(root); + expect(configPath).toMatch(/config\.json$/); + expect(config.plugin.path).toBe("packages/p"); + expect(config.logLevel).toBe("all"); + }); +}); + +describe("loadScenarios", () => { + test("duplicate ids error names the id and both files", () => { + const root = mkTree({ + ".opencode-ci/scenarios/a.yaml": PASS_SCENARIO, + ".opencode-ci/scenarios/b.yaml": PASS_SCENARIO, + }); + try { + loadScenarios(root); + expect.unreachable(); + } catch (e) { + expect(e).toBeInstanceOf(ConfigError); + const message = String((e as Error).message); + expect(message).toMatch(/my-case/); + expect(message).toMatch(/a\.yaml/); + expect(message).toMatch(/b\.yaml/); + } + }); + + test("scenario schema violation names the file and the field", () => { + const root = mkTree({ + ".opencode-ci/scenarios/bad.yaml": `id: bad +mode: scripted +prompt: hi +script: + - text: ok +judge: { rubric: "live only" } +`, + }); + try { + loadScenarios(root); + expect.unreachable(); + } catch (e) { + expect(e).toBeInstanceOf(ConfigError); + expect(String((e as Error).message)).toMatch(/bad\.yaml/); + expect(String((e as Error).message)).toMatch(/judge/); + } + }); + + test("scenarios come back sorted by filename, yaml + yml + json", () => { + const root = mkTree({ + ".opencode-ci/scenarios/c.json": JSON.stringify({ + id: "c-case", + mode: "scripted", + prompt: "hi", + script: [{ text: "ok" }], + }), + ".opencode-ci/scenarios/a.yaml": PASS_SCENARIO.replace( + "my-case", + "a-case", + ), + ".opencode-ci/scenarios/b.yml": PASS_SCENARIO.replace( + "my-case", + "b-case", + ), + }); + const scenarios = loadScenarios(root); + expect(scenarios.map((s) => s.id)).toEqual(["a-case", "b-case", "c-case"]); + }); + + test("missing scenarios dir yields no scenarios", () => { + expect(loadScenarios(mkTree({}))).toEqual([]); + }); +}); + +describe("fixtures still parse", () => { + const names = readdirSync(FIXTURES).sort(); + test("every fixture plugin loads config + scenarios", () => { + expect(names.length).toBeGreaterThan(0); + for (const name of names) { + const { config, scenarios } = loadAll(join(FIXTURES, name)); + expect( + scenarios.length, + `${name} should define at least one scenario`, + ).toBeGreaterThan(0); + expect(config.plugin.path).toBe("."); + } + }); + + test("tool-basic pass scenario keeps its shape", () => { + const { scenarios } = loadAll(join(FIXTURES, "tool-basic")); + const pass = scenarios.find((s) => s.id === "tool-basic-pass"); + expect(pass?.mode).toBe("scripted"); + expect(pass?.script?.length).toBe(2); + expect(pass?.expect.tools?.[0]?.name).toBe("echo_tool"); + }); +}); diff --git a/test/unit/config-schema.test.ts b/test/unit/config-schema.test.ts new file mode 100644 index 0000000..ff2bdaf --- /dev/null +++ b/test/unit/config-schema.test.ts @@ -0,0 +1,251 @@ +// M7: schema defaults, strictness and cross-field rules. +import { describe, expect, test } from "bun:test"; +import { + ConfigSchema, + MatcherSchema, + ScenarioSchema, +} from "../../src/config/schema.ts"; + +const scriptedBase = { + id: "ok-case", + mode: "scripted", + prompt: "hi", + script: [{ text: "done" }], + expect: { exitCode: 0 }, +}; + +describe("ConfigSchema defaults", () => { + test("empty object yields every default", () => { + const config = ConfigSchema.parse({}); + expect(config.plugin.path).toBe("."); + expect(config.plugin.options).toEqual({}); + expect(config.plugin.ui).toBeUndefined(); + expect(config.opencode.versions).toEqual(["2.0.26"]); + expect(config.expect).toEqual({ + tools: [], + commands: [], + agents: [], + skills: [], + mcp: [], + providers: [], + models: [], + }); + expect(config.live.keys).toEqual(["OPENCODE_API_KEY"]); + expect(config.live.repeat).toBe(3); + expect(config.live.pass).toBe(2); + expect(config.live.budgetUSD).toBe(1); + expect(config.live.timeoutMs).toBe(240_000); + expect(config.gate.enabled).toBe(true); + expect(config.gate.window).toBe(5); + expect(config.gate.threshold).toBe(0.9); + expect(config.gate.bootstrap).toBe("fail"); + expect(config.gate.workflow).toBeUndefined(); + expect(config.logLevel).toBe("info"); + }); + + test("partial blocks keep their defaults", () => { + const config = ConfigSchema.parse({ live: { model: "opencode/x" } }); + expect(config.live.model).toBe("opencode/x"); + expect(config.live.repeat).toBe(3); + expect(config.live.pass).toBe(2); + }); + + test("versions accepts semver and latest, non-empty only", () => { + expect( + ConfigSchema.parse({ opencode: { versions: ["2.0.26", "latest"] } }) + .opencode.versions, + ).toEqual(["2.0.26", "latest"]); + expect(ConfigSchema.safeParse({ opencode: { versions: [] } }).success).toBe( + false, + ); + expect( + ConfigSchema.safeParse({ opencode: { versions: ["v2"] } }).success, + ).toBe(false); + expect( + ConfigSchema.safeParse({ opencode: { versions: ["2.0"] } }).success, + ).toBe(false); + }); +}); + +describe("strictness: unknown keys fail", () => { + test("expect.tool typo fails", () => { + const parsed = ConfigSchema.safeParse({ expect: { tool: ["x"] } }); + expect(parsed.success).toBe(false); + }); + + test("unknown top-level config key fails", () => { + expect(ConfigSchema.safeParse({ bogus: 1 }).success).toBe(false); + }); + + test("unknown plugin key fails", () => { + expect( + ConfigSchema.safeParse({ plugin: { path: ".", bogus: 1 } }).success, + ).toBe(false); + }); + + test("unknown scenario key fails", () => { + expect( + ScenarioSchema.safeParse({ ...scriptedBase, bogus: 1 }).success, + ).toBe(false); + }); + + test("unknown matcher key fails", () => { + expect(MatcherSchema.safeParse({ containz: "x" }).success).toBe(false); + }); + + test("unknown expect key fails", () => { + expect( + ScenarioSchema.safeParse({ + ...scriptedBase, + expect: { exitCode: 0, bogus: 1 }, + }).success, + ).toBe(false); + }); +}); + +describe("cross-field rules", () => { + test("request is forbidden in live mode", () => { + const parsed = ScenarioSchema.safeParse({ + id: "live-x", + mode: "live", + prompt: "hi", + expect: { request: { system: { contains: "x" } } }, + }); + expect(parsed.success).toBe(false); + if (!parsed.success) { + expect(parsed.error.issues[0]?.message).toMatch(/expect\.request.*live/); + } + }); + + test("script is required in scripted mode", () => { + const { script: _dropped, ...noScript } = scriptedBase; + const parsed = ScenarioSchema.safeParse(noScript); + expect(parsed.success).toBe(false); + if (!parsed.success) { + expect(parsed.error.issues[0]?.message).toMatch(/"script".*required/); + } + }); + + test("script is forbidden in live mode", () => { + const parsed = ScenarioSchema.safeParse({ + id: "live-x", + mode: "live", + prompt: "hi", + script: [{ text: "x" }], + }); + expect(parsed.success).toBe(false); + if (!parsed.success) { + expect(parsed.error.issues[0]?.message).toMatch(/"script".*forbidden/); + } + }); + + test("judge is live-only", () => { + const parsed = ScenarioSchema.safeParse({ + ...scriptedBase, + judge: { rubric: "good?" }, + }); + expect(parsed.success).toBe(false); + if (!parsed.success) { + expect(parsed.error.issues[0]?.message).toMatch(/"judge".*live-only/); + } + }); + + test("judgeReply is scripted-only", () => { + const parsed = ScenarioSchema.safeParse({ + id: "live-x", + mode: "live", + prompt: "hi", + judgeReply: '{"pass":true}', + }); + expect(parsed.success).toBe(false); + if (!parsed.success) { + expect(parsed.error.issues[0]?.message).toMatch(/"judgeReply"/); + } + }); + + test("live judge scenario validates", () => { + const parsed = ScenarioSchema.safeParse({ + id: "live-x", + mode: "live", + prompt: "hi", + judge: { rubric: "The reply is helpful." }, + }); + expect(parsed.success).toBe(true); + }); + + test("pass must be <= repeat", () => { + const bad = ConfigSchema.safeParse({ + live: { repeat: 3, pass: 4 }, + }); + expect(bad.success).toBe(false); + if (!bad.success) { + expect(bad.error.issues[0]?.message).toMatch(/live\.pass.*live\.repeat/); + } + expect( + ConfigSchema.safeParse({ live: { repeat: 2, pass: 2 } }).success, + ).toBe(true); + }); +}); + +describe("matchers", () => { + test("leading (?i) is accepted as a matches string", () => { + expect(MatcherSchema.safeParse({ matches: "(?i)unhandled" }).success).toBe( + true, + ); + expect( + MatcherSchema.safeParse({ notMatches: "(?i)unhandled" }).success, + ).toBe(true); + }); + + test("empty matcher fails", () => { + const parsed = MatcherSchema.safeParse({}); + expect(parsed.success).toBe(false); + }); + + test("every matcher key validates", () => { + expect( + MatcherSchema.safeParse({ + equals: 1, + contains: "a", + notContains: "b", + matches: "c", + notMatches: "d", + includes: ["e"], + exists: true, + count: { gte: 1, lte: 2, eq: 1 }, + }).success, + ).toBe(true); + }); +}); + +describe("scenario ids and shapes", () => { + test("bad ids fail", () => { + for (const id of ["Bad", "-x", "a_b", "", "A"]) { + expect(ScenarioSchema.safeParse({ ...scriptedBase, id }).success).toBe( + false, + ); + } + expect( + ScenarioSchema.safeParse({ ...scriptedBase, id: "a-b-9" }).success, + ).toBe(true); + }); + + test("workspace record and plugin options validate", () => { + const parsed = ScenarioSchema.safeParse({ + ...scriptedBase, + plugin: { options: { mode: "strict" } }, + workspace: { ".env": "API_KEY=sk-test\n" }, + }); + expect(parsed.success).toBe(true); + }); + + test("scenario timeoutMs defaults to 120000", () => { + const parsed = ScenarioSchema.safeParse(scriptedBase); + expect(parsed.success).toBe(true); + if (parsed.success) { + expect(parsed.data.timeoutMs).toBe(120_000); + expect(parsed.data.expect.exitCode).toBe(0); + expect(parsed.data.expect.noErrors).toBe(true); + } + }); +}); diff --git a/test/unit/schema-sync.test.ts b/test/unit/schema-sync.test.ts new file mode 100644 index 0000000..58704e4 --- /dev/null +++ b/test/unit/schema-sync.test.ts @@ -0,0 +1,21 @@ +// M7: the committed JSON schemas must match what the zod schemas generate, +// so schema/config.json + schema/scenario.json can never drift. +import { expect, test } from "bun:test"; +import { mkdtempSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { writeSchemas } from "../../scripts/gen-schema.ts"; + +const REPO_ROOT = join(import.meta.dir, "..", ".."); + +test("committed schemas are in sync with the zod schemas", () => { + const dir = mkdtempSync(join(tmpdir(), "opencode-ci-schema-")); + writeSchemas(dir); + for (const file of ["config.json", "scenario.json"]) { + const fresh = readFileSync(join(dir, file), "utf8"); + const committed = readFileSync(join(REPO_ROOT, "schema", file), "utf8"); + expect(fresh, `${file} is stale: run \`bun scripts/gen-schema.ts\``).toBe( + committed, + ); + } +});