test: add regression coverage and core benchmarks

.github/workflows/ci.yml

@@ -0,0 +1,22 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  verify:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: pnpm/action-setup@v4
+        with:
+          version: 10.28.2
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+          cache: pnpm
+      - run: sudo apt-get update && sudo apt-get install -y ffmpeg
+      - run: pnpm install --frozen-lockfile
+      - run: pnpm verify

README.md

@@ -214,7 +214,9 @@ pnpm build        # tsc per package (core → tools → cli/mcp)
 pnpm test         # vitest
 pnpm test:video   # mock provider round-trip → real playable MP4 → ffprobe
 pnpm test:video:e2e # build + real HyperFrames compose→MP4 + built CLI/MCP smoke
+pnpm benchmark    # deterministic core correctness + latency/throughput benchmark
 pnpm typecheck
+pnpm verify       # build + typecheck + unit tests + benchmark + deterministic video test
 ```
 
 `test:video` is deterministic and never spends provider credits: a local fake Seedance endpoint
@@ -224,6 +226,11 @@ returns a real H.264 fixture, then OVS downloads it and verifies the result with
 commands fail with an actionable error when required video runtimes are missing; the ordinary
 test suite may skip runtime-heavy cases on machines without ffmpeg or a browser.
 
+`pnpm benchmark` is zero-key and deterministic. It builds the core package, verifies benchmark
+fixtures, then measures gate transitions, EDL validation/delivery summaries, composition-manifest
+validation, and narration estimation. Each suite carries a deliberately conservative throughput
+floor so large regressions fail while normal CI and developer-machine variance does not.
+
 ## License
 
 MIT — see [`LICENSE`](./LICENSE). Rendering uses the Apache-2.0 licensed

package.json

@@ -10,12 +10,14 @@
     "node": ">=22"
   },
   "scripts": {
+    "benchmark": "pnpm --filter @orkas/video-studio-core build && node scripts/benchmark-core.mjs",
     "build": "pnpm -r run build",
     "test": "vitest run",
     "test:video": "node scripts/test-video.mjs",
     "test:video:e2e": "node scripts/test-video.mjs --e2e",
     "test:watch": "vitest",
-    "typecheck": "pnpm -r run typecheck"
+    "typecheck": "pnpm -r run typecheck",
+    "verify": "pnpm build && pnpm typecheck && pnpm test && pnpm benchmark && pnpm test:video"
   },
   "devDependencies": {
     "@types/node": "^22.10.0",

packages/tools/test/composition-scaffold.test.ts

@@ -81,16 +81,17 @@ describe('manifest-owned HyperFrames scaffold', () => {
       const value = manifest();
       const scenes = value.scenes as Array<Record<string, unknown>>;
       scenes[0].narration_text = 'Narrated opening.';
+      const narrationIntent = {
+        route_ref: 'openai-compatible',
+        voice_ref: 'nova',
+        display_name: 'Nova',
+        language: 'en-US',
+        speed: 1,
+      };
       value.audio = {
         owner: 'none',
         tracks: [],
-        narration_intent: {
-          route_ref: 'openai-compatible',
-          voice_ref: 'nova',
-          display_name: 'Nova',
-          language: 'en-US',
-          speed: 1,
-        },
+        narration_intent: narrationIntent,
       };
       writeFileSync(join(project, 'composition-manifest.json'), JSON.stringify(value), 'utf8');
       await prepareComposition(project);
@@ -114,6 +115,11 @@ describe('manifest-owned HyperFrames scaffold', () => {
       writeFileSync(join(project, 'composition-manifest.json'), JSON.stringify(value), 'utf8');
       await reconcileComposition(project);
       await expect(audioQa()).resolves.toMatchObject({ ok: true, narration_required: false });
+
+      value.audio = { owner: 'none', tracks: [], narration_intent: narrationIntent };
+      writeFileSync(join(project, 'composition-manifest.json'), JSON.stringify(value), 'utf8');
+      await reconcileComposition(project);
+      await expect(audioQa()).resolves.toMatchObject({ ok: false, narration_required: true });
     } finally {
       rmSync(project, { recursive: true, force: true });
     }

packages/tools/test/render-draft.test.ts

@@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest';
 import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
-import { check, draft, render } from '../src/render/render';
+import { check, draft, render, snapshot } from '../src/render/render';
 import { isEnvironmentalDraftFailure } from '../src/render/composition-qa';
 
 function tmpProject(name: string): { root: string; composition: string; output: string; report: string } {
@@ -104,6 +104,43 @@ describe('design contract preflight', () => {
 });
 
 describe('composition draft gate', () => {
+  it('blocks snapshot from current narration facts before invoking HyperFrames', async () => {
+    const p = tmpProject('snapshot-narration-facts');
+    try {
+      writeHtml(p.composition, '<div>Launch</div>');
+      writeContract(p.composition);
+      writeSceneMap(p.composition, {
+        audio: { owner: 'none', tracks: [] },
+        scenes: [{ id: 's1', start: 0, duration: 10, headline: 'Launch', narration_text: 'Narrated opening.' }],
+      });
+
+      await expect(snapshot({ project: p.composition, output: join(p.root, 'preview.png') }))
+        .rejects.toThrow(/NARRATION_REQUIRED_BUT_NOT_MATERIALIZED/);
+      expect(existsSync(join(p.root, 'preview.png'))).toBe(false);
+    } finally {
+      rmSync(p.root, { recursive: true, force: true });
+    }
+  });
+
+  it('blocks draft from current narration facts before invoking HyperFrames', async () => {
+    const p = tmpProject('draft-narration-facts');
+    try {
+      writeHtml(p.composition, '<div>Launch</div>');
+      writeContract(p.composition);
+      writeSceneMap(p.composition, {
+        audio: { owner: 'none', tracks: [] },
+        scenes: [{ id: 's1', start: 0, duration: 10, headline: 'Launch', narration_text: 'Narrated opening.' }],
+      });
+
+      await expect(draft({ project: p.composition, output: p.output, reportPath: p.report }))
+        .resolves.toMatchObject({ ok: false, errorCode: 'E_AUDIO_TIMING_BLOCKED' });
+      expect(JSON.stringify(JSON.parse(readFileSync(p.report, 'utf8')))).toContain('NARRATION_REQUIRED_BUT_NOT_MATERIALIZED');
+      expect(existsSync(p.output)).toBe(false);
+    } finally {
+      rmSync(p.root, { recursive: true, force: true });
+    }
+  });
+
   it('blocks remote runtime resources before rendering', async () => {
     const p = tmpProject('remote-resource');
     try {

scripts/benchmark-core.mjs

@@ -0,0 +1,186 @@
+#!/usr/bin/env node
+import { performance } from 'node:perf_hooks';
+import {
+  assessDelivery,
+  estimateNarrationDuration,
+  resolveGateTransition,
+  summarizeEdl,
+  validateCompositionManifest,
+  validateEdl,
+} from '../packages/core/dist/index.js';
+
+const jsonOnly = process.argv.includes('--json');
+const multiplierRaw = Number(process.env.OVS_BENCH_MULTIPLIER || 1);
+const multiplier = Number.isFinite(multiplierRaw) && multiplierRaw > 0 ? multiplierRaw : 1;
+
+function assert(condition, message) {
+  if (!condition) throw new Error(`benchmark contract failed: ${message}`);
+}
+
+function percentile(values, ratio) {
+  const ordered = [...values].sort((a, b) => a - b);
+  return ordered[Math.min(ordered.length - 1, Math.floor(ordered.length * ratio))];
+}
+
+function round(value, digits = 2) {
+  const scale = 10 ** digits;
+  return Math.round(value * scale) / scale;
+}
+
+function runCase({ name, iterations, minOpsPerSecond, execute }) {
+  const count = Math.max(1, Math.round(iterations * multiplier));
+  let checksum = 0;
+  for (let index = 0; index < Math.min(500, count); index += 1) checksum ^= execute(index);
+
+  const samples = [];
+  for (let sample = 0; sample < 7; sample += 1) {
+    const started = performance.now();
+    for (let index = 0; index < count; index += 1) checksum ^= execute(index + sample);
+    samples.push(performance.now() - started);
+  }
+  const medianMs = percentile(samples, 0.5);
+  const p95Ms = percentile(samples, 0.95);
+  const opsPerSecond = count / (medianMs / 1000);
+  assert(opsPerSecond >= minOpsPerSecond, `${name} throughput ${round(opsPerSecond)} ops/s is below ${minOpsPerSecond}`);
+  return {
+    name,
+    iterations_per_sample: count,
+    samples: samples.length,
+    median_ms: round(medianMs, 3),
+    p95_ms: round(p95Ms, 3),
+    median_us_per_op: round((medianMs * 1000) / count, 3),
+    ops_per_second: Math.round(opsPerSecond),
+    minimum_ops_per_second: minOpsPerSecond,
+    checksum,
+    passed: true,
+  };
+}
+
+function segment(id, order, source = 'compose') {
+  const spec = source === 'edit'
+    ? { input_id: `clip-${id}`, in_sec: 0, out_sec: 5 }
+    : source === 'provided'
+      ? { asset_id: `asset-${id}`, kind: 'video' }
+      : { kind: 'motion-card' };
+  return { id, order, role: order === 0 ? 'hook' : 'body', layer: 'primary', source, target_sec: 5, spec };
+}
+
+const edl = {
+  aspect: '16:9',
+  total_target_sec: 60,
+  language: 'en',
+  delivery_promise: { type: 'hybrid', source_required: true, motion_min_ratio: 0.3 },
+  style_kit: { palette: ['#081018', '#f3f0e8', '#ffb000'], fonts: ['Inter'] },
+  segments: Array.from({ length: 12 }, (_, index) => segment(`s${index + 1}`, index, index % 3 === 0 ? 'edit' : 'compose')),
+  tracks: {},
+};
+
+const manifest = {
+  schema_version: 2,
+  composition: { id: 'main', width: 1920, height: 1080, duration: 60, target_duration: 60, fps: 30, language: 'en' },
+  scenes: Array.from({ length: 12 }, (_, index) => ({
+    id: `s${index + 1}`,
+    start: index * 5,
+    duration: 5,
+    approved_copy: [`Scene ${index + 1}`],
+    narration_refs: [],
+    source_shots: [`shot-${index + 1}`],
+    roles: [index === 0 ? 'hook' : index === 11 ? 'payoff' : 'body'],
+  })),
+  audio: { owner: 'none', tracks: [] },
+  art_direction: { aesthetic: { signature_device: 'timeline ribbon' } },
+};
+
+const transitions = [
+  [{ line: 'compose', artifact: 'composition', gate: 'preview', decision: 'revise', scope: 'visual_only', recovery: 'available' }, 'edit_and_restart_visual_qa'],
+  [{ line: 'compose', artifact: 'composition', gate: 'gate_d', decision: 'revise', scope: 'visual_only', recovery: 'not_available' }, 'edit_current_cycle'],
+  [{ line: 'compose', artifact: 'composition', gate: 'preview', decision: 'approve', scope: 'none', recovery: 'not_available' }, 'render_approved_preview'],
+  [{ line: 'edit', artifact: 'production', approvalStatus: 'approved', artifactState: 'unchanged', recovery: 'not_available' }, 'continue_from_existing_approval'],
+];
+
+assert(validateEdl(edl).ok, 'EDL fixture must be valid');
+assert(assessDelivery(edl).verdict === 'pass', 'delivery fixture must pass');
+assert(summarizeEdl(edl).split('\n').filter((line) => /^\s+\d+\./.test(line)).length === 12, 'EDL summary must include every primary segment');
+assert(validateCompositionManifest(manifest).ok, 'composition manifest fixture must be valid');
+for (const [input, expected] of transitions) {
+  assert(resolveGateTransition(input).next_action === expected, `gate transition must resolve ${expected}`);
+}
+
+const narrationSamples = [
+  'A concise narration line with a measured delivery window.',
+  '2017年，Transformer 与 GPT-4 改变了 AI。下一步，是 Agent。',
+  'Numbers such as 16:9 and 1920 by 1080 need conservative timing.',
+];
+
+const suites = [
+  runCase({
+    name: 'gate-transition',
+    iterations: 50_000,
+    minOpsPerSecond: 100_000,
+    execute(index) {
+      const [input, expected] = transitions[index % transitions.length];
+      const result = resolveGateTransition(input);
+      return result.next_action === expected ? result.allowed_ops.length + 1 : 0;
+    },
+  }),
+  runCase({
+    name: 'edl-validation',
+    iterations: 10_000,
+    minOpsPerSecond: 20_000,
+    execute() {
+      const result = validateEdl(edl);
+      return result.ok ? result.errors.length + result.warnings.length + 1 : 0;
+    },
+  }),
+  runCase({
+    name: 'edl-delivery-and-summary',
+    iterations: 20_000,
+    minOpsPerSecond: 10_000,
+    execute() {
+      const assessment = assessDelivery(edl);
+      return assessment.verdict === 'pass' ? summarizeEdl(edl).length : 0;
+    },
+  }),
+  runCase({
+    name: 'composition-manifest-validation',
+    iterations: 10_000,
+    minOpsPerSecond: 20_000,
+    execute() {
+      const result = validateCompositionManifest(manifest);
+      return result.ok ? (result.data?.scenes.length || 0) : 0;
+    },
+  }),
+  runCase({
+    name: 'narration-estimation',
+    iterations: 50_000,
+    minOpsPerSecond: 50_000,
+    execute(index) {
+      const result = estimateNarrationDuration(narrationSamples[index % narrationSamples.length], 1);
+      return Math.round(result.estimatedSec * 100);
+    },
+  }),
+];
+
+const report = {
+  schema_version: 1,
+  benchmark: 'orkas-video-studio-core',
+  node: process.version,
+  platform: `${process.platform}-${process.arch}`,
+  multiplier,
+  suites,
+  aggregate: {
+    passed: suites.every((suite) => suite.passed),
+    suite_count: suites.length,
+    total_timed_operations: suites.reduce((total, suite) => total + suite.iterations_per_sample * suite.samples, 0),
+  },
+};
+
+if (jsonOnly) {
+  process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
+} else {
+  process.stdout.write(`Core benchmark (${report.node}, ${report.platform})\n`);
+  for (const suite of suites) {
+    process.stdout.write(`${suite.name.padEnd(34)} ${String(suite.ops_per_second).padStart(9)} ops/s  p50=${String(suite.median_us_per_op).padStart(7)} us/op  p95=${suite.p95_ms} ms\n`);
+  }
+  process.stdout.write(`PASS ${report.aggregate.suite_count} suites, ${report.aggregate.total_timed_operations} timed operations\n`);
+}