diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..fd77cd6 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,22 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + +jobs: + verify: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + with: + version: 10.28.2 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: pnpm + - run: sudo apt-get update && sudo apt-get install -y ffmpeg + - run: pnpm install --frozen-lockfile + - run: pnpm verify diff --git a/README.md b/README.md index d02e8c1..ca7af7e 100644 --- a/README.md +++ b/README.md @@ -214,7 +214,9 @@ pnpm build # tsc per package (core → tools → cli/mcp) pnpm test # vitest pnpm test:video # mock provider round-trip → real playable MP4 → ffprobe pnpm test:video:e2e # build + real HyperFrames compose→MP4 + built CLI/MCP smoke +pnpm benchmark # deterministic core correctness + latency/throughput benchmark pnpm typecheck +pnpm verify # build + typecheck + unit tests + benchmark + deterministic video test ``` `test:video` is deterministic and never spends provider credits: a local fake Seedance endpoint @@ -224,6 +226,11 @@ returns a real H.264 fixture, then OVS downloads it and verifies the result with commands fail with an actionable error when required video runtimes are missing; the ordinary test suite may skip runtime-heavy cases on machines without ffmpeg or a browser. +`pnpm benchmark` is zero-key and deterministic. It builds the core package, verifies benchmark +fixtures, then measures gate transitions, EDL validation/delivery summaries, composition-manifest +validation, and narration estimation. Each suite carries a deliberately conservative throughput +floor so large regressions fail while normal CI and developer-machine variance does not. + ## License MIT — see [`LICENSE`](./LICENSE). Rendering uses the Apache-2.0 licensed diff --git a/package.json b/package.json index 43b9d34..3afdbe2 100644 --- a/package.json +++ b/package.json @@ -10,12 +10,14 @@ "node": ">=22" }, "scripts": { + "benchmark": "pnpm --filter @orkas/video-studio-core build && node scripts/benchmark-core.mjs", "build": "pnpm -r run build", "test": "vitest run", "test:video": "node scripts/test-video.mjs", "test:video:e2e": "node scripts/test-video.mjs --e2e", "test:watch": "vitest", - "typecheck": "pnpm -r run typecheck" + "typecheck": "pnpm -r run typecheck", + "verify": "pnpm build && pnpm typecheck && pnpm test && pnpm benchmark && pnpm test:video" }, "devDependencies": { "@types/node": "^22.10.0", diff --git a/packages/tools/test/composition-scaffold.test.ts b/packages/tools/test/composition-scaffold.test.ts index ac5862b..23a5a8a 100644 --- a/packages/tools/test/composition-scaffold.test.ts +++ b/packages/tools/test/composition-scaffold.test.ts @@ -81,16 +81,17 @@ describe('manifest-owned HyperFrames scaffold', () => { const value = manifest(); const scenes = value.scenes as Array>; scenes[0].narration_text = 'Narrated opening.'; + const narrationIntent = { + route_ref: 'openai-compatible', + voice_ref: 'nova', + display_name: 'Nova', + language: 'en-US', + speed: 1, + }; value.audio = { owner: 'none', tracks: [], - narration_intent: { - route_ref: 'openai-compatible', - voice_ref: 'nova', - display_name: 'Nova', - language: 'en-US', - speed: 1, - }, + narration_intent: narrationIntent, }; writeFileSync(join(project, 'composition-manifest.json'), JSON.stringify(value), 'utf8'); await prepareComposition(project); @@ -114,6 +115,11 @@ describe('manifest-owned HyperFrames scaffold', () => { writeFileSync(join(project, 'composition-manifest.json'), JSON.stringify(value), 'utf8'); await reconcileComposition(project); await expect(audioQa()).resolves.toMatchObject({ ok: true, narration_required: false }); + + value.audio = { owner: 'none', tracks: [], narration_intent: narrationIntent }; + writeFileSync(join(project, 'composition-manifest.json'), JSON.stringify(value), 'utf8'); + await reconcileComposition(project); + await expect(audioQa()).resolves.toMatchObject({ ok: false, narration_required: true }); } finally { rmSync(project, { recursive: true, force: true }); } diff --git a/packages/tools/test/render-draft.test.ts b/packages/tools/test/render-draft.test.ts index 95f2565..d116ad1 100644 --- a/packages/tools/test/render-draft.test.ts +++ b/packages/tools/test/render-draft.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest'; import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { check, draft, render } from '../src/render/render'; +import { check, draft, render, snapshot } from '../src/render/render'; import { isEnvironmentalDraftFailure } from '../src/render/composition-qa'; function tmpProject(name: string): { root: string; composition: string; output: string; report: string } { @@ -104,6 +104,43 @@ describe('design contract preflight', () => { }); describe('composition draft gate', () => { + it('blocks snapshot from current narration facts before invoking HyperFrames', async () => { + const p = tmpProject('snapshot-narration-facts'); + try { + writeHtml(p.composition, '
Launch
'); + writeContract(p.composition); + writeSceneMap(p.composition, { + audio: { owner: 'none', tracks: [] }, + scenes: [{ id: 's1', start: 0, duration: 10, headline: 'Launch', narration_text: 'Narrated opening.' }], + }); + + await expect(snapshot({ project: p.composition, output: join(p.root, 'preview.png') })) + .rejects.toThrow(/NARRATION_REQUIRED_BUT_NOT_MATERIALIZED/); + expect(existsSync(join(p.root, 'preview.png'))).toBe(false); + } finally { + rmSync(p.root, { recursive: true, force: true }); + } + }); + + it('blocks draft from current narration facts before invoking HyperFrames', async () => { + const p = tmpProject('draft-narration-facts'); + try { + writeHtml(p.composition, '
Launch
'); + writeContract(p.composition); + writeSceneMap(p.composition, { + audio: { owner: 'none', tracks: [] }, + scenes: [{ id: 's1', start: 0, duration: 10, headline: 'Launch', narration_text: 'Narrated opening.' }], + }); + + await expect(draft({ project: p.composition, output: p.output, reportPath: p.report })) + .resolves.toMatchObject({ ok: false, errorCode: 'E_AUDIO_TIMING_BLOCKED' }); + expect(JSON.stringify(JSON.parse(readFileSync(p.report, 'utf8')))).toContain('NARRATION_REQUIRED_BUT_NOT_MATERIALIZED'); + expect(existsSync(p.output)).toBe(false); + } finally { + rmSync(p.root, { recursive: true, force: true }); + } + }); + it('blocks remote runtime resources before rendering', async () => { const p = tmpProject('remote-resource'); try { diff --git a/scripts/benchmark-core.mjs b/scripts/benchmark-core.mjs new file mode 100644 index 0000000..22df6bb --- /dev/null +++ b/scripts/benchmark-core.mjs @@ -0,0 +1,186 @@ +#!/usr/bin/env node +import { performance } from 'node:perf_hooks'; +import { + assessDelivery, + estimateNarrationDuration, + resolveGateTransition, + summarizeEdl, + validateCompositionManifest, + validateEdl, +} from '../packages/core/dist/index.js'; + +const jsonOnly = process.argv.includes('--json'); +const multiplierRaw = Number(process.env.OVS_BENCH_MULTIPLIER || 1); +const multiplier = Number.isFinite(multiplierRaw) && multiplierRaw > 0 ? multiplierRaw : 1; + +function assert(condition, message) { + if (!condition) throw new Error(`benchmark contract failed: ${message}`); +} + +function percentile(values, ratio) { + const ordered = [...values].sort((a, b) => a - b); + return ordered[Math.min(ordered.length - 1, Math.floor(ordered.length * ratio))]; +} + +function round(value, digits = 2) { + const scale = 10 ** digits; + return Math.round(value * scale) / scale; +} + +function runCase({ name, iterations, minOpsPerSecond, execute }) { + const count = Math.max(1, Math.round(iterations * multiplier)); + let checksum = 0; + for (let index = 0; index < Math.min(500, count); index += 1) checksum ^= execute(index); + + const samples = []; + for (let sample = 0; sample < 7; sample += 1) { + const started = performance.now(); + for (let index = 0; index < count; index += 1) checksum ^= execute(index + sample); + samples.push(performance.now() - started); + } + const medianMs = percentile(samples, 0.5); + const p95Ms = percentile(samples, 0.95); + const opsPerSecond = count / (medianMs / 1000); + assert(opsPerSecond >= minOpsPerSecond, `${name} throughput ${round(opsPerSecond)} ops/s is below ${minOpsPerSecond}`); + return { + name, + iterations_per_sample: count, + samples: samples.length, + median_ms: round(medianMs, 3), + p95_ms: round(p95Ms, 3), + median_us_per_op: round((medianMs * 1000) / count, 3), + ops_per_second: Math.round(opsPerSecond), + minimum_ops_per_second: minOpsPerSecond, + checksum, + passed: true, + }; +} + +function segment(id, order, source = 'compose') { + const spec = source === 'edit' + ? { input_id: `clip-${id}`, in_sec: 0, out_sec: 5 } + : source === 'provided' + ? { asset_id: `asset-${id}`, kind: 'video' } + : { kind: 'motion-card' }; + return { id, order, role: order === 0 ? 'hook' : 'body', layer: 'primary', source, target_sec: 5, spec }; +} + +const edl = { + aspect: '16:9', + total_target_sec: 60, + language: 'en', + delivery_promise: { type: 'hybrid', source_required: true, motion_min_ratio: 0.3 }, + style_kit: { palette: ['#081018', '#f3f0e8', '#ffb000'], fonts: ['Inter'] }, + segments: Array.from({ length: 12 }, (_, index) => segment(`s${index + 1}`, index, index % 3 === 0 ? 'edit' : 'compose')), + tracks: {}, +}; + +const manifest = { + schema_version: 2, + composition: { id: 'main', width: 1920, height: 1080, duration: 60, target_duration: 60, fps: 30, language: 'en' }, + scenes: Array.from({ length: 12 }, (_, index) => ({ + id: `s${index + 1}`, + start: index * 5, + duration: 5, + approved_copy: [`Scene ${index + 1}`], + narration_refs: [], + source_shots: [`shot-${index + 1}`], + roles: [index === 0 ? 'hook' : index === 11 ? 'payoff' : 'body'], + })), + audio: { owner: 'none', tracks: [] }, + art_direction: { aesthetic: { signature_device: 'timeline ribbon' } }, +}; + +const transitions = [ + [{ line: 'compose', artifact: 'composition', gate: 'preview', decision: 'revise', scope: 'visual_only', recovery: 'available' }, 'edit_and_restart_visual_qa'], + [{ line: 'compose', artifact: 'composition', gate: 'gate_d', decision: 'revise', scope: 'visual_only', recovery: 'not_available' }, 'edit_current_cycle'], + [{ line: 'compose', artifact: 'composition', gate: 'preview', decision: 'approve', scope: 'none', recovery: 'not_available' }, 'render_approved_preview'], + [{ line: 'edit', artifact: 'production', approvalStatus: 'approved', artifactState: 'unchanged', recovery: 'not_available' }, 'continue_from_existing_approval'], +]; + +assert(validateEdl(edl).ok, 'EDL fixture must be valid'); +assert(assessDelivery(edl).verdict === 'pass', 'delivery fixture must pass'); +assert(summarizeEdl(edl).split('\n').filter((line) => /^\s+\d+\./.test(line)).length === 12, 'EDL summary must include every primary segment'); +assert(validateCompositionManifest(manifest).ok, 'composition manifest fixture must be valid'); +for (const [input, expected] of transitions) { + assert(resolveGateTransition(input).next_action === expected, `gate transition must resolve ${expected}`); +} + +const narrationSamples = [ + 'A concise narration line with a measured delivery window.', + '2017年,Transformer 与 GPT-4 改变了 AI。下一步,是 Agent。', + 'Numbers such as 16:9 and 1920 by 1080 need conservative timing.', +]; + +const suites = [ + runCase({ + name: 'gate-transition', + iterations: 50_000, + minOpsPerSecond: 100_000, + execute(index) { + const [input, expected] = transitions[index % transitions.length]; + const result = resolveGateTransition(input); + return result.next_action === expected ? result.allowed_ops.length + 1 : 0; + }, + }), + runCase({ + name: 'edl-validation', + iterations: 10_000, + minOpsPerSecond: 20_000, + execute() { + const result = validateEdl(edl); + return result.ok ? result.errors.length + result.warnings.length + 1 : 0; + }, + }), + runCase({ + name: 'edl-delivery-and-summary', + iterations: 20_000, + minOpsPerSecond: 10_000, + execute() { + const assessment = assessDelivery(edl); + return assessment.verdict === 'pass' ? summarizeEdl(edl).length : 0; + }, + }), + runCase({ + name: 'composition-manifest-validation', + iterations: 10_000, + minOpsPerSecond: 20_000, + execute() { + const result = validateCompositionManifest(manifest); + return result.ok ? (result.data?.scenes.length || 0) : 0; + }, + }), + runCase({ + name: 'narration-estimation', + iterations: 50_000, + minOpsPerSecond: 50_000, + execute(index) { + const result = estimateNarrationDuration(narrationSamples[index % narrationSamples.length], 1); + return Math.round(result.estimatedSec * 100); + }, + }), +]; + +const report = { + schema_version: 1, + benchmark: 'orkas-video-studio-core', + node: process.version, + platform: `${process.platform}-${process.arch}`, + multiplier, + suites, + aggregate: { + passed: suites.every((suite) => suite.passed), + suite_count: suites.length, + total_timed_operations: suites.reduce((total, suite) => total + suite.iterations_per_sample * suite.samples, 0), + }, +}; + +if (jsonOnly) { + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); +} else { + process.stdout.write(`Core benchmark (${report.node}, ${report.platform})\n`); + for (const suite of suites) { + process.stdout.write(`${suite.name.padEnd(34)} ${String(suite.ops_per_second).padStart(9)} ops/s p50=${String(suite.median_us_per_op).padStart(7)} us/op p95=${suite.p95_ms} ms\n`); + } + process.stdout.write(`PASS ${report.aggregate.suite_count} suites, ${report.aggregate.total_timed_operations} timed operations\n`); +}