mirror of
https://github.com/Orkas-AI/Orkas.git
synced 2026-11-07 06:16:03 -10:00
Retain native dependencies and diagnostic test execution
Preserve the reviewed user-facing behavior, public-edition boundaries and owning regression coverage. Prompt audit: Keep existing permission, ownership and recovery contracts at their current tool, runtime or test owner (Hard constraint). No separate resident instruction is added. Drop excluded hosted-only branches; retain shared side effects and tests. No optional prompt rewrite. Prompt confirmation: The requester confirmed the exact proposed final behaviors and existing resource placements by directing that the primary implementation be followed, then explicitly requested this PR and merge. Preserve that confirmed scope only. - Keep: The behavior and placement detailed in the audit above. Type: Hard constraint. Why: Preserve its existing canonical owner without duplicated resident instructions. - Drop: Superseded overlapping wording and excluded hosted-only behavior. Type: Hard constraint. Why: Preserve the confirmed public capability and authority boundary. Confirmed-by: Requester, exact final-state proposal and subsequent delivery request. Scope: The previously confirmed final behavior and existing placement described above; no optional prompt rewrite.
This commit is contained in:
wyt committed
1 parent
52a8050743
commit
b35d6f6f80
21 files changed
+549
-93
No files matched your search
@@ -33,6 +33,7 @@ const AVATAR_COLORS = new Set(
|
||||
const REQUIRED_BUILTIN_INVENTORY = Object.freeze({
|
||||
system_skills: Object.freeze([
|
||||
'agent-creator',
|
||||
'orkas-guide',
|
||||
'package-installer',
|
||||
'skill-creator',
|
||||
'web-app-sdk',
|
||||
|
||||
@@ -96,3 +96,17 @@ lock; after a forced kill, confirm no associated run remains before removing
|
||||
`run.lock`. Old attempt directories are deliberately retained for diagnosis;
|
||||
remove the whole checkpoint directory when that evidence is no longer needed.
|
||||
|
||||
Case-owned IPC failures can have identical messages and timestamps within one
|
||||
millisecond. Each receipt must identify a distinct console line; the reviewed
|
||||
log keeps the exact number of matching console/file pairs. Reusing a receipt
|
||||
for one line, accepting an unacknowledged occurrence, missing either copy, or
|
||||
adding an extra copy still fails. `test/eval/preview-recovery.test.ts` exercises
|
||||
these positive and negative controls without changing the raw error severity.
|
||||
|
||||
Native E2E can opt into `ORKAS_E2E_GPU_DIAGNOSTICS=1` to capture GPU/renderer
|
||||
process-exit reasons, exit codes, memory and feature status in the existing
|
||||
logs. It changes no GPU flag, rendering path, timeout or acceptance rule and
|
||||
records no URLs, profiles or environment values. Use the raw native failure and
|
||||
resource samples to diagnose a recurrence; a successful retry does not establish
|
||||
its cause.
|
||||
|
||||
Generated
+27
-16
@@ -10,7 +10,7 @@
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@earendil-works/pi-ai": "^1.3.0",
|
||||
"@earendil-works/pi-ai": "^1.5.0",
|
||||
"@modelcontextprotocol/sdk": "^1.32.0",
|
||||
"@modelcontextprotocol/server": "2.3.1",
|
||||
"adm-zip": "^0.6.1",
|
||||
@@ -45,6 +45,7 @@
|
||||
"ffmpeg-static": "^5.3.0",
|
||||
"semver": "7.8.5",
|
||||
"tar": "7.5.21",
|
||||
"typebox": "1.4.3",
|
||||
"typescript": "^6.0.3",
|
||||
"vitest": "^4.1.12"
|
||||
},
|
||||
@@ -621,21 +622,21 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@earendil-works/pi-ai": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-1.3.0.tgz",
|
||||
"integrity": "sha512-KaBI8x7q7sPoFUXICNGD9efHzDG+oRNwVvkA9jdIsSQWH1RjFssOlEGsbLtewUPPgCrvmz0l2KTgSLXTyj3acw==",
|
||||
"version": "1.5.0",
|
||||
"resolved": "https://registry.npmjs.org/@earendil-works/pi-ai/-/pi-ai-1.5.0.tgz",
|
||||
"integrity": "sha512-VeWxNqTHSqRCf8AcY/6kvNwh+neRL6MPxZPePVztIQy6qqK9MR/YoCSN/ZgDnIyCD3pqEXgRyQbzk4KV4uvjbg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@anthropic-ai/sdk": "0.129.0",
|
||||
"@aws-sdk/client-bedrock-runtime": "3.1127.0",
|
||||
"@earendil-works/pi-telemetry": "1.3.0",
|
||||
"@earendil-works/pi-schema": "1.5.0",
|
||||
"@earendil-works/pi-telemetry": "1.5.0",
|
||||
"@google/genai": "2.21.0",
|
||||
"@smithy/node-http-handler": "4.12.1",
|
||||
"http-proxy-agent": "9.1.0",
|
||||
"https-proxy-agent": "9.1.0",
|
||||
"openai": "7.19.0",
|
||||
"partial-json": "0.1.7",
|
||||
"typebox": "1.3.27"
|
||||
"partial-json": "0.1.7"
|
||||
},
|
||||
"bin": {
|
||||
"pi-ai": "dist/cli.js"
|
||||
@@ -644,10 +645,19 @@
|
||||
"node": ">=22.19.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@earendil-works/pi-schema": {
|
||||
"version": "1.5.0",
|
||||
"resolved": "https://registry.npmjs.org/@earendil-works/pi-schema/-/pi-schema-1.5.0.tgz",
|
||||
"integrity": "sha512-RW0MbT2tR65OY6xMsgB/yVBb7m/RTNwA+9hurSN1vVXfHZRVmVnXbruBPrx/PzbrQnW5yLVlMRWxErGDQtF1Hg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=22.19.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@earendil-works/pi-telemetry": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@earendil-works/pi-telemetry/-/pi-telemetry-1.3.0.tgz",
|
||||
"integrity": "sha512-ERGZr+aXjNJzoUbB7ZzHVH3g5otzMHjHLs2Fyizm12By5X+gBEhod9cJXFOk13S7M8a29Am+Mq21zm73mRJFLg==",
|
||||
"version": "1.5.0",
|
||||
"resolved": "https://registry.npmjs.org/@earendil-works/pi-telemetry/-/pi-telemetry-1.5.0.tgz",
|
||||
"integrity": "sha512-VW3tcVdnNfl/Vk8wHPTzZdKkxcsCOM12ppiO4DXnd9wMxUCmOfLt/wvUnWeKWbTVlVSRg0uqKtnnhE2UCsG2Bw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=22.19.0"
|
||||
@@ -1363,9 +1373,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@hono/node-server": {
|
||||
"version": "2.0.11",
|
||||
"resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-2.0.11.tgz",
|
||||
"integrity": "sha512-bjD221KPLoJTWUwso1J6fGKiTXEUFedG/s0visavY4zakFPkeGURMRNly+FhBHs7T8Dz4qHaZIMX9ZoJHSJtKA==",
|
||||
"version": "2.1.3",
|
||||
"resolved": "https://registry.npmjs.org/@hono/node-server/-/node-server-2.1.3.tgz",
|
||||
"integrity": "sha512-TA//nWMqPhbfdfneACk6t5a9eqbS9lABEPyKn0/xZTah3H3U2XaVg85rJFl0/Fyit0I552YDHgXGVSf3GwqbUw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
@@ -7570,9 +7580,10 @@
|
||||
}
|
||||
},
|
||||
"node_modules/typebox": {
|
||||
"version": "1.3.27",
|
||||
"resolved": "https://registry.npmjs.org/typebox/-/typebox-1.3.27.tgz",
|
||||
"integrity": "sha512-zu+jc1pcy4UiNThxikUr36f0Rybk9PEeCg/NE6adeWr/SKsdNO4EzZHYRDlv2YCVAfj3Odq3dESSo/jNyoBXzA==",
|
||||
"version": "1.4.3",
|
||||
"resolved": "https://registry.npmjs.org/typebox/-/typebox-1.4.3.tgz",
|
||||
"integrity": "sha512-u3nyf7ZciAv1GOijVpI0TDt9At5wmEi7lSHSlRrnjuThv1DcB7fgGBqqrz+fGZ54jAfTpOOX0JXwxSIg3pnxQA==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/typedarray": {
|
||||
|
||||
+12
-6
@@ -35,6 +35,7 @@
|
||||
"test:local-agents:prepare": "node scripts/run-local-agents-live.mjs --install-only",
|
||||
"test:runtime-path": "node scripts/run-tests.mjs run test/main/platform-process-coverage.test.ts test/main/features/connectors/local-cli.test.ts test/main/features/local_agents/bridge_skill_runner.test.ts test/main/features/generation_reference_assets.test.ts test/main/util/bundled-runtime.test.ts",
|
||||
"test:platform-native": "node scripts/run-platform-native-tests.mjs",
|
||||
"test:startup-native": "node test/gui/run-startup-native.mjs",
|
||||
"test:video-studio-inspect-canary": "node test/gui/run-inspect-canary.mjs",
|
||||
"test:windows-native": "node scripts/run-windows-native-tests.mjs",
|
||||
"test:windows-stability": "node scripts/run-windows-stability-tests.mjs",
|
||||
@@ -59,7 +60,11 @@
|
||||
"test:web-assist-input-canary": "node test/gui/run-web-assist-input-canary.mjs",
|
||||
"test:apps:e2e": "node scripts/run-e2e.cjs test --config playwright.apps.config.ts",
|
||||
"test:local-agents:lifecycle": "node scripts/run-local-agents-live.mjs --lifecycle --no-install",
|
||||
"benchmark:local-agents": "node scripts/run-local-agents-live.mjs --benchmark --no-install"
|
||||
"benchmark:local-agents": "node scripts/run-local-agents-live.mjs --benchmark --no-install",
|
||||
"test:performance:portfolio": "node scripts/run-tests.mjs run test/main/features/search/resource-cost.test.ts test/main/features/search/search-progress.test.ts test/main/features/search/chat-snippets.test.ts test/main/features/search/chat-snippet-failure.test.ts test/main/features/kb_indexer.test.ts test/main/features/project_library_indexer.test.ts test/main/features/file_indexer.test.ts test/main/model/core-agent/file-tools.test.ts test/main/model/core-agent/office-tools.test.ts test/main/model/core-agent/office-batch.test.ts test/main/features/office/office_engine.test.ts test/main/features/library_transfer_integration.test.ts test/main/features/saved_apps.test.ts test/main/features/local_agents/runner.test.ts test/main/features/connectors/mcp-client.test.ts test/main/logger-delivery.test.ts src/core-agent/test/process-session.test.ts src/core-agent/test/run-program-worker-pool.test.ts src/core-agent/test/agent-runner.test.ts test/main/util/tool-result-save-retry.test.ts test/main/model/core-agent/local-tools.test.ts test/main/model/local-tools.test.ts test/main/model/core-agent/file-edit-handoff.test.ts test/main/model/core-agent/repository-search.test.ts test/main/util/office-extract.test.ts test/main/features/conversation-history-worker.test.ts src/core-agent/test/persistent-session.test.ts src/core-agent/test/apply-patch.test.ts test/main/model/core-agent/file-performance.test.ts test/main/features/office/office-readonly_e2e.test.ts",
|
||||
"test:performance:ui": "node scripts/run-e2e.cjs test --config playwright.config.ts test/e2e/chat_e2e_send_responsiveness.spec.ts test/e2e/conversation_e2e_process_responsiveness.spec.ts test/e2e/search_e2e_rebuild.spec.ts",
|
||||
"test:video-studio-product-capture-canary": "node test/gui/run-product-capture-canary.mjs",
|
||||
"test:orkas-guide": "node scripts/run-tests.mjs run test/main/quality/orkas-guide-editions.test.ts test/main/quality/web-app-sdk-builtin.test.ts test/main/features/system_skills.test.ts test/main/model/skill-registry.test.ts test/main/util/builtin-resource-gate.test.ts"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@electron/asar": "3.4.1",
|
||||
@@ -75,10 +80,11 @@
|
||||
"semver": "7.8.5",
|
||||
"tar": "7.5.21",
|
||||
"typescript": "^6.0.3",
|
||||
"vitest": "^4.1.12"
|
||||
"vitest": "^4.1.12",
|
||||
"typebox": "1.4.3"
|
||||
},
|
||||
"dependencies": {
|
||||
"@earendil-works/pi-ai": "^1.3.0",
|
||||
"@earendil-works/pi-ai": "^1.5.0",
|
||||
"@modelcontextprotocol/sdk": "^1.32.0",
|
||||
"@modelcontextprotocol/server": "2.3.1",
|
||||
"adm-zip": "^0.6.1",
|
||||
@@ -97,12 +103,12 @@
|
||||
"tsx": "^4.21.0",
|
||||
"undici": "^7.30.1",
|
||||
"zod": "4.6.7",
|
||||
"fast-xml-parser": "5.7.3",
|
||||
"zod-to-json-schema": "3.25.2"
|
||||
"zod-to-json-schema": "3.25.2",
|
||||
"fast-xml-parser": "5.7.3"
|
||||
},
|
||||
"overrides": {
|
||||
"@anush008/tokenizers": "0.6.0",
|
||||
"@hono/node-server": "2.0.11",
|
||||
"@hono/node-server": "2.1.3",
|
||||
"esbuild": "0.28.1",
|
||||
"tar": "7.5.21",
|
||||
"zod": "4.6.7"
|
||||
|
||||
@@ -105,7 +105,15 @@ function resolveBase(repoRoot) {
|
||||
}
|
||||
|
||||
function selectGovernanceBase(requestedBase, introductions, isAncestor) {
|
||||
return introductions.find((commit) => isAncestor(requestedBase, commit)) || requestedBase;
|
||||
const nodes = [...new Set(introductions)];
|
||||
// Reintroducing the checker must not reset an already adopted policy, nor
|
||||
// let Git log order hide governed commits between the two introductions.
|
||||
if (nodes.some(node => node === requestedBase || isAncestor(node, requestedBase))) return requestedBase;
|
||||
const eligible = nodes.filter(node => isAncestor(requestedBase, node));
|
||||
if (!eligible.length) return requestedBase;
|
||||
const first = eligible.filter(node => eligible.every(other => node === other || isAncestor(node, other)));
|
||||
if (first.length !== 1) throw new Error('ambiguous governance adoption nodes; reconcile the commit baseline');
|
||||
return first[0];
|
||||
}
|
||||
|
||||
/** Do not apply a newly introduced commit policy retroactively to commits that
|
||||
@@ -113,7 +121,7 @@ function selectGovernanceBase(requestedBase, introductions, isAncestor) {
|
||||
* that narrower base remains authoritative. */
|
||||
function resolveEffectiveBase(repoRoot, requestedBase) {
|
||||
const output = runGit([
|
||||
'log', '--format=%H', '--diff-filter=A', 'HEAD', '--', CHECKER_PATH,
|
||||
'log', '--full-history', '--format=%H', '--diff-filter=A', 'HEAD', '--', CHECKER_PATH,
|
||||
], repoRoot);
|
||||
const introductions = output ? output.split('\n').filter(Boolean) : [];
|
||||
return selectGovernanceBase(requestedBase, introductions, (ancestor, descendant) => {
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { createHash } from 'node:crypto';
|
||||
import type { Reporter, FullConfig, Suite, TestCase, TestResult, FullResult } from '@playwright/test/reporter';
|
||||
import type { Reporter, FullConfig, Suite, TestCase, TestResult, FullResult, TestError } from '@playwright/test/reporter';
|
||||
import { analyzeWebAppLogs } from '../test/e2e/fixtures/web-app-log-analysis';
|
||||
import { sanitizeLogTextForUpload } from '../src/main/util/log-sanitize';
|
||||
|
||||
/** Structured outcomes, not console text, determine completion. */
|
||||
export default class CheckpointReporter implements Reporter {
|
||||
@@ -12,6 +13,8 @@ export default class CheckpointReporter implements Reporter {
|
||||
private files: Array<{ file: string; state: string; passed: number; failed: number; skipped: number; durationMs: number }> = [];
|
||||
private testFiles: string[] = [];
|
||||
private errors = 0;
|
||||
private runnerErrors: string[] = [];
|
||||
private omittedRunnerErrors = 0;
|
||||
private appAnalyses: Array<ReturnType<typeof analyzeWebAppLogs> & { case: string }> = [];
|
||||
private artifacts: string[] = [];
|
||||
private outputDir = '';
|
||||
@@ -27,7 +30,13 @@ export default class CheckpointReporter implements Reporter {
|
||||
private event(stream: string, chunk: string | Buffer, test?: TestCase) {
|
||||
fs.appendFileSync(this.events, JSON.stringify({ case: test?.id || 'setup', phase: 'worker', stream, text: chunk.toString() }) + '\n');
|
||||
}
|
||||
onError() { this.errors++; }
|
||||
onError(error: TestError) {
|
||||
this.errors++;
|
||||
if (this.runnerErrors.length < 20) this.runnerErrors.push(sanitizeLogTextForUpload(
|
||||
(error.message || error.value || 'Unhandled Playwright error').slice(0, 4000),
|
||||
).slice(0, 2000));
|
||||
else this.omittedRunnerErrors++;
|
||||
}
|
||||
onTestEnd(test: TestCase, result: TestResult) {
|
||||
const failed = result.status !== test.expectedStatus && result.status !== 'skipped';
|
||||
// A retry is preserved as a finding and never certified for reuse.
|
||||
@@ -63,7 +72,9 @@ export default class CheckpointReporter implements Reporter {
|
||||
}
|
||||
onEnd(result: FullResult) {
|
||||
fs.writeFileSync(this.report, JSON.stringify({ schema: 1, reason: result.status === 'passed' ? 'passed' : 'failed',
|
||||
batchUnits: this.testFiles.length, testFiles: this.testFiles, unhandledErrors: this.errors, inventory: this.inventory, files: this.files, appAnalyses: this.appAnalyses,
|
||||
batchUnits: this.testFiles.length, testFiles: this.testFiles, unhandledErrors: this.errors,
|
||||
runnerErrors: this.runnerErrors, omittedRunnerErrors: this.omittedRunnerErrors,
|
||||
inventory: this.inventory, files: this.files, appAnalyses: this.appAnalyses,
|
||||
artifactPaths: this.artifacts, outputDir: this.outputDir }), { mode: 0o600 });
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,14 @@ process.env.ORKAS_E2E_HIDE_WINDOW = debug ? '0' : '1';
|
||||
if (debug) process.env.PWDEBUG = '1';
|
||||
else delete process.env.PWDEBUG;
|
||||
|
||||
// Source E2E launches esbuild through tsx in Electron and its workers. Bound
|
||||
// Go's parallelism and soft heap target on Windows hosts without a pagefile;
|
||||
// otherwise compilation can exhaust commit space and lose the renderer.
|
||||
if (process.platform === 'win32') {
|
||||
process.env.GOMAXPROCS ||= '1';
|
||||
process.env.GOMEMLIMIT ||= '128MiB';
|
||||
}
|
||||
|
||||
// Playwright forces color in workers and strips it in the parent reporter.
|
||||
// Translate NO_COLOR before loading Playwright so worker environments never
|
||||
// contain conflicting controls, while the reporter retains plain output.
|
||||
|
||||
@@ -30,6 +30,7 @@ const commonSuites = [
|
||||
'src/core-agent/test/process-session.test.ts',
|
||||
'src/core-agent/test/workspace-state.test.ts',
|
||||
'src/core-agent/test/sandbox.test.ts',
|
||||
'src/core-agent/test/windows-cleanup-batch.test.ts',
|
||||
'test/main/model/core-agent/file-tools.test.ts',
|
||||
'test/main/model/core-agent/tool-result-tools.test.ts',
|
||||
'test/main/model/core-agent/local-tools.test.ts',
|
||||
@@ -46,6 +47,11 @@ const commonSuites = [
|
||||
'test/main/features/notification_permissions.test.ts',
|
||||
'test/main/features/connectors/mcp-client.test.ts',
|
||||
'test/main/util/shutdown.test.ts',
|
||||
'test/main/features/desktop_background.test.ts',
|
||||
'test/main/features/desktop_shutdown.test.ts',
|
||||
'test/main/features/login_startup_boot.test.ts',
|
||||
'test/main/features/login_startup.test.ts',
|
||||
'test/main/util/app-tray.test.ts',
|
||||
'test/main/features/generation_reference_assets.test.ts',
|
||||
'test/main/index-process.test.ts',
|
||||
'test/main/features/local_agents/base.test.ts',
|
||||
|
||||
@@ -4,7 +4,7 @@ import { existsSync } from 'node:fs';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { withWindowsGitOnPath } from './test-runtime-env.mjs';
|
||||
import { withWindowsGitOnPath, withWindowsPowerShellModules } from './test-runtime-env.mjs';
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
// The open-source app and its test virtual environment share this flat root.
|
||||
@@ -12,7 +12,7 @@ const repoRoot = resolve(here, '..');
|
||||
const localPython = process.platform === 'win32'
|
||||
? resolve(repoRoot, 'venv', 'Scripts', 'python.exe')
|
||||
: resolve(repoRoot, 'venv', 'bin', 'python');
|
||||
const testEnvironment = withWindowsGitOnPath(process.env);
|
||||
const testEnvironment = withWindowsPowerShellModules(withWindowsGitOnPath(process.env));
|
||||
const candidates = [testEnvironment.ORKAS_TEST_PYTHON, localPython, 'python3', 'python']
|
||||
.filter((value, index, all) => value && all.indexOf(value) === index);
|
||||
const python = candidates.find((candidate) => {
|
||||
|
||||
@@ -10,7 +10,7 @@ import { spawn } from 'node:child_process';
|
||||
import { createRequire } from 'node:module';
|
||||
import { dirname, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { withTestPython, withWindowsGitOnPath } from './test-runtime-env.mjs';
|
||||
import { withTestPython, withWindowsGitOnPath, withWindowsPowerShellModules } from './test-runtime-env.mjs';
|
||||
import { formatDriftReport, installedDependencyDrift } from './check-installed-dependencies.mjs';
|
||||
|
||||
// A merge that advanced the lockfile leaves stale packages installed, and the
|
||||
@@ -25,7 +25,7 @@ const here = dirname(fileURLToPath(import.meta.url));
|
||||
const require_ = createRequire(import.meta.url);
|
||||
const electronBin = require_('electron');
|
||||
const vitestBin = resolve(here, '..', 'node_modules', 'vitest', 'vitest.mjs');
|
||||
const testEnvironment = withTestPython(withWindowsGitOnPath(process.env));
|
||||
const testEnvironment = withTestPython(withWindowsPowerShellModules(withWindowsGitOnPath(process.env)));
|
||||
// Nested launcher calls must not overwrite their parent's checkpoint report.
|
||||
delete testEnvironment.ORKAS_TEST_BATCH_REPORT;
|
||||
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env node
|
||||
/** Deterministic GUI coverage for isolated product page layout and capture. */
|
||||
import { createRequire } from 'node:module';
|
||||
import { spawn } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { reviewInspectStderr, inspectCanaryExitCode } from './inspect-canary-logs.mjs';
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const pcDir = path.resolve(here, '../..');
|
||||
const require = createRequire(import.meta.url);
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'orkas-product-gui-canary-'));
|
||||
const data = path.join(root, 'data');
|
||||
fs.mkdirSync(data);
|
||||
const env = { ...process.env, ELECTRON_RUN_AS_NODE: '',
|
||||
ORKAS_WORKSPACE_ROOT: data,
|
||||
ORKAS_GUI_TEST_USER_DATA_DIR: path.join(root, 'chromium'),
|
||||
ORKAS_GUI_TEST_RUNNER: path.join(here, 'video-studio-product-capture-canary.ts'),
|
||||
};
|
||||
const child = spawn(require('electron'), [path.join(here, 'electron-main.mjs')], {
|
||||
cwd: pcDir, env, stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true,
|
||||
});
|
||||
let stdoutText = '';
|
||||
let stderrBytes = 0;
|
||||
let stderrText = '';
|
||||
let timedOut = false;
|
||||
child.stdout.on('data', chunk => { stdoutText += chunk.toString(); process.stdout.write(chunk); });
|
||||
child.stderr.setEncoding('utf8');
|
||||
child.stderr.on('data', chunk => {
|
||||
stderrBytes += Buffer.byteLength(chunk);
|
||||
if (stderrBytes <= 1024 * 1024) stderrText += chunk;
|
||||
process.stderr.write(chunk);
|
||||
});
|
||||
const stop = () => child.kill('SIGTERM');
|
||||
const signals = ['SIGINT', 'SIGTERM', 'SIGHUP'];
|
||||
for (const signal of signals) process.on(signal, stop);
|
||||
const timeout = setTimeout(() => { timedOut = true; child.kill('SIGKILL'); }, 180_000);
|
||||
try {
|
||||
const code = await new Promise((resolve) => {
|
||||
child.once('error', () => resolve(1));
|
||||
child.once('close', code => resolve(code ?? 1));
|
||||
});
|
||||
if (timedOut) process.stderr.write('[product-capture-canary] GUI host exceeded the 180s deadline\n');
|
||||
// All child output stays visible. Only bounded, reviewed Linux baseline
|
||||
// diagnostics receive warning classifications; unknown output fails closed.
|
||||
let reviewed = stderrText;
|
||||
const expected = [];
|
||||
const endpoint = stdoutText.match(/\[product-capture-canary\] expected closed endpoint (http:\/\/127\.0\.0\.1:\d+\/)/)?.[1];
|
||||
if (endpoint && stdoutText.includes('ok an unreachable page fails cleanly without leaving files')) {
|
||||
const rows = reviewed.split(/\r?\n/);
|
||||
reviewed = rows.filter(line => {
|
||||
const warning = /^\(node:\d+\) electron: Failed to load URL: (http:\/\/127\.0\.0\.1:\d+\/) with error: ERR_CONNECTION_REFUSED$/.exec(line);
|
||||
if (warning && warning[1] === endpoint && !expected.includes('closed-loopback')) { expected.push('closed-loopback'); return false; }
|
||||
if (expected.includes('closed-loopback') && /^\(Use `[^`]+ --trace-warnings \.\.\.` to show where the warning was created\)$/.test(line) && !expected.includes('node-warning-hint')) { expected.push('node-warning-hint'); return false; }
|
||||
return true;
|
||||
}).join('\n');
|
||||
}
|
||||
const report = { ...reviewInspectStderr(reviewed), expected };
|
||||
const reportDir = process.env.ORKAS_PRODUCT_CAPTURE_REPORT_DIR;
|
||||
if (reportDir) { fs.mkdirSync(reportDir, { recursive: true }); fs.writeFileSync(path.join(reportDir,'stdout.log'),stdoutText); fs.writeFileSync(path.join(reportDir,'stderr.log'),stderrText); fs.writeFileSync(path.join(reportDir,'analysis.json'),JSON.stringify(report,null,2)); }
|
||||
if (stderrBytes > 1024 * 1024) report.unexpected++;
|
||||
process.stdout.write(`[product-capture-canary] process log review: ${JSON.stringify(report)}\n`);
|
||||
if (report.unexpected) process.stderr.write('[product-capture-canary] process log review failed: unexpected stderr\n');
|
||||
process.exitCode = inspectCanaryExitCode(code, timedOut, report);
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
for (const signal of signals) process.off(signal, stop);
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
@@ -1,13 +1,29 @@
|
||||
import { afterEach, expect, it, vi } from 'vitest';
|
||||
import { checkLatencyBudget } from './latency-budget';
|
||||
import { checkLatencyBudget, LATENCY_TARGET_MS } from './latency-budget';
|
||||
import { createModelEvalLogCollector } from '../../scripts/model-eval-log-policy.mjs';
|
||||
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
it('surfaces a soft latency warning without converting it into a checkpoint error', () => {
|
||||
const collector = createModelEvalLogCollector('latency-contract');
|
||||
const write = process.stdout.write.bind(process.stdout);
|
||||
vi.spyOn(process.stdout, 'write').mockImplementation(chunk => {
|
||||
collector.ingest('stdout', String(chunk));
|
||||
return write(chunk);
|
||||
});
|
||||
checkLatencyBudget('fixture response', { elapsedMs: LATENCY_TARGET_MS + 1 }, 700);
|
||||
expect(collector.finish()).toMatchObject({ passed: true, requiresReview: true,
|
||||
coveragePassed: true, levelCounts: { warn: 1, error: 0 } });
|
||||
const unknown = createModelEvalLogCollector('unknown-stderr-control');
|
||||
unknown.ingest('stderr', 'Unexpected fixture stderr\n');
|
||||
expect(unknown.finish().passed).toBe(false);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{ platform: 'win32', limit: 800, firstRequest: 700 },
|
||||
{ platform: 'darwin', limit: 500, firstRequest: 500 },
|
||||
{ platform: 'linux', limit: 500, firstRequest: 500 },
|
||||
])('uses the approved $limit ms response budget on $platform', async ({ platform, limit, firstRequest }) => {
|
||||
{ platform: 'win32', limit: 1000, firstRequest: 1000, target: 500, ready: 3000, interactiveGap: 300, bulkGap: 500, multilineEdit: 300, paragraphEdit: 500 },
|
||||
{ platform: 'darwin', limit: 500, firstRequest: 500, target: 300, ready: 1000, interactiveGap: 150, bulkGap: 300, multilineEdit: 150, paragraphEdit: 300 },
|
||||
{ platform: 'linux', limit: 500, firstRequest: 500, target: 300, ready: 1000, interactiveGap: 150, bulkGap: 300, multilineEdit: 150, paragraphEdit: 300 },
|
||||
])('uses the approved $limit ms response budget on $platform', async ({ platform, limit, firstRequest, target, ready, interactiveGap, bulkGap, multilineEdit, paragraphEdit }) => {
|
||||
const descriptor = Object.getOwnPropertyDescriptor(process, 'platform')!;
|
||||
let budget: typeof import('./latency-budget');
|
||||
try {
|
||||
@@ -19,10 +35,24 @@ it.each([
|
||||
}
|
||||
expect(budget.LATENCY_LIMIT_MS).toBe(limit);
|
||||
expect(budget.FIRST_REQUEST_LATENCY_LIMIT_MS).toBe(firstRequest);
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
|
||||
expect(budget.LATENCY_TARGET_MS).toBe(target);
|
||||
expect(budget.LIBRARY_READY_TIMEOUT_MS).toBe(ready);
|
||||
expect(budget.INTERACTIVE_MAIN_GAP_LIMIT_MS).toBe(interactiveGap);
|
||||
expect(budget.BULK_MAIN_GAP_LIMIT_MS).toBe(bulkGap);
|
||||
expect(budget.COMPOSER_EDIT_LIMIT_MS).toEqual({ multiline: multilineEdit, paragraph: paragraphEdit });
|
||||
const warn = vi.spyOn(process.stdout, 'write').mockImplementation(() => true);
|
||||
for (const ms of [firstRequest - 1, firstRequest, firstRequest + 1]) {
|
||||
warn.mockClear();
|
||||
const check = () => budget.checkLatencyBudget('first request', { elapsedMs: ms }, budget.FIRST_REQUEST_LATENCY_LIMIT_MS);
|
||||
if (ms <= firstRequest) expect(check).not.toThrow();
|
||||
else expect(check).toThrow(`budget ${firstRequest} ms`);
|
||||
expect(warn).toHaveBeenCalledOnce();
|
||||
expect(JSON.parse(String(warn.mock.calls[0][0]).split('latency target exceeded ')[1]))
|
||||
.toMatchObject({ elapsed_ms: ms, target_ms: target, limit_ms: firstRequest });
|
||||
}
|
||||
for (const { ms, warnings, fails } of [
|
||||
{ ms: 300, warnings: 0, fails: false },
|
||||
{ ms: 301, warnings: 1, fails: false },
|
||||
{ ms: target, warnings: 0, fails: false },
|
||||
{ ms: target + 1, warnings: 1, fails: false },
|
||||
{ ms: limit - 1, warnings: 1, fails: false },
|
||||
{ ms: limit, warnings: 1, fails: false },
|
||||
{ ms: limit + 1, warnings: 1, fails: true },
|
||||
@@ -35,9 +65,9 @@ it.each([
|
||||
else expect(check).not.toThrow();
|
||||
expect(warn).toHaveBeenCalledTimes(warnings);
|
||||
if (warnings) {
|
||||
expect(warn.mock.calls[0][0]).toBe('[latency-warning]');
|
||||
expect(JSON.parse(warn.mock.calls[0][1])).toEqual({
|
||||
metric: 'fixture request', elapsed_ms: ms, target_ms: 300, limit_ms: limit,
|
||||
expect(warn.mock.calls[0][0]).toMatch(/^\[00:00:00.000\] \[warn\] \[\(latency-budget\)\] latency target exceeded /);
|
||||
expect(JSON.parse(String(warn.mock.calls[0][0]).split('latency target exceeded ')[1])).toEqual({
|
||||
metric: 'fixture request', elapsed_ms: ms, target_ms: target, limit_ms: limit,
|
||||
phases: [{ phase: 'prepared', ms: 100 }],
|
||||
});
|
||||
}
|
||||
@@ -47,11 +77,3 @@ it.each([
|
||||
it.each([NaN, Infinity, -1])('rejects an invalid measurement (%s) instead of passing silently', ms => {
|
||||
expect(() => checkLatencyBudget('fixture request', { elapsedMs: ms })).toThrow('Invalid latency measurement');
|
||||
});
|
||||
|
||||
it.each([700, 701])('enforces the Windows first-request boundary at %i ms while retaining warnings', ms => {
|
||||
const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
|
||||
const check = () => checkLatencyBudget('first request', { elapsedMs: ms }, 700);
|
||||
if (ms === 700) expect(check).not.toThrow();
|
||||
else expect(check).toThrow('budget 700 ms');
|
||||
expect(JSON.parse(warn.mock.calls[0][1])).toMatchObject({ target_ms: 300, limit_ms: 700 });
|
||||
});
|
||||
@@ -1,8 +1,17 @@
|
||||
export const LATENCY_TARGET_MS = 300;
|
||||
// Fixed Windows tolerances approved on 2026-10-29; other platforms retain their budgets.
|
||||
export const LATENCY_TARGET_MS = process.platform === 'win32' ? 500 : 300;
|
||||
// Platform-specific response budget approved for lower-spec Windows hosts.
|
||||
export const LATENCY_LIMIT_MS = process.platform === 'win32' ? 800 : 500;
|
||||
export const LATENCY_LIMIT_MS = process.platform === 'win32' ? 1000 : 500;
|
||||
// First-request setup has a separate, approved budget on lower-spec Windows hosts.
|
||||
export const FIRST_REQUEST_LATENCY_LIMIT_MS = process.platform === 'win32' ? 700 : LATENCY_LIMIT_MS;
|
||||
export const FIRST_REQUEST_LATENCY_LIMIT_MS = process.platform === 'win32' ? 1000 : LATENCY_LIMIT_MS;
|
||||
|
||||
// Readiness is a functional-test watchdog, separate from response latency.
|
||||
export const LIBRARY_READY_TIMEOUT_MS = process.platform === 'win32' ? 3000 : 1000;
|
||||
export const INTERACTIVE_MAIN_GAP_LIMIT_MS = process.platform === 'win32' ? 300 : 150;
|
||||
export const BULK_MAIN_GAP_LIMIT_MS = process.platform === 'win32' ? 500 : 300;
|
||||
export const COMPOSER_EDIT_LIMIT_MS = process.platform === 'win32'
|
||||
? { multiline: 300, paragraph: 500 }
|
||||
: { multiline: 150, paragraph: 300 };
|
||||
|
||||
export interface LatencySample {
|
||||
elapsedMs: number;
|
||||
@@ -17,10 +26,12 @@ export function checkLatencyBudget(metric: string, sample: LatencySample, limitM
|
||||
throw new Error('Invalid latency measurement');
|
||||
}
|
||||
if (elapsedMs > LATENCY_TARGET_MS) {
|
||||
console.warn('[latency-warning]', JSON.stringify({
|
||||
// Keep severity in the shared structured log protocol. stdout avoids
|
||||
// Playwright adding a non-structured stderr case header to this warning.
|
||||
process.stdout.write('[00:00:00.000] [warn] [(latency-budget)] latency target exceeded ' + JSON.stringify({
|
||||
metric, elapsed_ms: elapsedMs, target_ms: LATENCY_TARGET_MS,
|
||||
limit_ms: limitMs, phases,
|
||||
}));
|
||||
}) + '\n');
|
||||
}
|
||||
if (elapsedMs > limitMs) {
|
||||
throw new Error(`${metric} took ${elapsedMs.toFixed(1)} ms (budget ${limitMs} ms)`);
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
'use strict';
|
||||
// Test-only caller attribution. API duration includes OS cache/decoding; it is
|
||||
// not a physical-disk counter. Nested append->write calls must not be summed.
|
||||
const fs = require('node:fs');
|
||||
const crypto = require('node:crypto');
|
||||
const { syncBuiltinESMExports } = require('node:module');
|
||||
|
||||
function profileSyncCosts(inScope) {
|
||||
const report = { fs: {}, syncFsMs: 0, hashMs: 0, hashCalls: 0, hashBytes: 0 };
|
||||
const restore = [];
|
||||
let depth = 0, active = true;
|
||||
for (const name of ['readFileSync', 'writeFileSync', 'appendFileSync', 'statSync', 'lstatSync',
|
||||
'copyFileSync', 'renameSync', 'rmSync', 'mkdirSync', 'existsSync']) {
|
||||
const original = fs[name];
|
||||
fs[name] = function (...args) {
|
||||
if (!active || depth || !inScope(args[0])) return Reflect.apply(original, this, args);
|
||||
const stat = report.fs[name] ||= { calls: 0, ms: 0, bytes: 0, failures: 0 };
|
||||
const start = performance.now();
|
||||
let result, failed = false;
|
||||
depth++;
|
||||
try { result = Reflect.apply(original, this, args); return result; }
|
||||
catch (error) { failed = true; throw error; }
|
||||
finally {
|
||||
const elapsed = performance.now() - start; depth--;
|
||||
stat.calls++; stat.ms += elapsed; report.syncFsMs += elapsed;
|
||||
if (failed) stat.failures++;
|
||||
const value = name === 'readFileSync' ? result
|
||||
: name === 'writeFileSync' || name === 'appendFileSync' ? args[1] : null;
|
||||
if (!failed && (typeof value === 'string' || Buffer.isBuffer(value))) stat.bytes += Buffer.byteLength(value);
|
||||
}
|
||||
};
|
||||
restore.push(() => { fs[name] = original; });
|
||||
}
|
||||
const createHash = crypto.createHash;
|
||||
crypto.createHash = function (...args) {
|
||||
const hash = Reflect.apply(createHash, this, args);
|
||||
if (!active || args[0] !== 'sha256') return hash;
|
||||
report.hashCalls++;
|
||||
for (const name of ['update', 'digest']) {
|
||||
const original = hash[name];
|
||||
hash[name] = function (...values) {
|
||||
if (!active) return Reflect.apply(original, this, values);
|
||||
const start = performance.now();
|
||||
try { return Reflect.apply(original, this, values); }
|
||||
finally {
|
||||
report.hashMs += performance.now() - start;
|
||||
if (name === 'update') {
|
||||
const value = values[0];
|
||||
if (typeof value === 'string') report.hashBytes += Buffer.byteLength(value, values[1]);
|
||||
else if (ArrayBuffer.isView(value)) report.hashBytes += value.byteLength;
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
return hash;
|
||||
};
|
||||
restore.push(() => { crypto.createHash = createHash; });
|
||||
syncBuiltinESMExports();
|
||||
return { report, stop() {
|
||||
if (!active) return report;
|
||||
active = false;
|
||||
for (const undo of restore.reverse()) undo();
|
||||
syncBuiltinESMExports();
|
||||
return report;
|
||||
} };
|
||||
}
|
||||
module.exports = { profileSyncCosts };
|
||||
@@ -5,13 +5,13 @@ import { describe, expect, it } from 'vitest';
|
||||
|
||||
const source = readFileSync(path.resolve(process.cwd(), 'scripts/run-e2e.cjs'), 'utf8');
|
||||
|
||||
function launch(args: string[], inherited: Record<string, string> = {}) {
|
||||
function launch(args: string[], inherited: Record<string, string> = {}, platform = 'darwin') {
|
||||
const env = { ...inherited };
|
||||
const argv = ['node', 'scripts/run-e2e.cjs', ...args];
|
||||
let loadedEnv: Record<string, string> | undefined;
|
||||
let parsedArgv: string[] | undefined;
|
||||
runInNewContext(source, {
|
||||
process: { env, argv },
|
||||
process: { env, argv, platform },
|
||||
require(name: string) {
|
||||
if (name !== 'playwright/lib/program') throw new Error(`Unexpected runner import: ${name}`);
|
||||
loadedEnv = { ...env };
|
||||
@@ -23,6 +23,21 @@ function launch(args: string[], inherited: Record<string, string> = {}) {
|
||||
}
|
||||
|
||||
describe('E2E runner desktop isolation', () => {
|
||||
it.each<Record<string, string>>([{}, { GOMAXPROCS: '', GOMEMLIMIT: '' }])('bounds Windows source compilation before Playwright loads with %j', inherited => {
|
||||
const result = launch(['test'], inherited, 'win32');
|
||||
expect(result).toMatchObject({ GOMAXPROCS: '1', GOMEMLIMIT: '128MiB', ORKAS_E2E_HIDE_WINDOW: '1' });
|
||||
});
|
||||
|
||||
it('preserves explicit Windows compiler resource budgets', () => {
|
||||
const result = launch(['test'], { GOMAXPROCS: '2', GOMEMLIMIT: '256MiB' }, 'win32');
|
||||
expect(result).toMatchObject({ GOMAXPROCS: '2', GOMEMLIMIT: '256MiB' });
|
||||
});
|
||||
|
||||
it.each(['darwin', 'linux'])('leaves compiler budgets unchanged on %s', platform => {
|
||||
const result = launch(['test'], {}, platform);
|
||||
expect(result.GOMAXPROCS).toBeUndefined(); expect(result.GOMEMLIMIT).toBeUndefined();
|
||||
});
|
||||
|
||||
it.each<{ name: string; env: Record<string, string> }>([
|
||||
{ name: 'clean shell', env: {} },
|
||||
{ name: 'inherited visible mode', env: { ORKAS_E2E_SHOW_WINDOW: '1' } },
|
||||
|
||||
@@ -79,6 +79,10 @@ const PROCESS_OWNER_SUITE: Record<string, ProcessOwner> = {
|
||||
suite: 'src/core-agent/test/sandbox.test.ts',
|
||||
evidence: 'SandboxExecutor',
|
||||
},
|
||||
'src/core-agent/src/sandbox/windows-cleanup-batch.ts': {
|
||||
suite: 'src/core-agent/test/windows-cleanup-batch.test.ts',
|
||||
evidence: 'killProcessTree',
|
||||
},
|
||||
'src/core-agent/src/tools/process-session.ts': {
|
||||
suite: 'src/core-agent/test/process-session.test.ts',
|
||||
evidence: 'getProcessSessionTools',
|
||||
@@ -188,6 +192,11 @@ describe('platform process test ownership', () => {
|
||||
'test/main/features/connectors/mcp-client.test.ts',
|
||||
'test/main/util/shutdown.test.ts',
|
||||
'test/main/util/migrate-source-data-root.test.ts',
|
||||
'test/main/features/desktop_background.test.ts',
|
||||
'test/main/features/desktop_shutdown.test.ts',
|
||||
'test/main/features/login_startup_boot.test.ts',
|
||||
'test/main/features/login_startup.test.ts',
|
||||
'test/main/util/app-tray.test.ts',
|
||||
]) expect(selected).toContain(suite);
|
||||
for (const suite of selected.filter(arg => arg.endsWith('.test.ts'))) {
|
||||
expect(fs.existsSync(path.resolve(suite)), `native selection has a missing suite: ${suite}`).toBe(true);
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
import { afterEach, expect, it } from 'vitest';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { execFileSync, spawnSync } from 'node:child_process';
|
||||
import { removeGitFixture } from '../../helpers/remove-git-fixture';
|
||||
|
||||
const roots: string[] = [];
|
||||
afterEach(() => roots.splice(0).forEach(removeGitFixture));
|
||||
|
||||
it('does not erase governed commits when the checker is deleted and reintroduced', () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'orkas-governance-node-'));
|
||||
roots.push(root);
|
||||
const git = (...args: string[]) => execFileSync('git', ['-C', root, ...args], { encoding: 'utf8' }).trim();
|
||||
const commit = (message: string) => { git('add', '.'); git('commit', '-qm', message); return git('rev-parse', 'HEAD'); };
|
||||
git('init', '-q');
|
||||
git('config', 'user.email', '[email protected]');
|
||||
git('config', 'user.name', 'Fixture');
|
||||
fs.writeFileSync(path.join(root, 'README.md'), 'fixture\n');
|
||||
const base = commit('before governance');
|
||||
const relative = 'PC/scripts/check-prompt-commit-metadata.cjs';
|
||||
const script = path.join(root, relative);
|
||||
fs.mkdirSync(path.dirname(script), { recursive: true });
|
||||
const bytes = fs.readFileSync(path.resolve(process.cwd(), 'scripts/check-prompt-commit-metadata.cjs'));
|
||||
fs.writeFileSync(script, bytes);
|
||||
commit('adopt checker');
|
||||
fs.writeFileSync(path.join(root, 'CLAUDE.md'), 'Synthetic governed change.\n');
|
||||
const unaudited = commit('missing required metadata');
|
||||
fs.unlinkSync(script);
|
||||
commit('temporarily remove checker');
|
||||
fs.writeFileSync(script, bytes);
|
||||
const reintroduced = commit('reintroduce checker');
|
||||
const run = (start: string) => spawnSync(process.env.ORKAS_TEST_NODE || process.execPath,
|
||||
[script, '--base', start], { cwd: root, encoding: 'utf8' });
|
||||
const result = run(base);
|
||||
expect(result.status).toBe(1);
|
||||
expect(result.stderr).toContain(unaudited.slice(0, 12));
|
||||
expect(result.stderr).toContain('missing complete Prompt audit metadata');
|
||||
// A genuinely later event base keeps its narrow scope.
|
||||
expect(run(reintroduced).status).toBe(0);
|
||||
});
|
||||
@@ -28,6 +28,43 @@ describe('quality › validateSkillFile', () => {
|
||||
expect(r.violations).toEqual([]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['LF', '\n', '---', true],
|
||||
['CRLF', '\r\n', '---', true],
|
||||
['trailing horizontal whitespace', '\n', '--- \t', true],
|
||||
['closing fence at EOF', '\n', '---', false],
|
||||
])('accepts inline dashes without losing later metadata: %s', (_label, eol, close, withBody) => {
|
||||
const content = [
|
||||
'---',
|
||||
'description: Convert Markdown --- tables included --- to PDF',
|
||||
'name: md2pdf',
|
||||
close,
|
||||
...(withBody ? ['# Instructions', 'Process a document.'] : []),
|
||||
].join(eol as string);
|
||||
const result = validateSkillFile({ relpath: 'SKILL.md', content });
|
||||
expect(result.ok).toBe(true);
|
||||
expect(result.violations).toEqual([]);
|
||||
});
|
||||
|
||||
it.each(['', '---suffix', '----'])('rejects inline dashes without a standalone closing fence: %s', (close) => {
|
||||
const content = [
|
||||
'---', 'name: md2pdf', 'description: Convert Markdown --- to PDF', close,
|
||||
].join('\n');
|
||||
const result = validateSkillFile({ relpath: 'SKILL.md', content });
|
||||
expect(result.ok).toBe(false);
|
||||
expect(result.violations.map((v) => v.rule)).toContain('frontmatter_unparseable');
|
||||
});
|
||||
|
||||
it('still scans executable body content after metadata containing inline dashes', () => {
|
||||
const content = [
|
||||
'---', 'description: Convert Markdown --- to PDF', 'name: md2pdf', '---',
|
||||
'```bash', 'cat ~/.ssh/config', '```',
|
||||
].join('\n');
|
||||
const result = validateSkillFile({ relpath: 'SKILL.md', content });
|
||||
expect(result.ok).toBe(false);
|
||||
expect(result.violations.map((v) => v.rule)).toEqual(['no_credential_path_read']);
|
||||
});
|
||||
|
||||
it('flags missing frontmatter description as advisory', () => {
|
||||
const content = '---\nname: x\n---\n';
|
||||
const r = validateSkillFile({ relpath: 'SKILL.md', content });
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
import { createHash } from 'node:crypto';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { analyzeWebAppLogs, expectedBrowserDenialIndices, expectedResourceDenialIndex,
|
||||
type BrowserDenialObservation } from '../../e2e/fixtures/web-app-log-analysis';
|
||||
|
||||
const detail = 'Failed to load resource: the server responded with a status of 403 (Forbidden)';
|
||||
const url = 'chat-app://fixture/index.html?denial-probe=one';
|
||||
const record = { diagnosticIndex: 2, type: 'error', url, detail };
|
||||
const request = { url, after: 2, status: 403 };
|
||||
const receipt = { start: 2, end: 2, diagnosticIndex: 2, reason: 'expected-resource-denial' as const,
|
||||
requestHash: 'a'.repeat(64), status: 403 };
|
||||
const log = `[main:stdout] fixture ready\n[renderer:error] ${detail}\n`;
|
||||
|
||||
describe('expected browser resource denial evidence', () => {
|
||||
it('attributes only the exact request and phase while retaining original severity', () => {
|
||||
expect(expectedResourceDenialIndex([record], request)).toBe(2);
|
||||
expect(analyzeWebAppLogs(log).passed).toBe(false);
|
||||
const result = analyzeWebAppLogs(log, [receipt]);
|
||||
expect(result).toMatchObject({ passed: true, requiresReview: true, coveragePassed: true,
|
||||
expectedResourceDenialCount: 1, originalLevelCounts: { error: 1 }, levelCounts: { error: 0, warn: 1 } });
|
||||
expect(result.analyzedLineCount).toBe(result.capturedLineCount);
|
||||
expect(result.findings).toContainEqual(expect.objectContaining({ source: 'expected-resource-denial', severity: 'warning' }));
|
||||
});
|
||||
it.each([
|
||||
['different request', { ...record, url: url + '-other' }],
|
||||
['earlier phase', { ...record, diagnosticIndex: 1 }],
|
||||
['page exception', { ...record, type: 'pageerror' }],
|
||||
['different error', { ...record, detail: 'Unexpected permission failure' }],
|
||||
['missing browser location', { ...record, url: undefined }],
|
||||
])('rejects %s', (_name, candidate) => {
|
||||
expect(() => expectedResourceDenialIndex([candidate], request)).toThrow('exactly one attributed');
|
||||
});
|
||||
it('rejects missing, duplicate and successful response evidence', () => {
|
||||
expect(() => expectedResourceDenialIndex([], request)).toThrow();
|
||||
expect(() => expectedResourceDenialIndex([record, { ...record, diagnosticIndex: 3 }], request)).toThrow();
|
||||
expect(() => expectedResourceDenialIndex([record], { ...request, status: 200 })).toThrow();
|
||||
});
|
||||
it('keeps an unrelated or later identical 403 fatal', () => {
|
||||
const result = analyzeWebAppLogs(log + `[renderer:error] ${detail}\n`, [receipt]);
|
||||
expect(result).toMatchObject({ passed: false, coveragePassed: true, expectedResourceDenialCount: 1,
|
||||
originalLevelCounts: { error: 2 }, levelCounts: { error: 1, warn: 1 } });
|
||||
});
|
||||
it.each([
|
||||
{ ...receipt, status: 200 }, { ...receipt, requestHash: '' }, { ...receipt, end: 3 },
|
||||
])('fails closed on invalid denial receipts', candidate => {
|
||||
expect(() => analyzeWebAppLogs(log, [candidate])).toThrow();
|
||||
});
|
||||
it('cannot classify an unrelated error using a denial receipt', () => {
|
||||
expect(() => analyzeWebAppLogs(log.replace(detail, 'Unexpected error'), [receipt])).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('expected CORS/CSP denial receipts', () => {
|
||||
const target = 'chat-app://foreign/index.html';
|
||||
const origin = 'chat-app://current';
|
||||
const cases: Array<[BrowserDenialObservation['kind'], string, string]> = [
|
||||
['cors-preflight',
|
||||
`Access to fetch at '${target}' from origin '${origin}' has been blocked by CORS policy: Response to preflight request doesn't pass access control check: No 'Access-Control-Allow-Origin' header is present on the requested resource.`,
|
||||
'Failed to load resource: net::ERR_FAILED'],
|
||||
['cors-scheme',
|
||||
`Access to fetch at '${target}' from origin '${origin}' has been blocked by CORS policy: Cross origin requests are only supported for protocol schemes: chat-media, chrome, chrome-extension, chrome-untrusted, data, http, https.`,
|
||||
'Failed to load resource: net::ERR_FAILED'],
|
||||
['csp-connect',
|
||||
`Connecting to '${target}' violates the following Content Security Policy directive: "connect-src 'self' http: https: ws: wss:". The action has been blocked.`,
|
||||
`Fetch API cannot load ${target}. Refused to connect because it violates the document's Content Security Policy.`],
|
||||
];
|
||||
it.each(cases)('attributes only the observed %s pair and preserves raw errors', (kind, first, second) => {
|
||||
const records = [first, second].map((detail, i) => ({ type: 'error', diagnosticIndex: i + 2, detail }));
|
||||
const observation = { kind, url: target, origin, after: 2 };
|
||||
expect(expectedBrowserDenialIndices(records, observation)).toEqual([2, 3]);
|
||||
const raw = [first, second].map(detail => `[renderer:error] ${detail}`).join('\n');
|
||||
expect(analyzeWebAppLogs(raw).passed).toBe(false);
|
||||
const receipts = [first, second].map((detail, i) => ({ start: i + 1, end: i + 1, diagnosticIndex: i + 2,
|
||||
reason: 'expected-browser-denial' as const, messageHash: createHash('sha256').update(detail).digest('hex') }));
|
||||
expect(analyzeWebAppLogs(raw, receipts)).toMatchObject({ passed: true, coveragePassed: true,
|
||||
originalLevelCounts: { error: 2 }, levelCounts: { error: 0, warn: 2 }, expectedBrowserDenialCount: 2 });
|
||||
expect(analyzeWebAppLogs(raw + '\n[renderer:error] Failed to load resource: net::ERR_FAILED', receipts).passed).toBe(false);
|
||||
for (const wrong of [{ ...observation, url: target + '-other' }, { ...observation, after: 3 }]) {
|
||||
expect(() => expectedBrowserDenialIndices(records, wrong)).toThrow();
|
||||
}
|
||||
expect(() => expectedBrowserDenialIndices(records.slice(0, 1), observation)).toThrow();
|
||||
expect(() => expectedBrowserDenialIndices([...records, { ...records[1], diagnosticIndex: 4 }], observation)).toThrow();
|
||||
expect(() => expectedBrowserDenialIndices(records.map(record => ({ ...record, type: 'pageerror' })), observation)).toThrow();
|
||||
expect(() => analyzeWebAppLogs(raw.replace(first, 'Unrelated error'), receipts)).toThrow();
|
||||
expect(() => analyzeWebAppLogs(raw, [{ ...receipts[0], end: 2 }])).toThrow();
|
||||
});
|
||||
it('rejects missing/invalid observations before accepting any receipt', () => {
|
||||
const request: BrowserDenialObservation = { kind: 'cors-preflight', url: target, origin, after: 0 };
|
||||
for (const wrong of [{ ...request, origin: '' }, { ...request, after: -1 },
|
||||
{ ...request, url: target + '\nforged' }, { ...request, kind: 'unknown' as any }]) {
|
||||
expect(() => expectedBrowserDenialIndices([], wrong)).toThrow();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -4,6 +4,7 @@ import path from 'node:path';
|
||||
import ts from 'typescript';
|
||||
import { spawn, spawnSync } from 'node:child_process';
|
||||
import { afterEach, expect, it } from 'vitest';
|
||||
import { analyzeWebAppLogs } from '../../e2e/fixtures/web-app-log-analysis';
|
||||
|
||||
const pcRoot = path.resolve(__dirname, '../../..');
|
||||
const node = process.env.ORKAS_TEST_NODE || process.execPath;
|
||||
@@ -11,6 +12,35 @@ const python = process.env.ORKAS_TEST_PYTHON!;
|
||||
const roots: string[] = [];
|
||||
afterEach(() => roots.splice(0).forEach(root => fs.rmSync(root, { recursive: true, force: true })));
|
||||
|
||||
it('retains bounded Playwright collection errors even before onBegin', async () => {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'orkas-collection-errors-'));
|
||||
roots.push(root);
|
||||
const report = path.join(root, 'report.json');
|
||||
const previous = process.env.ORKAS_TEST_BATCH_REPORT;
|
||||
process.env.ORKAS_TEST_BATCH_REPORT = report;
|
||||
try {
|
||||
const { default: Reporter } = await import('../../../scripts/checkpoint-reporter');
|
||||
const reporter = new Reporter();
|
||||
reporter.onError({ message: 'Collection failed: fixture dependency unavailable' });
|
||||
reporter.onError({ message: 'Cannot load /Users/me/private/customer.txt; token=controlled-secret https://example.invalid/?access_token=controlled-query' });
|
||||
for (let index = 0; index < 24; index++) reporter.onError({ message: 'x'.repeat(3000) });
|
||||
reporter.onBegin({ projects: [{ outputDir: root }] } as any, { allTests: () => [] } as any);
|
||||
reporter.onEnd({ status: 'failed' } as any);
|
||||
const result = JSON.parse(fs.readFileSync(report, 'utf8'));
|
||||
expect(result.unhandledErrors).toBe(26);
|
||||
expect(result.runnerErrors).toHaveLength(20);
|
||||
expect(result.runnerErrors[0]).toBe('Collection failed: fixture dependency unavailable');
|
||||
expect(result.runnerErrors[1]).toContain('Cannot load');
|
||||
expect(result.runnerErrors[1]).not.toMatch(/customer\.txt|controlled-secret|controlled-query|\/Users\/me/);
|
||||
expect(result.runnerErrors[2]).toHaveLength(2000);
|
||||
expect(result.omittedRunnerErrors).toBe(6);
|
||||
expect(result.reason).toBe('failed');
|
||||
} finally {
|
||||
if (previous === undefined) delete process.env.ORKAS_TEST_BATCH_REPORT;
|
||||
else process.env.ORKAS_TEST_BATCH_REPORT = previous;
|
||||
}
|
||||
});
|
||||
|
||||
function fixture(kind: 'python' | 'e2e') {
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'orkas-runner-resume-'));
|
||||
roots.push(root);
|
||||
@@ -28,6 +58,7 @@ function fixture(kind: 'python' | 'e2e') {
|
||||
}
|
||||
}
|
||||
write('test/e2e/fixtures/web-app-log-analysis.ts', fs.readFileSync(path.join(pcRoot, 'test/e2e/fixtures/web-app-log-analysis.ts'), 'utf8'));
|
||||
write('src/main/util/log-sanitize.ts', fs.readFileSync(path.join(pcRoot, 'src/main/util/log-sanitize.ts'), 'utf8'));
|
||||
write('node_modules/playwright/package.json', '{}');
|
||||
write('node_modules/playwright/cli.js', `require(${JSON.stringify(path.join(path.dirname(require.resolve('playwright/package.json')), 'cli.js'))});`);
|
||||
write('node_modules/playwright/lib/program.js', `module.exports = require(${JSON.stringify(require.resolve('playwright/lib/program'))});`);
|
||||
@@ -229,9 +260,9 @@ it.each(['electron-log-required', 'electron-logs'])(
|
||||
compilerOptions: { target: ts.ScriptTarget.ES2022 },
|
||||
}).outputText;
|
||||
const Owner = new Function('process', 'expect', 'path', 'mkdirSync', 'writeFileSync', 'rmSync',
|
||||
'unlinkSync', 'collectLogFiles', 'randomUUID', code + '; return DisposeOwner;')(
|
||||
'unlinkSync', 'collectLogFiles', 'randomUUID', 'analyzeWebAppLogs', code + '; return DisposeOwner;')(
|
||||
{ env: { ORKAS_TEST_BATCH_REPORT: 'fixture-report' } }, expect, path, fs.mkdirSync, fs.writeFileSync,
|
||||
fs.rmSync, fs.unlinkSync, () => '', () => 'snapshot');
|
||||
fs.rmSync, fs.unlinkSync, () => '', () => 'snapshot', analyzeWebAppLogs);
|
||||
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'orkas-checkpoint-dispose-'));
|
||||
roots.push(root);
|
||||
const appRoot = path.join(root, 'app');
|
||||
@@ -241,7 +272,8 @@ it.each(['electron-log-required', 'electron-logs'])(
|
||||
const failure = new Error('injected checkpoint attachment failure');
|
||||
const fixture = Object.assign(new Owner(), {
|
||||
root: appRoot, page: null, apiResponses: [], expectedApiFailures: [], cliStub: false,
|
||||
rendererPageErrors: [], recoveredRendererErrors: new Set(), diagnostics: ['[fixture-cleanup] completed'], tracePaths: [],
|
||||
rendererPageErrors: [], recoveredRendererErrors: new Set(), resourceDenials: new Map(), browserDenials: new Map(),
|
||||
expectedRendererFailures: new Map(), expectedInvokeFailures: [], diagnostics: ['[fixture-cleanup] completed'], tracePaths: [],
|
||||
closeCurrentApp: async () => { released.push('app'); },
|
||||
closeStubServer: async () => { released.push('stub'); },
|
||||
testInfo: {
|
||||
|
||||
@@ -55,29 +55,25 @@ const COMMANDER_COMMON_INITIAL_BUDGET_TOKENS = 6_500;
|
||||
// task-turn retention. Keep its cost separate so the existing corpus cannot grow.
|
||||
const BROWSER_DEFINITION_BUDGET_TOKENS = 1_000;
|
||||
|
||||
const BUILTIN_AGENT_INITIAL_SURFACE_BUDGETS: Readonly<Record<
|
||||
string,
|
||||
{ tools: number; tokens: number }
|
||||
>> = Object.freeze({
|
||||
ContentWriter: { tools: 22, tokens: 7_300 },
|
||||
DeepResearcher: { tools: 24, tokens: 7_300 },
|
||||
VideoStudio: { tools: 25, tokens: 9_500 },
|
||||
VoiceStudio: { tools: 23, tokens: 7_900 },
|
||||
PptMaker: { tools: 16, tokens: 8_100 },
|
||||
ImageStudio: { tools: 20, tokens: 8_400 },
|
||||
OfficeWorker: { tools: 21, tokens: 12_000 },
|
||||
ProductDeveloper: { tools: 25, tokens: 8_800 },
|
||||
UIDesigner: { tools: 25, tokens: 8_800 },
|
||||
SeoGeoAgent: { tools: 24, tokens: 8_500 },
|
||||
StockAnalyser: { tools: 22, tokens: 7_300 },
|
||||
// September 21 packaging migration: preserve fixed capabilities, adding
|
||||
// coverage for the five newly bundled Agents. Measured tokens are
|
||||
// 6,471/4,344/6,471/5,189/4,365; caps allow at most about 10% wording growth.
|
||||
ECommerceResearcher: { tools: 18, tokens: 7_100 },
|
||||
ECommerceReviewer: { tools: 11, tokens: 4_700 },
|
||||
ECommerceWriter: { tools: 18, tokens: 7_100 },
|
||||
ECommerceAnalyzer: { tools: 15, tokens: 5_700 },
|
||||
ECommerceOperator: { tools: 13, tokens: 4_800 },
|
||||
// Capability inventory remains bounded independently of initial token overhead.
|
||||
// agent-initial-overhead.test.ts owns the uniform, fully assembled request budget.
|
||||
const BUILTIN_AGENT_INITIAL_TOOL_COUNTS: Readonly<Record<string, number>> = Object.freeze({
|
||||
ContentWriter: 22,
|
||||
DeepResearcher: 24,
|
||||
VideoStudio: 25,
|
||||
VoiceStudio: 23,
|
||||
PptMaker: 16,
|
||||
ImageStudio: 20,
|
||||
OfficeWorker: 21,
|
||||
ProductDeveloper: 25,
|
||||
UIDesigner: 25,
|
||||
SeoGeoAgent: 24,
|
||||
StockAnalyser: 22,
|
||||
ECommerceResearcher: 18,
|
||||
ECommerceReviewer: 11,
|
||||
ECommerceWriter: 18,
|
||||
ECommerceAnalyzer: 15,
|
||||
ECommerceOperator: 13,
|
||||
});
|
||||
|
||||
describe('injected tool surface budget', () => {
|
||||
@@ -235,7 +231,7 @@ describe('injected tool surface budget', () => {
|
||||
}
|
||||
});
|
||||
|
||||
it('keeps each built-in Agent initial schema surface within budget', () => {
|
||||
it('retains reviewed built-in Agent initial schema inventories', () => {
|
||||
const previousMode = process.env.ORKAS_TOOL_LOADING_MODE;
|
||||
process.env.ORKAS_TOOL_LOADING_MODE = 'scoped';
|
||||
const measurements: string[] = [];
|
||||
@@ -258,17 +254,13 @@ describe('injected tool surface budget', () => {
|
||||
createToolLoadTool(surface),
|
||||
];
|
||||
const tokens = initialTools.reduce((sum, tool) => sum + definitionTokens(tool), 0);
|
||||
const budget = BUILTIN_AGENT_INITIAL_SURFACE_BUDGETS[scenario.name];
|
||||
const count = BUILTIN_AGENT_INITIAL_TOOL_COUNTS[scenario.name];
|
||||
|
||||
if (!budget) throw new Error(`${scenario.name} is missing an initial schema budget`);
|
||||
if (count === undefined) throw new Error(`${scenario.name} is missing an initial schema inventory`);
|
||||
expect(
|
||||
initialTools.length,
|
||||
`${scenario.name} sends ${initialTools.length} common-corpus schemas; budget ${budget.tools}`,
|
||||
).toBeLessThanOrEqual(budget.tools);
|
||||
expect(
|
||||
tokens,
|
||||
`${scenario.name} sends ${tokens} common-corpus schema tokens; budget ${budget.tokens}`,
|
||||
).toBeLessThanOrEqual(budget.tokens);
|
||||
`${scenario.name} sends ${initialTools.length} common-corpus schemas; reviewed count ${count}`,
|
||||
).toBeLessThanOrEqual(count);
|
||||
measurements.push(`${scenario.name}=${initialTools.length}/${tokens}`);
|
||||
}
|
||||
} finally {
|
||||
@@ -300,7 +292,7 @@ describe('injected tool surface budget', () => {
|
||||
});
|
||||
const regressedCount = availableTools.filter((tool) => regressed.isActive(tool.name)).length;
|
||||
|
||||
expect(regressedCount).toBeGreaterThan(BUILTIN_AGENT_INITIAL_SURFACE_BUDGETS.ContentWriter.tools);
|
||||
expect(regressedCount).toBeGreaterThan(BUILTIN_AGENT_INITIAL_TOOL_COUNTS.ContentWriter);
|
||||
} finally {
|
||||
if (previousMode === undefined) delete process.env.ORKAS_TOOL_LOADING_MODE;
|
||||
else process.env.ORKAS_TOOL_LOADING_MODE = previousMode;
|
||||
|
||||
Reference in new issue
Block a user