diff --git a/.boundaries-baseline.json b/.boundaries-baseline.json index ee2a2d16fb2..ee5357a106f 100644 --- a/.boundaries-baseline.json +++ b/.boundaries-baseline.json @@ -1,3 +1,3 @@ { - "issues": 141 + "issues": 141 } diff --git a/.gitignore b/.gitignore index edcd2941646..8b7b6488987 100644 --- a/.gitignore +++ b/.gitignore @@ -38,6 +38,7 @@ build.log tcr-dry.log .agent-setup/ .agent-recordings/ +.vitest-profile/ .pi-subagents/ sbom-source.cdx.json *.junit.xml diff --git a/package.json b/package.json index d5ee0cbf5a4..4e2211cbe0e 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,9 @@ "agent:test": "node scripts/agent-check.mjs test", "agent:playwright": "node scripts/agent-check.mjs playwright", "agent:harness": "node scripts/agent-harness.mjs", + "profile:vitest": "node scripts/vitest-profile.mjs", + "profile:ci-tests": "node scripts/ci-test-phase-profile.mjs", + "test:inventory": "node scripts/test-inventory.mjs", "format:check": "turbo run format:check", "lint": "turbo run lint", "lint:styles": "turbo run lint:styles", diff --git a/packages/@n8n/agents/vitest.integration.config.mjs b/packages/@n8n/agents/vitest.integration.config.mjs index 9a90dc4efff..3e0edc98329 100644 --- a/packages/@n8n/agents/vitest.integration.config.mjs +++ b/packages/@n8n/agents/vitest.integration.config.mjs @@ -2,6 +2,7 @@ import { dirname, resolve } from 'path'; import { fileURLToPath } from 'url'; import { config as loadEnv } from 'dotenv'; import { defineConfig } from 'vitest/config'; +import { profilingReporters } from '@n8n/vitest-config/node'; const __dirname = dirname(fileURLToPath(import.meta.url)); loadEnv({ path: resolve(__dirname, '.env'), quiet: true }); @@ -10,6 +11,7 @@ export default defineConfig({ test: { include: ['src/__tests__/integration/**/*.test.ts'], setupFiles: ['vitest.integration.setup.ts'], + reporters: profilingReporters(['default']), testTimeout: 120_000, hookTimeout: 30_000, }, diff --git a/packages/@n8n/vitest-config/frontend.ts b/packages/@n8n/vitest-config/frontend.ts index acb65de4fcb..bbb15c76dfe 100644 --- a/packages/@n8n/vitest-config/frontend.ts +++ b/packages/@n8n/vitest-config/frontend.ts @@ -4,6 +4,7 @@ import type { InlineConfig } from 'vitest/node'; import { changedFileCoverage } from './changed-file-coverage.js'; import { coverageExcludes } from './coverage-excludes.js'; +import { profilingConfig, profilingReporters } from './profiling.js'; // Resolves to the empty component that stands in for `.svg` imports (see below). const svgStub = fileURLToPath(new URL('./svg-stub.js', import.meta.url)); @@ -11,6 +12,7 @@ const svgStub = fileURLToPath(new URL('./svg-stub.js', import.meta.url)); export const createVitestConfig = (options: InlineConfig = {}) => { const vitestConfig = defineConfig({ test: { + ...profilingConfig(), // Redirect the node-icon `.svg` imports (the ~80 files under // `N8nIcon/nodes/`) to an inert component. `node-icons.ts` is imported // lazily (on the first `node:*` icon render), so its bulk async @@ -34,7 +36,7 @@ export const createVitestConfig = (options: InlineConfig = {}) => { // spies set up once don't leak across tests. Packages may override via `options`. restoreMocks: true, environment: 'jsdom', - // CI shards the frontend suite 2-way (`--shard=N/2`). A package with fewer + // CI shards the frontend suite across runners. A package with fewer // test files than shards leaves a shard with nothing to run, and vitest // treats that as an error — so a sparse package (a freshly scaffolded // module, a config-only package) would fail outright. Default it on here so @@ -45,7 +47,6 @@ export const createVitestConfig = (options: InlineConfig = {}) => { // Externalized, pnpm can link them to a second vitest copy. Vitest 5 bundles // `expect` into `vitest`, so a second copy breaks snapshots and `.rejects`. server: { deps: { inline: ['vitest-mock-extended', '@testing-library/jest-dom'] } }, - reporters: process.env.CI === 'true' ? ['default', 'junit'] : ['default'], outputFile: { junit: './junit.xml' }, coverage: { enabled: false, @@ -60,6 +61,9 @@ export const createVitestConfig = (options: InlineConfig = {}) => { }, }, ...options, + reporters: profilingReporters( + options.reporters ?? (process.env.CI === 'true' ? ['default', 'junit'] : ['default']), + ), }, }); diff --git a/packages/@n8n/vitest-config/node.ts b/packages/@n8n/vitest-config/node.ts index 55733a6178d..b2663c4d077 100644 --- a/packages/@n8n/vitest-config/node.ts +++ b/packages/@n8n/vitest-config/node.ts @@ -6,6 +6,9 @@ import type { InlineConfig } from 'vitest/node'; import { changedFileCoverage } from './changed-file-coverage.js'; import { coverageExcludes } from './coverage-excludes.js'; +import { profilingConfig, profilingReporters } from './profiling.js'; + +export { profilingReporters } from './profiling.js'; /** * Pin dual-build (ESM+CJS) deps to their CJS entry so a single class identity is shared @@ -52,6 +55,7 @@ export const forkPoolOptions = (): InlineConfig => { * Use this when you need to spread the config into workspace projects. */ export const createBaseInlineConfig = (options: InlineConfig = {}): InlineConfig => ({ + ...profilingConfig(), silent: true, globals: true, // Restore `vi.spyOn` spies to their original implementation before each test, so @@ -62,7 +66,6 @@ export const createBaseInlineConfig = (options: InlineConfig = {}): InlineConfig // Externalized, pnpm can link it to a second vitest copy, which breaks snapshot state. server: { deps: { inline: ['vitest-mock-extended'] } }, ...forkPoolOptions(), - reporters: process.env.CI === 'true' ? ['default', 'junit'] : ['default'], outputFile: { junit: './junit.xml' }, ...(process.env.COVERAGE_ENABLED === 'true' ? { @@ -78,6 +81,9 @@ export const createBaseInlineConfig = (options: InlineConfig = {}): InlineConfig } : {}), ...options, + reporters: profilingReporters( + options.reporters ?? (process.env.CI === 'true' ? ['default', 'junit'] : ['default']), + ), }); export const createVitestConfig = ( diff --git a/packages/@n8n/vitest-config/phase-reporter.ts b/packages/@n8n/vitest-config/phase-reporter.ts new file mode 100644 index 00000000000..222d1cc7fd6 --- /dev/null +++ b/packages/@n8n/vitest-config/phase-reporter.ts @@ -0,0 +1,21 @@ +import type { Reporter } from 'vitest/node'; + +export class PhaseReporter implements Reporter { + private firstModuleStart?: string; + private lastModuleEnd?: string; + + onTestModuleStart() { + this.firstModuleStart ??= new Date().toISOString(); + } + + onTestModuleEnd() { + this.lastModuleEnd = new Date().toISOString(); + } + + onTestRunEnd() { + if (!this.firstModuleStart || !this.lastModuleEnd) return; + process.stdout.write( + `N8N_VITEST_PHASE ${JSON.stringify({ version: 1, firstModuleStart: this.firstModuleStart, lastModuleEnd: this.lastModuleEnd })}\n`, + ); + } +} diff --git a/packages/@n8n/vitest-config/profiling.ts b/packages/@n8n/vitest-config/profiling.ts new file mode 100644 index 00000000000..47049c194d7 --- /dev/null +++ b/packages/@n8n/vitest-config/profiling.ts @@ -0,0 +1,24 @@ +import type { InlineConfig } from 'vitest/node'; + +import { PhaseReporter } from './phase-reporter.js'; + +export const profilingReporters = ( + reporters: NonNullable, +): NonNullable => + process.env.VITEST_PHASE_PROFILE === 'true' + ? [...(Array.isArray(reporters) ? reporters : [reporters]), new PhaseReporter()] + : reporters; + +export const profilingConfig = (): InlineConfig => { + const profileDir = process.env.VITEST_RUNNER_PROFILE_DIR; + const profileKinds = new Set(process.env.VITEST_RUNNER_PROFILE_KINDS?.split(',') ?? []); + if (!profileDir) return {}; + + const execArgv = [ + ...(profileKinds.has('cpu') ? ['--cpu-prof', `--cpu-prof-dir=${profileDir}`] : []), + ...(profileKinds.has('heap') ? ['--heap-prof', `--heap-prof-dir=${profileDir}`] : []), + ]; + + // Keep diagnostic profiles readable. These settings are only active through the local profiler. + return execArgv.length > 0 ? { execArgv, fileParallelism: false } : {}; +}; diff --git a/scripts/ci-test-phase-profile.mjs b/scripts/ci-test-phase-profile.mjs new file mode 100644 index 00000000000..60c02cf71ca --- /dev/null +++ b/scripts/ci-test-phase-profile.mjs @@ -0,0 +1,224 @@ +#!/usr/bin/env node + +import { execFile } from 'node:child_process'; +import { parseArgs, promisify } from 'node:util'; + +const exec = promisify(execFile); +const USAGE = `Usage: pnpm profile:ci-tests -- --run [flags] + +Profile the Unit tests jobs of a completed CI workflow run. Requires gh auth. +All percentages use each job's elapsed time as the denominator. + +Phases: + runnerSetup Job start through test step start (includes checkout, install, build) + testCommand Test step start through end (includes Vitest setup and tests) + teardown Test step end through job end (includes uploads and post actions) + +GitHub step timings cannot split Vitest setup from tests. Use the local +profile:vitest command for that split without changing CI. +Cache counts include all Turbo tasks in the test step, including dependencies. + +Flags: + --run Completed GitHub Actions workflow run ID + --repo GitHub repository (default: n8n-io/n8n) + --node-version Select a named Node-version matrix (master runs only) + --json Print a single JSON document + -h, --help Show this help +`; + +const { values } = parseArgs({ + args: process.argv[2] === '--' ? process.argv.slice(3) : process.argv.slice(2), + options: { + run: { type: 'string' }, + repo: { type: 'string', default: 'n8n-io/n8n' }, + 'node-version': { type: 'string' }, + json: { type: 'boolean', default: false }, + help: { type: 'boolean', short: 'h', default: false }, + }, + strict: true, +}); + +if (values.help) { + process.stdout.write(USAGE); + process.exit(0); +} +if (!/^\d+$/.test(values.run ?? '')) throw new Error('--run must be a GitHub Actions run ID'); +if (!/^[\w.-]+\/[\w.-]+$/.test(values.repo)) throw new Error('--repo must be owner/name'); + +async function ghApi(endpoint, log = false) { + const { stdout } = await exec( + 'gh', + ['api', ...(log ? ['--allow-escape-sequences'] : []), endpoint], + { maxBuffer: 32 * 1024 * 1024 }, + ); + return stdout; +} + +async function runJobs() { + const jobs = []; + for (let page = 1; ; page++) { + const response = JSON.parse( + await ghApi(`repos/${values.repo}/actions/runs/${values.run}/jobs?per_page=100&page=${page}`), + ); + jobs.push(...response.jobs); + if (response.jobs.length < 100) return jobs; + } +} + +function seconds(start, end) { + const elapsed = (Date.parse(end) - Date.parse(start)) / 1000; + if (!Number.isFinite(elapsed) || elapsed < 0) throw new Error('Invalid job step timestamps'); + return elapsed; +} + +function phaseProfile(job) { + const match = job.name.match( + /^Unit tests(?: \(([^)]+)\))? \/ (Backend Unit Tests|CLI Unit Tests(?: \(\d+\/\d+\))?|Backend Integration Tests|CLI Integration Tests(?: \(\d+\/\d+\))?|Nodes Unit Tests|Frontend \(\d+\/\d+\))$/, + ); + if (!match || (values['node-version'] && match[1] !== values['node-version'])) return null; + if (job.conclusion !== 'success') return null; + const test = job.steps.find((step) => + /^(?:Test$|Test Unit \(|Test Integration \(|Test Nodes$)/.test(step.name), + ); + if (!test || test.conclusion !== 'success') return null; + const totalSeconds = seconds(job.started_at, job.completed_at); + const phases = { + runnerSetup: seconds(job.started_at, test.started_at), + testCommand: seconds(test.started_at, test.completed_at), + teardown: seconds(test.completed_at, job.completed_at), + }; + return { + id: job.id, + name: job.name, + entryPoint: match[2].replace(/ \(\d+\/\d+\)$/, ''), + nodeVersion: match[1] ?? null, + totalSeconds, + phases, + phasePercent: Object.fromEntries( + Object.entries(phases).map(([name, duration]) => [name, (duration / totalSeconds) * 100]), + ), + testStep: { startedAt: test.started_at, completedAt: test.completed_at }, + }; +} + +function testCache(log, testStep) { + const clean = log.replace(/\u001B\[[0-?]*[ -/]*[@-~]/g, ''); + const summaries = []; + const startedAt = Date.parse(testStep.startedAt); + const completedAt = Date.parse(testStep.completedAt); + for (const line of clean.split(/\r?\n/)) { + const timestamp = line.match(/^(\d{4}-\d\d-\d\dT[^Z]+Z)/)?.[1]; + if (!timestamp) continue; + const time = Date.parse(timestamp); + if (time < startedAt - 1000 || time > completedAt + 1000) continue; + const cached = line.match(/Cached:\s+(\d+) cached, (\d+) total/); + if (cached) summaries.push({ cached: Number(cached[1]), total: Number(cached[2]) }); + } + return summaries.at(-1) ?? null; +} + +function testActivity(log, cache) { + if (cache?.total > 0 && cache.cached === cache.total) return 'fully-cached'; + const clean = log.replace(/\u001B\[[0-?]*[ -/]*[@-~]/g, ''); + if (/Test Files\s+\d+ (?:passed|failed)/.test(clean)) return 'test-results-reported'; + if (/\[janitor:test-scoped\].*skipping/.test(clean)) return 'skipped'; + if (/\[janitor:test-scoped\].*(?:→ full suite|scoping to [1-9]\d* file\(s\))/.test(clean)) + return 'test-run-invoked'; + return 'unknown'; +} + +function groupEntries(jobs) { + const groups = new Map(); + for (const job of jobs) { + const key = `${job.nodeVersion ?? 'workflow-default'}/${job.entryPoint}`; + const group = groups.get(key) ?? []; + group.push(job); + groups.set(key, group); + } + return [...groups.values()] + .map((group) => { + const slowest = group.reduce((a, b) => (a.totalSeconds >= b.totalSeconds ? a : b)); + return { + entryPoint: slowest.entryPoint, + nodeVersion: slowest.nodeVersion, + shards: group.length, + slowestJobId: slowest.id, + elapsedSeconds: slowest.totalSeconds, + phases: slowest.phases, + phasePercent: slowest.phasePercent, + cache: slowest.cache, + activity: slowest.activity, + }; + }) + .sort((a, b) => b.elapsedSeconds - a.elapsedSeconds); +} + +function table(entries) { + const column = (value, width) => String(value).padStart(width); + return [ + 'Node Entry point Jobs Total Runner setup Test command Teardown Turbo cache Activity', + ...entries.map((row) => + [ + (row.nodeVersion ?? 'default').padEnd(8), + row.entryPoint.padEnd(28), + column(row.shards, 4), + column(`${row.elapsedSeconds}s`, 6), + column(`${row.phasePercent.runnerSetup.toFixed(1)}%`, 13), + column(`${row.phasePercent.testCommand.toFixed(1)}%`, 13), + column(`${row.phasePercent.teardown.toFixed(1)}%`, 10), + row.cache ? `${row.cache.cached}/${row.cache.total}` : '?', + row.activity.padEnd(22), + ].join(' '), + ), + ].join('\n'); +} + +async function main() { + const run = JSON.parse(await ghApi(`repos/${values.repo}/actions/runs/${values.run}`)); + if (run.status !== 'completed') throw new Error('The workflow run must be complete'); + const jobs = (await runJobs()).map(phaseProfile).filter(Boolean); + if (!jobs.length) + throw new Error('No successful Unit tests jobs matched this run and Node version'); + for (const job of jobs) { + try { + const log = await ghApi(`repos/${values.repo}/actions/jobs/${job.id}/logs`, true); + job.cache = testCache(log, job.testStep); + job.activity = testActivity(log, job.cache); + } catch { + job.cache = null; + job.activity = 'unknown'; + } + } + const output = { + schemaVersion: 1, + source: `https://github.com/${values.repo}/actions/runs/${values.run}`, + run: { + id: run.id, + name: run.name, + createdAt: run.created_at, + conclusion: run.conclusion, + headSha: run.head_sha, + attempt: run.run_attempt, + }, + definitions: { + runnerSetup: + 'Job start to test command start (includes checkout, setup-nodejs, install, and Build)', + testCommand: 'Test step start to end (includes in-process setup, tests, and coverage)', + teardown: 'Test step end to job end (includes uploads and post actions)', + }, + entryPoints: groupEntries(jobs), + jobs, + }; + if (values.json) process.stdout.write(`${JSON.stringify(output)}\n`); + else { + process.stdout.write(`${table(output.entryPoints)}\n`); + process.stdout.write( + 'Percentages use the slowest shard per entry point. Test command includes Vitest setup.\n', + ); + } +} + +main().catch((error) => { + process.stderr.write(`ci-test-phase-profile: ${error.message}\n`); + process.exitCode = 1; +}); diff --git a/scripts/test-inventory-core.mjs b/scripts/test-inventory-core.mjs new file mode 100644 index 00000000000..c850ea9b1d9 --- /dev/null +++ b/scripts/test-inventory-core.mjs @@ -0,0 +1,259 @@ +import { spawn } from 'node:child_process'; +import { existsSync, readFileSync } from 'node:fs'; +import { isAbsolute, join, relative } from 'node:path'; + +const hasVitestScript = (packageJson, name) => + packageJson.scripts?.[name]?.includes('vitest') ?? false; + +export function configFromScript(script) { + return script.match(/--config(?:=|\s+)([^\s]+)/)?.[1]; +} + +export function unitTarget(packageJson, packageDir) { + const isFrontend = + packageDir.startsWith('packages/frontend/') || + /^packages\/modules\/[^/]+\/frontend$/.test(packageDir); + if (isFrontend && hasVitestScript(packageJson, 'test')) { + const changed = Boolean(packageJson.scripts?.['test:changed']); + return { + lane: changed ? 'frontend-unit' : 'frontend-nightly', + scope: changed ? 'changed' : 'nightly only', + script: 'test', + }; + } + if (packageJson.name === 'n8n' && hasVitestScript(packageJson, 'test:unit')) { + return { lane: 'cli-unit', scope: 'changed', script: 'test:unit' }; + } + if (packageJson.name === 'n8n-nodes-base' && hasVitestScript(packageJson, 'test')) { + return { lane: 'nodes-unit', scope: 'changed', script: 'test' }; + } + if (hasVitestScript(packageJson, 'test:unit')) { + return { lane: 'backend-unit', scope: 'full', script: 'test:unit' }; + } + if (hasVitestScript(packageJson, 'test')) { + return { lane: 'not-in-unit-ci', scope: 'not run', script: 'test' }; + } + return undefined; +} + +export function integrationTarget(packageJson) { + if (!hasVitestScript(packageJson, 'test:integration')) return undefined; + return { + lane: packageJson.name === 'n8n' ? 'cli-integration' : 'backend-integration', + scope: packageJson.name === 'n8n' ? 'changed' : 'full', + script: 'test:integration', + }; +} + +function filePath(file, target, repoRoot) { + const path = file.replace(/^\[[^\]]+\]\s+/, ''); + return relative(repoRoot, isAbsolute(path) ? path : join(repoRoot, target.dir, path)); +} + +function listedFiles(output, target, repoRoot) { + return [ + ...new Set( + output + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => /\.(?:test|spec)\.[cm]?[jt]sx?$/.test(line)) + .map((line) => filePath(line, target, repoRoot)), + ), + ] + .sort() + .map((path) => ({ path, tests: null })); +} + +function leafCount(tasks) { + let count = 0; + for (const task of tasks) { + if (Array.isArray(task.tasks)) count += leafCount(task.tasks); + else if (task.type !== 'suite') count++; + } + return count; +} + +function countDynamicTests(serialized, target, repoRoot) { + let tests; + try { + tests = JSON.parse(serialized); + } catch { + const start = serialized.lastIndexOf('\n['); + const end = serialized.lastIndexOf(']'); + if (start === -1 || end === -1) throw new Error('Vitest did not produce JSON'); + tests = JSON.parse(serialized.slice(start + 1, end + 1)); + } + if (!Array.isArray(tests)) throw new Error('Vitest did not produce a test list'); + const counts = new Map(); + for (const test of tests) { + if (typeof test.file !== 'string') throw new Error('Vitest test has no file'); + const path = filePath(test.file, target, repoRoot); + const count = Array.isArray(test.tasks) ? leafCount(test.tasks) : 1; + counts.set(path, (counts.get(path) ?? 0) + count); + } + const fileCounts = [...counts] + .sort(([a], [b]) => a.localeCompare(b)) + .map(([path, count]) => ({ path, tests: count })); + return { + files: fileCounts.length, + fileCounts, + tests: [...counts.values()].reduce((a, b) => a + b, 0), + }; +} + +export function collectTarget(target, index, { outputDir, dynamic, timeoutSeconds, repoRoot }) { + return new Promise((resolve) => { + const outputFile = join(outputDir, `${index}.json`); + const env = { + ...process.env, + CI: 'true', + COVERAGE_ENABLED: 'false', + DB_SQLITE_POOL_SIZE: '4', + DB_TYPE: 'sqlite', + N8N_LOG_LEVEL: 'silent', + TZ: 'UTC', + }; + let activePid; + let settled = false; + const finish = (result) => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolve({ ...target, ...result }); + }; + const timer = setTimeout(() => { + try { + if (activePid) process.kill(-activePid, 'SIGKILL'); + } catch {} + finish({ error: `timed out after ${timeoutSeconds}s` }); + }, timeoutSeconds * 1000); + const runList = (filesOnly) => { + const args = [ + 'exec', + 'vitest', + 'list', + ...(filesOnly ? ['--filesOnly'] : ['--staticParse=false', `--json=${outputFile}`]), + '--passWithNoTests', + '--no-color', + ...(target.config ? ['--config', target.config] : []), + ]; + const child = spawn('pnpm', args, { + cwd: join(repoRoot, target.dir), + detached: true, + env, + stdio: ['ignore', 'pipe', 'pipe'], + }); + activePid = child.pid; + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (data) => (stdout += data)); + child.stderr.on('data', (data) => (stderr += data)); + child.on('error', (error) => finish({ error: error.message })); + child.on('close', (code) => { + if (settled) return; + if (code !== 0) { + finish({ error: `exited with ${code}: ${stderr.trim().slice(-500)}` }); + return; + } + if (filesOnly) { + const fileCounts = listedFiles(stdout, target, repoRoot); + finish({ + files: fileCounts.length, + fileCounts, + tests: null, + ...(dynamic && { + warning: + 'Vitest dynamic parsing could not count this package. File count is available.', + }), + }); + return; + } + try { + const serialized = existsSync(outputFile) ? readFileSync(outputFile, 'utf8') : stdout; + finish(countDynamicTests(serialized, target, repoRoot)); + } catch { + runList(true); + } + }); + }; + if (dynamic && target.package === '@n8n/db') { + const child = spawn('pnpm', ['gen:migration-index'], { + cwd: join(repoRoot, target.dir), + detached: true, + env, + stdio: ['ignore', 'pipe', 'pipe'], + }); + activePid = child.pid; + let stderr = ''; + child.stdout.resume(); + child.stderr.on('data', (data) => (stderr += data)); + child.on('error', (error) => finish({ error: error.message })); + child.on('close', (code) => { + if (settled) return; + if (code !== 0) { + finish({ + error: `migration index generation exited with ${code}: ${stderr.trim().slice(-500)}`, + }); + return; + } + runList(false); + }); + } else { + runList(!dynamic); + } + }); +} + +export function aggregateInventory(results, dynamic) { + const ordered = [...results].sort( + (a, b) => + a.lane.localeCompare(b.lane) || + (b.tests ?? -1) - (a.tests ?? -1) || + a.package.localeCompare(b.package), + ); + const laneMap = new Map(); + for (const result of ordered) { + const lane = laneMap.get(result.lane) ?? { + lane: result.lane, + packages: 0, + files: 0, + tests: dynamic ? 0 : null, + errors: 0, + missingCaseCounts: 0, + }; + lane.packages++; + lane.files += result.files ?? 0; + if (dynamic) lane.tests += result.tests ?? 0; + lane.errors += result.error ? 1 : 0; + lane.missingCaseCounts += dynamic && result.tests == null ? 1 : 0; + laneMap.set(result.lane, lane); + } + const lanes = [...laneMap.values()].sort((a, b) => a.lane.localeCompare(b.lane)); + const prResults = ordered.filter(({ scope }) => scope !== 'nightly only' && scope !== 'not run'); + const prLanes = lanes.filter( + ({ lane }) => lane !== 'frontend-nightly' && lane !== 'not-in-unit-ci', + ); + const missingCaseCounts = dynamic + ? prLanes.reduce((total, lane) => total + lane.missingCaseCounts, 0) + : 0; + return { + schemaVersion: 1, + mode: dynamic ? 'dynamic' : 'files', + note: dynamic + ? 'Dynamic mode counts configured test cases. Change-scoped PR CI jobs may run fewer tests.' + : 'File mode counts configured test files. Change-scoped PR CI jobs may run fewer files. Use --dynamic for case counts.', + ci: { + packages: new Set(prResults.map((result) => result.package)).size, + targets: prResults.length, + files: prLanes.reduce((total, lane) => total + lane.files, 0), + tests: + dynamic && missingCaseCounts === 0 + ? prLanes.reduce((total, lane) => total + lane.tests, 0) + : null, + missingCaseCounts, + errors: prLanes.reduce((total, lane) => total + lane.errors, 0), + }, + lanes, + packages: ordered, + }; +} diff --git a/scripts/test-inventory.mjs b/scripts/test-inventory.mjs new file mode 100644 index 00000000000..b447e30943a --- /dev/null +++ b/scripts/test-inventory.mjs @@ -0,0 +1,166 @@ +#!/usr/bin/env node + +import { globSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { parseArgs } from 'node:util'; +import { fileURLToPath } from 'node:url'; + +import { + aggregateInventory, + collectTarget, + configFromScript, + integrationTarget, + unitTarget, +} from './test-inventory-core.mjs'; + +const REPO_ROOT = fileURLToPath(new URL('..', import.meta.url)); +const USAGE = `Usage: pnpm test:inventory -- [flags] + +Lists the Vitest suites that unit and integration CI jobs select. +File mode lists configured test files. Dynamic mode imports modules and counts cases. +Change-scoped PR jobs may run fewer tests than the configured totals. + +Flags: + --dynamic Import test modules and expand generated test cases + --package Only inspect one workspace package + --json Print JSON instead of a table + --concurrency Package collections to run at once (default: 4) + --timeout Timeout for each package (default: 60 files, 600 dynamic) + -h, --help Show this help +`; + +const { values } = parseArgs({ + args: process.argv[2] === '--' ? process.argv.slice(3) : process.argv.slice(2), + options: { + concurrency: { type: 'string', default: '4' }, + dynamic: { type: 'boolean', default: false }, + help: { type: 'boolean', short: 'h', default: false }, + json: { type: 'boolean', default: false }, + package: { type: 'string' }, + timeout: { type: 'string' }, + }, + strict: true, +}); + +if (values.help) { + process.stdout.write(USAGE); + process.exit(0); +} + +function positiveInteger(value, flag) { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed <= 0) throw new Error(`${flag} must be positive`); + return parsed; +} + +const concurrency = positiveInteger(values.concurrency, '--concurrency'); +const timeoutSeconds = positiveInteger( + values.timeout ?? (values.dynamic ? '600' : '60'), + '--timeout', +); + +let targets = []; +for (const packageFile of globSync('packages/**/package.json', { + cwd: REPO_ROOT, + exclude: ['**/dist/**', '**/node_modules/**'], +})) { + const packageDir = dirname(packageFile); + const packageJson = JSON.parse(readFileSync(join(REPO_ROOT, packageFile), 'utf8')); + for (const target of [unitTarget(packageJson, packageDir), integrationTarget(packageJson)]) { + if (!target) continue; + targets.push({ + ...target, + config: configFromScript(packageJson.scripts[target.script]), + dir: packageDir, + package: packageJson.name, + }); + } +} +if (values.package) targets = targets.filter((target) => target.package === values.package); +if (!targets.length) { + const error = 'No matching Vitest package or CI target was found'; + if (values.json) { + await new Promise((resolve) => + process.stdout.write(`${JSON.stringify({ schemaVersion: 1, error })}\n`, resolve), + ); + } else { + process.stderr.write(`${error}\n`); + } + process.exit(1); +} + +const outputDir = mkdtempSync(join(tmpdir(), 'n8n-test-inventory-')); +const results = []; +let next = 0; +try { + await Promise.all( + Array.from({ length: Math.min(concurrency, targets.length) }, async () => { + while (next < targets.length) { + const index = next++; + results[index] = await collectTarget(targets[index], index, { + outputDir, + dynamic: values.dynamic, + timeoutSeconds, + repoRoot: REPO_ROOT, + }); + } + }), + ); +} finally { + rmSync(outputDir, { recursive: true, force: true }); +} + +const output = aggregateInventory(results, values.dynamic); +const incomplete = results.some( + (result) => result.error || (values.dynamic && result.tests === null), +); +if (values.json) { + await new Promise((resolve) => + process.stdout.write(`${JSON.stringify(output, null, 2)}\n`, resolve), + ); + process.exit(incomplete ? 1 : 0); +} + +function table(rows, columns) { + const widths = columns.map(({ header, value }) => + Math.max(header.length, ...rows.map((row) => String(value(row)).length)), + ); + const line = (cells) => + cells.map((value, index) => String(value).padEnd(widths[index])).join(' '); + return [ + line(columns.map(({ header }) => header)), + line(widths.map((width) => '-'.repeat(width))), + ...rows.map((row) => line(columns.map(({ value }) => value(row)))), + ].join('\n'); +} + +process.stdout.write(`${output.note}\n\n`); +const laneColumns = [ + { header: 'CI lane', value: (row) => row.lane }, + { header: 'Packages', value: (row) => row.packages }, + { header: 'Files', value: (row) => row.files }, + ...(values.dynamic + ? [ + { header: 'Expanded cases', value: (row) => row.tests }, + { header: 'Missing counts', value: (row) => row.missingCaseCounts || '' }, + ] + : []), + { header: 'Errors', value: (row) => row.errors || '' }, +]; +process.stdout.write(`${table(output.lanes, laneColumns)}\n\n`); +const packageColumns = [ + { header: 'Package', value: (row) => row.package }, + { header: 'CI lane', value: (row) => row.lane }, + { header: 'CI scope', value: (row) => row.scope }, + { header: 'Files', value: (row) => row.files ?? '-' }, + ...(values.dynamic ? [{ header: 'Expanded cases', value: (row) => row.tests ?? '-' }] : []), +]; +process.stdout.write(`${table(output.packages, packageColumns)}\n`); + +const errors = results.filter((result) => result.error); +if (errors.length) { + process.stderr.write('\nCollection errors:\n'); + for (const error of errors) process.stderr.write(`- ${error.package}: ${error.error}\n`); +} +process.exitCode = incomplete ? 1 : 0; diff --git a/scripts/vitest-phase-markers.mjs b/scripts/vitest-phase-markers.mjs new file mode 100644 index 00000000000..c53c54c4d17 --- /dev/null +++ b/scripts/vitest-phase-markers.mjs @@ -0,0 +1,31 @@ +/** Find the active test-module window in Vitest reporter output. */ +export function vitestWindow(log, start, end) { + const startedAt = new Date(start).getTime(); + const completedAt = new Date(end).getTime(); + const markers = []; + for (const line of log.replace(/\u001B\[[0-?]*[ -/]*[@-~]/g, '').split(/\r?\n/)) { + const payload = line.match(/N8N_VITEST_PHASE (\{[^\r\n]+\})/)?.[1]; + if (!payload) continue; + try { + const marker = JSON.parse(payload); + const first = Date.parse(marker.firstModuleStart); + const last = Date.parse(marker.lastModuleEnd); + if ( + marker.version === 1 && + Number.isFinite(first) && + Number.isFinite(last) && + first >= startedAt - 1000 && + last <= completedAt + 1000 && + first <= last + ) { + markers.push({ first, last }); + } + } catch { + // Ignore malformed or unrelated log lines. + } + } + if (!markers.length) return null; + const first = Math.max(startedAt, Math.min(...markers.map((marker) => marker.first))); + const last = Math.min(completedAt, Math.max(...markers.map((marker) => marker.last))); + return last < first ? null : { first, last, processes: markers.length }; +} diff --git a/scripts/vitest-profile.mjs b/scripts/vitest-profile.mjs new file mode 100644 index 00000000000..ee833e03474 --- /dev/null +++ b/scripts/vitest-profile.mjs @@ -0,0 +1,697 @@ +#!/usr/bin/env node +/** Profile Vitest shards without requiring CI or a remote cache. */ +import { spawn, execFile, execFileSync } from 'node:child_process'; +import { createWriteStream, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { readFile } from 'node:fs/promises'; +import { dirname, join, resolve } from 'node:path'; +import { parseArgs } from 'node:util'; + +import { vitestWindow } from './vitest-phase-markers.mjs'; + +const REPO_ROOT = resolve(import.meta.dirname, '..'); +const DEFAULT_OUTPUT_DIR = resolve(REPO_ROOT, '.vitest-profile'); + +const USAGE = `Usage: pnpm profile:vitest -- [flags] + +Runs each shard sequentially so every measurement models a dedicated runner. +Full-suite estimates are available only when all shards complete successfully. + +Flags: + --scope Suite group to profile (default: editor) + --package Workspace package name for --scope package + --task Package test script (default: test) + --file Test file filter, relative to the package + --runner Direct Vitest or exact Turbo path (default: direct for one package) + --shards Comma-separated shard totals (default: 2,4) + --indexes Shard indexes to run for each total (default: all; partial results are samples) + --coverage Coverage modes to measure (default: both) + --workers Vitest max workers per runner (default: 4) + --turbo-concurrency Turbo package-task concurrency (default: 10) + --runs Repetitions of each case (default: 1) + --sample-ms Process resource sample interval (default: 1000) + --profile none, imports, coverage, cpu, heap, or all (default: none) + --skip-prepare Do not warm required build outputs before measuring + --allow-concurrent Run even when another Vitest process is active + --continue-on-failure Measure all requested runs after a test failure + --output Logs and summary output (default: .vitest-profile) + --dry-run Print commands without running them + --json Print only the final JSON summary + -h, --help Show this help + +Examples: + pnpm profile:vitest -- --shards 2,4 --coverage both + pnpm profile:vitest -- --scope package --package n8n-core --task test:unit --shards 4 + pnpm profile:vitest -- --scope package --package n8n-core --file src/example.test.ts --shards 1 + pnpm profile:vitest -- --shards 4 --indexes 1 --coverage on --profile coverage + pnpm profile:vitest -- --shards 4 --indexes 1 --coverage off --profile cpu +`; + +function fail(message) { + process.stderr.write(`vitest-profile: ${message}\n`); + process.exit(2); +} + +function positiveInteger(value, flag) { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed <= 0) fail(`${flag} must be a positive integer`); + return parsed; +} + +function integerList(value, flag) { + const parsed = value.split(',').map((item) => positiveInteger(item.trim(), flag)); + return [...new Set(parsed)]; +} + +function formatSeconds(seconds) { + if (seconds < 60) return `${seconds.toFixed(1)}s`; + const minutes = Math.floor(seconds / 60); + return `${minutes}m${String(Math.round(seconds % 60)).padStart(2, '0')}s`; +} + +function formatMb(kb) { + return `${Math.round(kb / 1024)}MB`; +} + +function activeVitestProcesses() { + try { + return execFileSync('ps', ['-axo', 'pid=,command='], { encoding: 'utf8' }) + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => /\bvitest(?:\.mjs)?\s+(?:run|list|watch|dev|related)\b/.test(line)); + } catch { + return []; + } +} + +function stripAnsi(value) { + return value.replace(/\u001B\[[0-?]*[ -/]*[@-~]/g, ''); +} + +function parseDuration(value, unit) { + const number = Number(value); + if (unit === 'ms') return number / 1000; + if (unit === 'm') return number * 60; + return number; +} + +function parseVitestSummaries(log) { + const clean = stripAnsi(log); + const lines = clean.split(/\r?\n/); + const summaries = []; + let current = {}; + + for (const line of lines) { + const files = + line.match(/Test Files\s+.*?\((\d+)\)/) ?? line.match(/Test Files\s+(\d+) passed/); + if (files) current.testFiles = Number(files[1]); + const tests = line.match(/Tests\s+.*?\((\d+)\)/) ?? line.match(/Tests\s+(\d+) passed/); + if (tests) current.tests = Number(tests[1]); + const duration = line.match(/Duration\s+([\d.]+)(ms|s|m)(?:\s|$)/); + if (duration) { + current.durationSeconds = parseDuration(duration[1], duration[2]); + summaries.push(current); + current = {}; + } + } + + return summaries; +} + +function parseTurboSummary(log) { + const clean = stripAnsi(log); + const tasks = [...clean.matchAll(/Tasks:\s+(\d+) successful, (\d+) total/g)].at(-1); + const cached = [...clean.matchAll(/Cached:\s+(\d+) cached, (\d+) total/g)].at(-1); + const time = [...clean.matchAll(/Time:\s+([^\r\n]+)/g)].at(-1); + return { + ...(tasks && { successfulTasks: Number(tasks[1]), totalTasks: Number(tasks[2]) }), + ...(cached && { cachedTasks: Number(cached[1]) }), + ...(time && { reportedTime: time[1].trim() }), + }; +} + +function getProcessRows() { + return new Promise((resolveRows) => { + execFile( + 'ps', + ['-axo', 'pid=,ppid=,%cpu=,rss='], + { maxBuffer: 8 * 1024 * 1024 }, + (error, stdout) => { + if (error) return resolveRows([]); + const rows = stdout + .split(/\r?\n/) + .map((line) => line.trim().split(/\s+/)) + .filter((parts) => parts.length === 4) + .map(([pid, ppid, cpu, rss]) => ({ + pid: Number(pid), + ppid: Number(ppid), + cpu: Number(cpu), + rssKb: Number(rss), + })); + resolveRows(rows); + }, + ); + }); +} + +function aggregateProcessTree(rows, rootPid) { + const descendants = new Set([rootPid]); + let changed = true; + while (changed) { + changed = false; + for (const row of rows) { + if (descendants.has(row.ppid) && !descendants.has(row.pid)) { + descendants.add(row.pid); + changed = true; + } + } + } + + return rows.reduce( + (total, row) => { + if (!descendants.has(row.pid)) return total; + total.cpuPercent += row.cpu; + total.rssKb += row.rssKb; + total.processes += 1; + return total; + }, + { cpuPercent: 0, rssKb: 0, processes: 0 }, + ); +} + +function nodeProfileArgs(profileKinds, profileDir, target) { + const args = []; + if (profileKinds.includes('cpu')) { + args.push('--cpu-prof', `--cpu-prof-dir=${profileDir}`, `--cpu-prof-name=${target}.cpuprofile`); + } + if (profileKinds.includes('heap')) { + args.push( + '--heap-prof', + `--heap-prof-dir=${profileDir}`, + `--heap-prof-name=${target}.heapprofile`, + ); + } + return args; +} + +function vitestArgs(testCase, options) { + return [ + ...(options.file ? [options.file] : []), + ...(options.directTask?.args.includes('--passWithNoTests') ? [] : ['--passWithNoTests']), + `--shard=${testCase.index}/${testCase.shards}`, + `--maxWorkers=${options.workers}`, + ...(options.profileKinds.includes('imports') ? ['--experimental.importDurations.print'] : []), + ]; +} + +function directTask(packageDir, task) { + const { scripts } = JSON.parse(readFileSync(resolve(packageDir, 'package.json'), 'utf8')); + const tokens = scripts?.[task]?.trim().split(/\s+/) ?? []; + const vitestIndex = tokens.indexOf('vitest'); + if (vitestIndex < 0 || tokens[vitestIndex + 1] !== 'run') { + fail(`--runner direct requires a "vitest run" script; ${task} is not one. Use --runner turbo.`); + } + const prefix = tokens.slice(0, vitestIndex); + if (prefix[0] === 'cross-env') prefix.shift(); + if (prefix.some((token) => !/^[A-Za-z_][A-Za-z_0-9]*=[^\s]+$/.test(token))) { + fail(`cannot reproduce the environment of ${task} in direct mode; use --runner turbo`); + } + const args = tokens.slice(vitestIndex + 2); + if (args.some((token) => /^--(?:shard|maxWorkers|coverage)(?:[.=]|$)/.test(token))) { + fail(`--task ${task} sets a profiler-owned Vitest flag; use --runner turbo`); + } + return { + args, + env: Object.fromEntries( + prefix.map((entry) => { + const separator = entry.indexOf('='); + return [entry.slice(0, separator), entry.slice(separator + 1)]; + }), + ), + }; +} + +function commandForCase(testCase, options, profileDir) { + if (options.runner === 'direct') { + return { + command: process.execPath, + args: [ + ...nodeProfileArgs(options.profileKinds, resolve(profileDir, 'main'), 'vitest-main'), + options.vitestEntry, + 'run', + ...options.directTask.args, + ...vitestArgs(testCase, options), + ], + cwd: options.packageDir, + }; + } + + const filters = + options.scope === 'editor' + ? ['--filter=n8n-editor-ui'] + : options.scope === 'frontend' + ? ['--filter=./packages/frontend/**', '--filter=./packages/modules/*/frontend'] + : [`--filter=${options.package}`]; + return { + command: 'pnpm', + args: [ + 'exec', + 'turbo', + 'run', + options.task, + '--continue', + '--only', + '--force', + '--env-mode=loose', + ...filters, + `--concurrency=${options.turboConcurrency}`, + '--summarize', + '--', + ...vitestArgs(testCase, options), + ], + cwd: REPO_ROOT, + }; +} + +function prepareBuild(options) { + const filters = + options.scope === 'editor' + ? ['--filter=n8n-editor-ui'] + : options.scope === 'frontend' + ? ['--filter=./packages/frontend/**', '--filter=./packages/modules/*/frontend'] + : [`--filter=${options.package}`]; + const command = 'pnpm'; + const args = ['exec', 'turbo', 'run', 'build', ...filters, '--summarize']; + if (options.dryRun) { + if (!options.json) process.stdout.write(`[prepare] ${command} ${args.join(' ')}\n`); + return Promise.resolve({ skipped: false, dryRun: true }); + } + + const logPath = resolve(options.outputDir, 'prepare.log'); + const logStream = createWriteStream(logPath, { flags: 'w' }); + const startedAt = Date.now(); + if (!options.json) process.stdout.write('[prepare] warming build outputs... '); + const child = spawn(command, args, { + cwd: REPO_ROOT, + env: { ...process.env, CI: 'true', FORCE_COLOR: '0' }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + child.stdout.pipe(logStream, { end: false }); + child.stderr.pipe(logStream, { end: false }); + + return new Promise((resolvePrepare, rejectPrepare) => { + child.once('error', rejectPrepare); + child.once('close', (exitCode, signal) => { + logStream.end(); + const result = { + skipped: false, + exitCode, + signal, + wallSeconds: (Date.now() - startedAt) / 1000, + log: logPath, + }; + if (!options.json) { + process.stdout.write( + exitCode === 0 + ? `ok ${formatSeconds(result.wallSeconds)}\n` + : `failed (${signal ?? exitCode})\n`, + ); + } + resolvePrepare(result); + }); + }); +} + +async function runCase(testCase, options) { + const label = [ + options.suiteLabel, + `${testCase.index}-of-${testCase.shards}`, + `coverage-${testCase.coverage ? 'on' : 'off'}`, + `workers-${options.workers}`, + `run-${testCase.run}`, + ].join('-'); + const logPath = resolve(options.outputDir, `${label}.log`); + const profileDir = resolve(options.outputDir, `${label}-profiles`); + const { command, args, cwd } = commandForCase(testCase, options, profileDir); + const invocation = { bin: command, args, cwd }; + + if (options.dryRun) { + return { ...testCase, label, command: invocation, dryRun: true }; + } + + mkdirSync(resolve(profileDir, 'main'), { recursive: true }); + mkdirSync(resolve(profileDir, 'runner'), { recursive: true }); + const logStream = createWriteStream(logPath, { flags: 'w' }); + const startedAt = Date.now(); + const child = spawn(command, args, { + cwd, + env: { + ...process.env, + ...(options.directTask?.env ?? {}), + AFFECTED_PACKAGES: '', + CHANGED_FILES: '', + CI: 'true', + COVERAGE_ENABLED: testCase.coverage ? 'true' : 'false', + ...(options.profileKinds.includes('coverage') + ? { DEBUG: [process.env.DEBUG, 'vitest:coverage'].filter(Boolean).join(',') } + : {}), + FORCE_COLOR: '0', + VITEST_PHASE_PROFILE: 'true', + VITEST_RUNNER_PROFILE_DIR: resolve(profileDir, 'runner'), + VITEST_RUNNER_PROFILE_KINDS: options.profileKinds.join(','), + }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + + child.stdout.pipe(logStream, { end: false }); + child.stderr.pipe(logStream, { end: false }); + + const resources = { peakCpuPercent: 0, peakRssKb: 0, peakProcesses: 0, samples: 0 }; + let sampling = false; + const sample = async () => { + if (sampling || child.exitCode !== null) return; + sampling = true; + const aggregate = aggregateProcessTree(await getProcessRows(), child.pid); + resources.peakCpuPercent = Math.max(resources.peakCpuPercent, aggregate.cpuPercent); + resources.peakRssKb = Math.max(resources.peakRssKb, aggregate.rssKb); + resources.peakProcesses = Math.max(resources.peakProcesses, aggregate.processes); + resources.samples += 1; + sampling = false; + }; + const sampler = setInterval(sample, options.sampleMs); + await sample(); + + const outcome = await new Promise((resolveOutcome) => { + child.once('error', (error) => + resolveOutcome({ exitCode: null, signal: null, error: error.message }), + ); + child.once('close', (exitCode, signal) => resolveOutcome({ exitCode, signal })); + }); + clearInterval(sampler); + await sample(); + logStream.end(); + await new Promise((resolveEnd) => logStream.once('finish', resolveEnd)); + const endedAt = Date.now(); + + const log = await readFile(logPath, 'utf8'); + const window = vitestWindow(log, startedAt, endedAt); + const vitestPhases = window + ? { + setupSeconds: (window.first - startedAt) / 1000, + runSeconds: (window.last - window.first) / 1000, + teardownSeconds: (endedAt - window.last) / 1000, + processes: window.processes, + } + : null; + const vitest = parseVitestSummaries(log); + const slowestVitest = vitest.reduce( + (slowest, summary) => + summary.durationSeconds > (slowest?.durationSeconds ?? 0) ? summary : slowest, + null, + ); + + return { + ...testCase, + ...outcome, + label, + command: invocation, + log: logPath, + profileDir, + wallSeconds: (endedAt - startedAt) / 1000, + vitestPhases, + resources, + turbo: parseTurboSummary(log), + vitest, + slowestVitest, + }; +} + +function aggregateResults(results) { + const groups = new Map(); + for (const result of results.filter((item) => !item.dryRun)) { + const key = `${result.shards}:${result.coverage}:${result.run}`; + const group = groups.get(key) ?? { + shards: result.shards, + coverage: result.coverage, + run: result.run, + results: [], + }; + group.results.push(result); + groups.set(key, group); + } + + return [...groups.values()].map((group) => { + const complete = + group.results.length === group.shards && group.results.every((r) => r.exitCode === 0); + const sampledWallSeconds = Math.max(...group.results.map((result) => result.wallSeconds)); + const sampledRunnerSeconds = group.results.reduce((sum, result) => sum + result.wallSeconds, 0); + return { + shards: group.shards, + coverage: group.coverage, + run: group.run, + measuredShards: group.results.length, + complete, + sampledWallSeconds, + sampledRunnerSeconds, + predictedWallSeconds: complete ? sampledWallSeconds : null, + totalRunnerSeconds: complete ? sampledRunnerSeconds : null, + peakRssKb: Math.max(...group.results.map((result) => result.resources.peakRssKb)), + peakCpuPercent: Math.max(...group.results.map((result) => result.resources.peakCpuPercent)), + }; + }); +} + +function printResult(result) { + const vitest = result.slowestVitest; + const details = [ + formatSeconds(result.wallSeconds), + vitest?.testFiles ? `${vitest.testFiles} files` : 'files unknown', + vitest?.durationSeconds ? `vitest ${formatSeconds(vitest.durationSeconds)}` : 'vitest unknown', + `peak ${formatMb(result.resources.peakRssKb)}`, + `CPU ${Math.round(result.resources.peakCpuPercent)}%`, + ]; + process.stdout.write(`${result.exitCode === 0 ? 'ok' : 'failed'} ${details.join(', ')}\n`); +} + +let values; +try { + ({ values } = parseArgs({ + args: process.argv[2] === '--' ? process.argv.slice(3) : process.argv.slice(2), + options: { + scope: { type: 'string', default: 'editor' }, + package: { type: 'string' }, + task: { type: 'string' }, + file: { type: 'string' }, + runner: { type: 'string' }, + shards: { type: 'string', default: '2,4' }, + indexes: { type: 'string', default: 'all' }, + coverage: { type: 'string', default: 'both' }, + workers: { type: 'string', default: '4' }, + 'turbo-concurrency': { type: 'string', default: '10' }, + runs: { type: 'string', default: '1' }, + 'sample-ms': { type: 'string', default: '1000' }, + profile: { type: 'string', default: 'none' }, + 'skip-prepare': { type: 'boolean', default: false }, + 'allow-concurrent': { type: 'boolean', default: false }, + 'continue-on-failure': { type: 'boolean', default: false }, + output: { type: 'string' }, + 'dry-run': { type: 'boolean', default: false }, + json: { type: 'boolean', default: false }, + help: { type: 'boolean', default: false, short: 'h' }, + }, + strict: true, + })); +} catch (error) { + fail(error.message); +} + +if (values.help) { + process.stdout.write(USAGE); + process.exit(0); +} +if (!['editor', 'frontend', 'package'].includes(values.scope)) { + fail('--scope must be editor, frontend, or package'); +} +if (values.scope === 'package' && !values.package) fail('--package is required for package scope'); +const runner = values.runner ?? (values.scope === 'frontend' ? 'turbo' : 'direct'); +if (!['direct', 'turbo'].includes(runner)) fail('--runner must be direct or turbo'); +if (values.scope === 'frontend' && runner === 'direct') { + fail('direct runner requires a single package scope'); +} +if (!['off', 'on', 'both'].includes(values.coverage)) { + fail('--coverage must be off, on, or both'); +} +const validProfileKinds = ['imports', 'coverage', 'cpu', 'heap']; +const profileKinds = + values.profile === 'none' + ? [] + : values.profile === 'all' + ? validProfileKinds + : values.profile.split(',').map((kind) => kind.trim()); +const invalidProfileKind = profileKinds.find((kind) => !validProfileKinds.includes(kind)); +if (invalidProfileKind) fail(`unknown profile kind "${invalidProfileKind}"`); +if (values.scope === 'frontend' && profileKinds.some((kind) => kind === 'cpu' || kind === 'heap')) { + fail('cpu and heap profiles require a single package scope'); +} +if (runner !== 'direct' && profileKinds.some((kind) => kind === 'cpu' || kind === 'heap')) { + fail('cpu and heap profiles require --runner direct'); +} + +const packageName = values.scope === 'editor' ? 'n8n-editor-ui' : values.package; +let packageDir; +if (packageName) { + try { + packageDir = execFileSync( + 'pnpm', + ['--filter', packageName, 'exec', 'node', '-e', 'process.stdout.write(process.cwd())'], + { cwd: REPO_ROOT, encoding: 'utf8' }, + ).trim(); + } catch { + fail(`could not resolve workspace package "${packageName}"`); + } +} +let vitestEntry; +if (packageDir) { + try { + const requireFromPackage = createRequire(resolve(packageDir, 'package.json')); + vitestEntry = join(dirname(requireFromPackage.resolve('vitest')), 'vitest.mjs'); + } catch { + fail(`vitest is not available in ${packageName}`); + } +} +const task = values.task ?? 'test'; +const taskConfig = runner === 'direct' ? directTask(packageDir, task) : undefined; + +const options = { + scope: values.scope, + package: packageName, + packageDir, + suiteLabel: + [packageName, values.file] + .filter(Boolean) + .join('-') + .replaceAll(/[^A-Za-z0-9._-]/g, '-') + .replace(/^-+/, '') || values.scope, + task, + file: values.file, + runner, + vitestEntry, + directTask: taskConfig, + shards: integerList(values.shards, '--shards'), + indexes: values.indexes === 'all' ? 'all' : integerList(values.indexes, '--indexes'), + coverage: + values.coverage === 'both' ? [false, true] : values.coverage === 'on' ? [true] : [false], + workers: positiveInteger(values.workers, '--workers'), + turboConcurrency: positiveInteger(values['turbo-concurrency'], '--turbo-concurrency'), + runs: positiveInteger(values.runs, '--runs'), + sampleMs: positiveInteger(values['sample-ms'], '--sample-ms'), + profileKinds, + skipPrepare: values['skip-prepare'], + allowConcurrent: values['allow-concurrent'], + continueOnFailure: values['continue-on-failure'], + outputDir: values.output ? resolve(process.cwd(), values.output) : DEFAULT_OUTPUT_DIR, + dryRun: values['dry-run'], + json: values.json, +}; + +const cases = []; +for (const shards of options.shards) { + const indexes = + options.indexes === 'all' + ? Array.from({ length: shards }, (_, index) => index + 1) + : options.indexes; + for (const index of indexes) { + if (index > shards) fail(`shard index ${index} exceeds shard total ${shards}`); + for (const coverage of options.coverage) { + for (let run = 1; run <= options.runs; run++) cases.push({ shards, index, coverage, run }); + } + } +} + +async function main() { + if (!options.dryRun) mkdirSync(options.outputDir, { recursive: true }); + if (!options.dryRun && !options.allowConcurrent) { + const active = activeVitestProcesses(); + if (active.length > 0) { + fail( + `${active.length} other Vitest process(es) are active; wait for them or use --allow-concurrent`, + ); + } + } + const preparation = options.skipPrepare ? { skipped: true } : await prepareBuild(options); + if (preparation.exitCode !== undefined && preparation.exitCode !== 0) { + const summary = { schemaVersion: 1, options, preparation, results: [], aggregates: [] }; + writeFileSync( + resolve(options.outputDir, 'summary.json'), + JSON.stringify(summary, null, 2) + '\n', + ); + if (options.json) process.stdout.write(`${JSON.stringify(summary)}\n`); + process.exit(1); + } + const results = []; + for (const [position, testCase] of cases.entries()) { + const label = [ + options.suiteLabel, + `${testCase.index}-of-${testCase.shards}`, + `coverage-${testCase.coverage ? 'on' : 'off'}`, + `workers-${options.workers}`, + `run-${testCase.run}`, + ].join('-'); + const command = commandForCase( + testCase, + options, + resolve(options.outputDir, `${label}-profiles`), + ); + if (!options.json) { + process.stdout.write( + `[${position + 1}/${cases.length}] ${options.suiteLabel} shard ${testCase.index}/${testCase.shards}, coverage ${testCase.coverage ? 'on' : 'off'}... `, + ); + if (options.dryRun) process.stdout.write(`${command.command} ${command.args.join(' ')}\n`); + } + const result = await runCase(testCase, options); + results.push(result); + if (!options.json && !options.dryRun) printResult(result); + if (!options.dryRun) { + writeFileSync( + resolve(options.outputDir, 'summary.json'), + JSON.stringify({ schemaVersion: 1, options, preparation, results }, null, 2) + '\n', + ); + } + if (!result.dryRun && result.exitCode !== 0 && !options.continueOnFailure) break; + } + + const summary = { + schemaVersion: 1, + options, + preparation, + results, + aggregates: aggregateResults(results), + }; + if (!options.dryRun) { + writeFileSync( + resolve(options.outputDir, 'summary.json'), + JSON.stringify(summary, null, 2) + '\n', + ); + } + if (options.json) { + process.stdout.write(`${JSON.stringify(summary)}\n`); + } else if (!options.dryRun) { + process.stdout.write('\nPredicted CI shape (sequential local measurements):\n'); + for (const aggregate of summary.aggregates) { + const timing = aggregate.complete + ? `wall ${formatSeconds(aggregate.predictedWallSeconds)}, runner total ${formatSeconds(aggregate.totalRunnerSeconds)}` + : `${aggregate.measuredShards}/${aggregate.shards} measured, sampled max ${formatSeconds(aggregate.sampledWallSeconds)}`; + process.stdout.write( + `${aggregate.shards} shards, coverage ${aggregate.coverage ? 'on' : 'off'}: ${timing}, peak ${formatMb(aggregate.peakRssKb)}\n`, + ); + } + process.stdout.write(`summary: ${resolve(options.outputDir, 'summary.json')}\n`); + } + + process.exitCode = results.some((result) => !result.dryRun && result.exitCode !== 0) ? 1 : 0; +} + +main().catch((error) => { + process.stderr.write(`vitest-profile: ${error.stack ?? error.message}\n`); + process.exit(1); +});