ci: finish main validation and balance complete test costs

.github/workflows/ci.yml

@@ -21,10 +21,11 @@ permissions:
 
 concurrency:
   # Complete qualification finishes even while main receives more pushes.
+  # Main feedback also finishes, coalescing newer pushes to one pending run.
   # Scheduled/manual main runs coalesce to one active and the latest pending;
   # release and label-triggered qualification use exact-revision groups.
   group: ci-${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || inputs.source_sha != '' || contains(github.event.pull_request.labels.*.name, 'ci:full')) && 'qualification' || 'feedback' }}-${{ inputs.source_sha || github.event.pull_request.number || github.ref }}
-  cancel-in-progress: ${{ github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' && inputs.source_sha == '' && !contains(github.event.pull_request.labels.*.name, 'ci:full') }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' && inputs.source_sha == '' && !contains(github.event.pull_request.labels.*.name, 'ci:full') }}
 
 env:
   HUSKY: "0"
@@ -289,6 +290,7 @@ jobs:
           --reporter=default
           --reporter=junit
           --reporter=json
+          --reporter=./scripts/ci/test-timing-reporter.mjs
           --outputFile.junit=test-results.xml
           --outputFile.json=test-results.json
       - name: Merge fast-suite JUnit and timing reports
@@ -299,10 +301,11 @@ jobs:
           --reporter=default
           --reporter=junit
           --reporter=json
+          --reporter=./scripts/ci/test-timing-reporter.mjs
           --outputFile.junit=test-results.xml
           --outputFile.json=test-results.json
       - name: Summarize slowest test files
-        if: needs.changes.outputs.code == 'true' && !cancelled() && hashFiles('test-results.json') != ''
+        if: needs.changes.outputs.code == 'true' && !cancelled() && hashFiles('test-timings.json') != ''
         env:
           SHARD_RESULT: ${{ needs.test-shard.result }}
           MERGE_RESULT: ${{ needs.changes.outputs.tier == 'full' && steps.merge.outcome || steps.merge-fast.outcome }}
@@ -315,6 +318,7 @@ jobs:
           path: |
             test-results.xml
             test-results.json
+            test-timings.json
             coverage/
           if-no-files-found: error
           retention-days: 30

.gitignore

@@ -36,6 +36,7 @@ coverage/
 .vitest/
 .vitest-reports/
 test-results.xml
+test-timings.json
 .turbo/
 .cache/
 docs/.vitepress/cache/

docs/testing.md

@@ -263,15 +263,21 @@ CI owns coverage; aggregate and domain/contract thresholds are maintained in
 Coverage does not replace named boundary or real-provider scenarios.
 
 CI runs six native Vitest shards without retries. Each shard emits a blob
-report; the merge job produces JUnit and JSON timing reports, plus aggregate
-coverage during full qualification. It retains available
+report; the merge job produces JUnit and JSON diagnostics, a module timing
+report, plus aggregate coverage during full qualification. It retains available
 reports even when a shard fails, uploads available diagnostics, and writes the
 slowest files to the workflow summary. Partial reports are marked diagnostic
 only. Failed, cancelled, or missing selected shards still fail the required gate,
 and aggregate coverage thresholds remain enforced in full qualification.
 Only successful complete full-tier merges publish the small `test-timings` artifact, with
 checkout-relative paths; the build job downloads that artifact for the sequencer
-instead of the coverage/JUnit bundle. Missing history falls back to native
+instead of the coverage/JUnit bundle. Timing weights come from Vitest module
+diagnostics and include tests and hooks, worker preparation, environment,
+imports, and setup. Import timings already include transform waits, which are
+not added again. The weights describe file costs under the measured worker and
+coverage settings; their sum is not workflow elapsed time. The diagnostic
+artifact retains the phase breakdown, while shard history keeps only each
+file's total. Legacy test-body-only history and missing history fall back to native
 sharding. Static checks, documentation, build, package verification, and
 real-system lanes remain separate jobs so one kind of evidence cannot stand in
 for another. Workflow syntax and expression validation runs inside the required
@@ -868,11 +874,11 @@ should require this aggregate, `Static checks`, `Portable tests`, and
 reopening, commits, and edits; correcting a title refreshes only this lightweight
 check. It reads title text through an environment variable, executes no PR code,
 and has no token permissions. Its separate concurrency group cannot cancel
-test runs. Fast runs
-cancel superseded feedback for the same PR or main branch. Scheduled/manual
-qualification has a separate concurrency group: active work finishes and only
-the latest pending run is retained. Release qualification is grouped by its
-exact source SHA and cannot be cancelled by newer main pushes. Provider and
+test runs. Fast PR runs cancel superseded feedback for the same PR. Main push
+runs keep one active run and the latest pending run for that branch.
+Scheduled/manual qualification has a separate concurrency group: active work
+finishes and only the latest pending run is retained. Release qualification uses
+its exact source SHA and cannot be cancelled by newer main pushes. Provider and
 website lanes inherit their caller's scheduling; reusable
 workflows have no independent cancellation group. Direct manual lane dispatches
 run independently on disposable hosted runners. Publication

scripts/ci/test-report.mjs

@@ -11,19 +11,18 @@ const historyPath = join(historyDirectory, "test-results.json");
 await rm(historyPath, { force: true });
 const report = z
   .object({
+    version: z.literal(1),
     success: z.boolean(),
-    numFailedTests: z.number().int().nonnegative(),
-    testResults: z.array(
+    files: z.array(
       z.object({
         name: z.string().min(1),
-        startTime: z.number().finite(),
-        endTime: z.number().finite(),
+        durationMs: z.number().finite().nonnegative(),
       }),
     ),
   })
-  .parse(JSON.parse(await readFile("test-results.json", "utf8")));
+  .parse(JSON.parse(await readFile("test-timings.json", "utf8")));
 
-const files = report.testResults.map(({ name, startTime, endTime }) => {
+const files = report.files.map(({ name, durationMs }) => {
   const path = relative(root, resolve(root, name));
   if (
     path === "" ||
@@ -32,17 +31,11 @@ const files = report.testResults.map(({ name, startTime, endTime }) => {
     isAbsolute(path)
   )
     throw new Error(`Test report path is outside the checkout: ${name}`);
-  const duration = endTime - startTime;
-  if (!Number.isFinite(duration) || duration < 0)
-    throw new Error(`Test report has an invalid duration: ${name}`);
-  return { name: path.split(sep).join("/"), startTime: 0, endTime: duration };
+  return { name: path.split(sep).join("/"), durationMs };
 });
 const complete = process.env.SHARD_RESULT === "success";
 const passed =
-  complete &&
-  process.env.MERGE_RESULT === "success" &&
-  report.success &&
-  report.numFailedTests === 0;
+  complete && process.env.MERGE_RESULT === "success" && report.success;
 // Successful shards and merged reports must account for the complete portable
 // inventory. Coverage thresholds alone cannot prove that every file ran.
 if (passed && process.env.CI_VERIFY_TEST_INVENTORY === "true") {
@@ -71,13 +64,16 @@ const summary = [
     ? "Successful timing history is available."
     : "This report is diagnostic only and is not a successful timing baseline.",
   "",
+  "Durations include tests and hooks, worker preparation, environment, imports, and setup. They are scheduling weights, not summed workflow wall time.",
+  "",
   "| Slowest file | Duration (ms) |",
   "| --- | ---: |",
   ...files
-    .toSorted((left, right) => right.endTime - left.endTime)
+    .toSorted((left, right) => right.durationMs - left.durationMs)
     .slice(0, 20)
     .map(
-      ({ name, endTime }) => `| ${name.replaceAll("|", "\\|")} | ${endTime} |`,
+      ({ name, durationMs }) =>
+        `| ${name.replaceAll("|", "\\|")} | ${durationMs} |`,
     ),
   "",
 ].join("\n");
@@ -89,6 +85,6 @@ if (passed) {
   await mkdir(historyDirectory, { recursive: true });
   await writeFile(
     historyPath,
-    JSON.stringify({ success: true, numFailedTests: 0, testResults: files }),
+    JSON.stringify({ version: 1, success: true, files }),
   );
 }

scripts/ci/test-timing-reporter.mjs

@@ -0,0 +1,40 @@
+import { writeFile } from "node:fs/promises";
+
+/** @implements {import("vitest/node").Reporter} */
+export default class TestTimingReporter {
+  /**
+   * @param {ReadonlyArray<import("vitest/node").TestModule>} modules
+   * @param {ReadonlyArray<import("vitest/node").SerializedError>} errors
+   * @param {import("vitest/node").TestRunEndReason} reason
+   */
+  async onTestRunEnd(modules, errors, reason) {
+    const files = modules.map((module) => {
+      const diagnostic = module.diagnostic();
+      // These disjoint phases match Vitest's per-file wall-time accounting.
+      // Import timings already include transform waits; do not add them again.
+      const phases = {
+        execution: diagnostic.duration,
+        preparation: diagnostic.prepareDuration,
+        environment: diagnostic.environmentSetupDuration,
+        imports: diagnostic.collectDuration,
+        setup: diagnostic.setupDuration,
+      };
+      return {
+        name: module.moduleId,
+        durationMs: Object.values(phases).reduce(
+          (total, value) => total + value,
+          0,
+        ),
+        phases,
+      };
+    });
+    await writeFile(
+      "test-timings.json",
+      JSON.stringify({
+        version: 1,
+        success: reason === "passed" && errors.length === 0,
+        files,
+      }),
+    );
+  }
+}

tests/boundary/filesystem/ciRequiredLanes.test.ts

@@ -249,12 +249,9 @@ it("keeps fast feedback separate from complete qualification and release proof",
   ]);
   expect(ci.concurrency.group).toContain("'qualification' || 'feedback'");
   expect(ci.concurrency.group).toContain("inputs.source_sha ||");
-  for (const fullEvent of ["schedule", "workflow_dispatch"])
-    expect(ci.concurrency["cancel-in-progress"]).toContain(
-      `github.event_name != '${fullEvent}'`,
-    );
-  expect(ci.concurrency["cancel-in-progress"]).toContain(
-    "inputs.source_sha == ''",
+  expect(ci.concurrency.group).toContain("github.ref");
+  expect(ci.concurrency["cancel-in-progress"]).toBe(
+    "${{ github.event_name == 'pull_request' && inputs.source_sha == '' && !contains(github.event.pull_request.labels.*.name, 'ci:full') }}",
   );
   expect(ci.jobs.required?.if).toBe("${{ always() }}");
   const shards = ci.jobs["test-shard"]?.steps ?? [];

tests/boundary/filesystem/ciTestReports.test.ts

@@ -1,5 +1,5 @@
 import { execFile } from "node:child_process";
-import { mkdir, readFile, writeFile } from "node:fs/promises";
+import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
 import { join, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import { promisify } from "node:util";
@@ -54,6 +54,13 @@ it("collects failed-shard diagnostics before enforcing the shard result", async
   expect(upload).toBeGreaterThan(merge);
   expect(gate).toBeGreaterThan(upload);
   expect(steps[merge]?.if).not.toContain("needs.test-shard.result");
+  const merges = steps.filter((step) => step.run?.includes("--reporter=json"));
+  expect(merges).toHaveLength(2);
+  for (const step of merges)
+    expect(step.run).toContain(
+      "--reporter=./scripts/ci/test-timing-reporter.mjs",
+    );
+  expect(steps[upload]?.with?.path).toContain("test-timings.json");
   const command = steps[gate]?.run;
   if (command === undefined) throw new Error("Missing shard result gate");
   for (const result of ["failure", "cancelled", "skipped", ""]) {
@@ -107,11 +114,11 @@ it("rejects successful shard reports with missing or duplicate portable files",
   await workflowGit(root, ["add", "tests"]);
   const run = async (names: readonly string[]) => {
     await writeFile(
-      join(root, "test-results.json"),
+      join(root, "test-timings.json"),
       JSON.stringify({
+        version: 1,
         success: true,
-        numFailedTests: 0,
-        testResults: names.map((name) => ({ name, startTime: 0, endTime: 1 })),
+        files: names.map((name) => ({ name, durationMs: 1 })),
       }),
     );
     return execute(process.execPath, [fileURLToPath(script)], {
@@ -173,20 +180,25 @@ it("exports portable timing history and removes it on a failed, partial, or inva
   const summary = join(root, "summary.md");
   const history = join(root, ".cache/ci-test-history/test-results.json");
   const report = {
+    version: 1,
     success: true,
-    numFailedTests: 0,
-    testResults: [
+    files: [
       {
         name: join(root, "tests/slow.test.ts"),
-        startTime: 10,
-        endTime: 110,
-        assertionResults: [{ large: "not timing data" }],
+        durationMs: 100,
+        phases: {
+          execution: 20,
+          preparation: 20,
+          environment: 10,
+          imports: 40,
+          setup: 10,
+        },
       },
-      { name: join(root, "tests/fast.test.ts"), startTime: 20, endTime: 30 },
+      { name: join(root, "tests/fast.test.ts"), durationMs: 10 },
     ],
   };
   const run = async (value: unknown, shards = "success", merge = "success") => {
-    await writeFile(join(root, "test-results.json"), JSON.stringify(value));
+    await writeFile(join(root, "test-timings.json"), JSON.stringify(value));
     await writeFile(summary, "");
     return execute(process.execPath, [fileURLToPath(script)], {
       cwd: root,
@@ -200,7 +212,8 @@ it("exports portable timing history and removes it on a failed, partial, or inva
   };
   await run(report);
   const contents = await readFile(history, "utf8");
-  expect(contents).not.toContain("assertionResults");
+  expect(contents).not.toContain("phases");
+  expect(contents).not.toContain("startTime");
   expect(contents).not.toContain(root);
   expect(
     parseTestDurations(contents, resolve(root, "different-checkout")),
@@ -211,7 +224,7 @@ it("exports portable timing history and removes it on a failed, partial, or inva
     ]),
   );
   for (const [value, shards, merge] of [
-    [{ ...report, success: false, numFailedTests: 1 }, "failure", "failure"],
+    [{ ...report, success: false }, "failure", "failure"],
     [report, "cancelled", "success"],
     [report, "success", "failure"],
   ] as const) {
@@ -224,8 +237,125 @@ it("exports portable timing history and removes it on a failed, partial, or inva
   await expect(
     run({
       ...report,
-      testResults: [{ name: "../outside.test.ts", startTime: 1, endTime: 3 }],
+      files: [{ name: "../outside.test.ts", durationMs: 2 }],
+    }),
+  ).rejects.toMatchObject({ code: 1 });
+  await expect(readFile(history)).rejects.toMatchObject({ code: "ENOENT" });
+  await run(report);
+  await expect(
+    run({
+      ...report,
+      files: [{ name: "tests/invalid.test.ts", durationMs: -1 }],
     }),
   ).rejects.toMatchObject({ code: 1 });
   await expect(readFile(history)).rejects.toMatchObject({ code: "ENOENT" });
 });
+
+it("recovers complete module costs from real blob reports without executing tests again", async () => {
+  const root = await createTestTempDirectory("rea-ci-module-timing-");
+  const config = join(root, "vitest.config.mjs");
+  await writeFile(
+    config,
+    'export default { test: { globals: true, include: ["sample.test.js"], maxWorkers: 1 } };',
+  );
+  await writeFile(
+    join(root, "sample.test.js"),
+    [
+      'import { writeFileSync } from "node:fs";',
+      "beforeAll(() => new Promise(resolve => setTimeout(resolve, 80)));",
+      "afterAll(() => new Promise(resolve => setTimeout(resolve, 80)));",
+      'it("has cheap assertions and expensive outer hooks", () => { writeFileSync("executed", "yes"); expect(1).toBe(1); });',
+    ].join("\n"),
+  );
+  const vitest = fileURLToPath(
+    new URL("../../../node_modules/vitest/vitest.mjs", import.meta.url),
+  );
+  const reporter = fileURLToPath(
+    new URL("../../../scripts/ci/test-timing-reporter.mjs", import.meta.url),
+  );
+  const run = (args: string[]) =>
+    execute(process.execPath, [vitest, ...args, "--config", config], {
+      cwd: root,
+      env: process.env,
+      timeout: 20_000,
+    });
+  await run(["run", "--reporter=blob"]);
+  expect(await readFile(join(root, "executed"), "utf8")).toBe("yes");
+  await rm(join(root, "executed"));
+  await run([
+    "--merge-reports",
+    `--reporter=${reporter}`,
+    "--reporter=json",
+    "--outputFile.json=body.json",
+  ]);
+  await expect(readFile(join(root, "executed"))).rejects.toMatchObject({
+    code: "ENOENT",
+  });
+  const report = z
+    .object({
+      version: z.literal(1),
+      success: z.literal(true),
+      files: z.array(
+        z.object({
+          name: z.string(),
+          durationMs: z.number(),
+          phases: z.record(z.string(), z.number()),
+        }),
+      ),
+    })
+    .parse(JSON.parse(await readFile(join(root, "test-timings.json"), "utf8")));
+  expect(report.files).toHaveLength(1);
+  const file = report.files[0]!;
+  const json = z
+    .object({
+      testResults: z.array(
+        z.object({ startTime: z.number(), endTime: z.number() }),
+      ),
+    })
+    .parse(JSON.parse(await readFile(join(root, "body.json"), "utf8")));
+  const body = json.testResults[0]!;
+  expect(
+    file.phases.execution! - (body.endTime - body.startTime),
+  ).toBeGreaterThan(100);
+  expect(file.phases.imports).toBeGreaterThan(0);
+  expect(file.phases.preparation).toBeGreaterThan(0);
+  expect(file.durationMs).toBeCloseTo(
+    Object.values(file.phases).reduce((sum, value) => sum + value, 0),
+  );
+  await execute(process.execPath, [fileURLToPath(script)], {
+    cwd: root,
+    env: { ...process.env, SHARD_RESULT: "success", MERGE_RESULT: "success" },
+  });
+  expect(
+    parseTestDurations(
+      await readFile(
+        join(root, ".cache/ci-test-history/test-results.json"),
+        "utf8",
+      ),
+      root,
+    )?.get("sample.test.js"),
+  ).toBe(file.durationMs);
+
+  await writeFile(
+    join(root, "sample.test.js"),
+    'it("fails", () => { throw new Error("fixture failure"); });',
+  );
+  await expect(run(["run", "--reporter=blob"])).rejects.toMatchObject({
+    code: 1,
+  });
+  await expect(
+    run(["--merge-reports", `--reporter=${reporter}`]),
+  ).rejects.toMatchObject({ code: 1 });
+  const failed = JSON.parse(
+    await readFile(join(root, "test-timings.json"), "utf8"),
+  ) as unknown;
+  expect(failed).toMatchObject({ version: 1, success: false });
+  // Even a caller accidentally marking the merge successful cannot publish this report.
+  await execute(process.execPath, [fileURLToPath(script)], {
+    cwd: root,
+    env: { ...process.env, SHARD_RESULT: "success", MERGE_RESULT: "success" },
+  });
+  await expect(
+    readFile(join(root, ".cache/ci-test-history/test-results.json")),
+  ).rejects.toMatchObject({ code: "ENOENT" });
+});

tests/boundary/filesystem/ciTestSequencer.test.ts

@@ -17,24 +17,24 @@ describe("CI test history", () => {
     const root = resolve("/workspace/rea");
     const durations = parseTestDurations(
       JSON.stringify({
+        version: 1,
         success: true,
-        numFailedTests: 0,
-        testResults: [
+        files: [
           {
             name: join(root, "tests/slow.test.ts"),
-            startTime: 10,
-            endTime: 110,
+            durationMs: 100,
           },
           {
             name: join(root, "tests/slow.test.ts"),
-            startTime: 20,
-            endTime: 80,
+            durationMs: 60,
           },
           {
             name: resolve(root, "../outside.test.ts"),
-            startTime: 10,
-            endTime: 1000,
+            durationMs: 990,
           },
+          { name: "tests/negative.test.ts", durationMs: -1 },
+          { name: "tests/zero.test.ts", durationMs: 0 },
+          { name: "tests/invalid.test.ts", durationMs: "100" },
           null,
         ],
       }),
@@ -43,7 +43,20 @@ describe("CI test history", () => {
 
     expect(durations).toEqual(new Map([["tests/slow.test.ts", 100]]));
     expect(
-      parseTestDurations('{"success": false, "numFailedTests": 1}', root),
+      parseTestDurations('{"version": 1, "success": false, "files": []}', root),
+    ).toBeUndefined();
+    // Legacy test-body spans must not be mixed with complete module costs.
+    expect(
+      parseTestDurations(
+        JSON.stringify({
+          success: true,
+          numFailedTests: 0,
+          testResults: [
+            { name: "tests/legacy.test.ts", startTime: 0, endTime: 100 },
+          ],
+        }),
+        root,
+      ),
     ).toBeUndefined();
     expect(parseTestDurations("not json", root)).toBeUndefined();
     expect(parseTestDurations("null", root)).toBeUndefined();

tests/ciTestSequencer.ts

@@ -5,14 +5,13 @@ import { BaseSequencer, type TestSpecification } from "vitest/node";
 import { z } from "zod";
 
 const reportSchema = z.object({
+  version: z.literal(1),
   success: z.literal(true),
-  numFailedTests: z.literal(0),
-  testResults: z.array(z.unknown()),
+  files: z.array(z.unknown()),
 });
 const fileSchema = z.object({
   name: z.string(),
-  startTime: z.number(),
-  endTime: z.number(),
+  durationMs: z.number().finite().positive(),
 });
 
 export function parseTestDurations(
@@ -30,15 +29,12 @@ export function parseTestDurations(
   if (!report.success) return undefined;
 
   const durations = new Map<string, number>();
-  for (const value of report.data.testResults) {
+  for (const value of report.data.files) {
     const result = fileSchema.safeParse(value);
     if (!result.success) continue;
-    const { name, startTime, endTime } = result.data;
-    const duration = endTime - startTime;
+    const { name, durationMs } = result.data;
     const path = relative(root, resolve(root, name));
     if (
-      !Number.isFinite(duration) ||
-      duration <= 0 ||
       path === "" ||
       path === ".." ||
       path.startsWith(".." + sep) ||
@@ -46,7 +42,7 @@ export function parseTestDurations(
     )
       continue;
     const key = path.split(sep).join("/");
-    durations.set(key, Math.max(durations.get(key) ?? 0, duration));
+    durations.set(key, Math.max(durations.get(key) ?? 0, durationMs));
   }
   return durations.size ? durations : undefined;
 }