diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6c81e257d..131f46379 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,6 +118,8 @@ jobs: name: rea-test-runtime path: . - run: npm run test:ci:shard:run -- --shard=${{ matrix.shard }} + env: + REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json - uses: actions/upload-artifact@v4 if: ${{ !cancelled() }} with: @@ -133,6 +135,9 @@ jobs: needs: changes if: needs.changes.outputs.code == 'true' runs-on: ubuntu-latest + permissions: + contents: read + actions: read steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -141,6 +146,28 @@ jobs: cache: npm - run: npm ci - run: npm run test:prepare + - name: Find successful default-branch timing history + id: history + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + CI_REPOSITORY: ${{ github.repository }} + CI_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + run_id="$(gh api --method GET \ + "repos/${CI_REPOSITORY}/actions/workflows/ci.yml/runs" \ + -f branch="${CI_DEFAULT_BRANCH}" -f event=push -f status=success -f per_page=1 \ + --jq '.workflow_runs[0].id // empty')" + echo "run-id=${run_id}" >> "${GITHUB_OUTPUT}" + - name: Download previous timing report + if: steps.history.outputs.run-id != '' + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: test-results + run-id: ${{ steps.history.outputs.run-id }} + github-token: ${{ github.token }} + path: .cache/ci-test-history - uses: actions/upload-artifact@v4 with: name: rea-test-runtime @@ -148,6 +175,7 @@ jobs: dist/ skills/ .cache/mcp-tool-catalog.json + .cache/ci-test-history/test-results.json docs/public/product-catalog.json docs/verification/managed-conformance-*.json include-hidden-files: true @@ -305,7 +333,14 @@ jobs: name: windows-native-controls path: native/windows/build/ - run: npm run verify:windows-native:artifact + - name: Resolve npm cache before verifier HOME isolation + id: npm-cache + run: | + cache_path="$(npm config get cache)" + echo "path=${cache_path}" >> "${GITHUB_OUTPUT}" - run: npm run verify:package + env: + npm_config_cache: ${{ steps.npm-cache.outputs.path }} - name: Verify real Node Inspector discovery and observation timeout-minutes: 10 run: npm run verify:inspector diff --git a/docs/testing.md b/docs/testing.md index 82c6f0263..2d6369d89 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -835,6 +835,12 @@ target-free analysis, configuration backups/recovery, Evidence and owned lifecycle; Inspector checks execute source-owned loopback targets and special filename cases. +POSIX package verification resolves the runner's configured npm cache before +creating disposable client homes and passes that location to its child installs. +This reuses the download cache restored by `setup-node` while keeping client +configuration isolated. Installs retain their existing registry and integrity +checks. + The Apple artifact CLI/MCP lane uses one `macos-15` job to avoid adding demand for scarce macOS runner capacity. It runs the artifact-format and metadata verifiers, followed by one consolidated focused invocation containing all @@ -847,6 +853,15 @@ catalog, and managed-conformance metadata. Each shard still installs the locked dependencies and runs its assigned coverage-enabled tests; it no longer regenerates the same metadata independently. +The build job also retrieves the merged timing report from the last successful +default-branch push and includes that same report in each shard's snapshot. +CI assigns discovered files by historical duration, with a deterministic +fallback weight for new files, rather than dividing only by file count. Every +discovered file still belongs to exactly one shard; worker limits, project +ordering, coverage instrumentation, and aggregate thresholds remain unchanged. +Missing or unusable history falls back to Vitest's default assignment. Only +timing data is reused; persistent compile caches remain disabled. + These native baseline checks complement the Linux source-test shards and the separate Apple-artifact and real-provider lanes. Actual Hopper, Ghidra, IDA, browser and managed-tool claims require their corresponding verification lanes. diff --git a/tests/boundary/filesystem/ciTestSequencer.test.ts b/tests/boundary/filesystem/ciTestSequencer.test.ts new file mode 100644 index 000000000..6177d9481 --- /dev/null +++ b/tests/boundary/filesystem/ciTestSequencer.test.ts @@ -0,0 +1,155 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; + +import { fc, it } from "@fast-check/vitest"; +import { describe, expect } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +import { + allocateByDuration, + parseTestDurations, + readTestDurations, +} from "../../ciTestSequencer.js"; + +describe("CI test history", () => { + it("uses only successful in-root file durations", () => { + const root = resolve("/workspace/rea"); + const durations = parseTestDurations( + JSON.stringify({ + success: true, + numFailedTests: 0, + testResults: [ + { + name: join(root, "tests/slow.test.ts"), + startTime: 10, + endTime: 110, + }, + { + name: join(root, "tests/slow.test.ts"), + startTime: 20, + endTime: 80, + }, + { + name: resolve(root, "../outside.test.ts"), + startTime: 10, + endTime: 1000, + }, + null, + ], + }), + root, + ); + + expect(durations).toEqual(new Map([["tests/slow.test.ts", 100]])); + expect( + parseTestDurations('{"success": false, "numFailedTests": 1}', root), + ).toBeUndefined(); + expect(parseTestDurations("not json", root)).toBeUndefined(); + expect(parseTestDurations("null", root)).toBeUndefined(); + }); + + it("falls back for missing history while preserving real filesystem errors", async () => { + const temporaryRoot = await createTestTempDirectory("rea-ci-sequencer-"); + const missing = await readTestDurations( + join(temporaryRoot, "missing.json"), + temporaryRoot, + ); + expect(missing).toBeUndefined(); + + const directory = join(temporaryRoot, "directory"); + await mkdir(directory); + const reportPath = join(temporaryRoot, "report.json"); + await writeFile(reportPath, "{"); + expect(await readTestDurations(reportPath, temporaryRoot)).toBeUndefined(); + + await expect( + readTestDurations(directory, temporaryRoot), + ).rejects.toMatchObject({ + code: "EISDIR", + }); + }); +}); + +describe("duration shard allocation", () => { + it.prop([ + fc.uniqueArray(fc.integer({ min: 0, max: 10_000 }), { + minLength: 1, + maxLength: 80, + }), + fc.integer({ min: 3, max: 8 }), + ])( + "is complete, disjoint, reference-preserving, and input-order independent", + (ids, shardCount) => { + const items = ids.map((id, index) => ({ + id, + path: `tests/${id}.test.ts`, + project: `project-${id % 3}`, + pool: id % 2 === 0 ? "forks" : "threads", + duration: index > 0 && id % 5 === 0 ? undefined : (id % 17) + 1, + })); + const specifications = items.map((item) => ({ + item, + key: `${item.path}\0${item.project}\0${item.pool}`, + duration: item.duration, + })); + const shards = allocateByDuration(specifications, shardCount); + const reversedShards = allocateByDuration( + [...specifications].reverse(), + shardCount, + ); + + expect(shards).toBeDefined(); + expect(reversedShards).toEqual(shards); + const assigned = shards!.flat(); + expect(assigned).toHaveLength(items.length); + expect(new Set(assigned).size).toBe(items.length); + expect(assigned.every((item) => items.includes(item))).toBe(true); + }, + ); + + it("keeps tied path, project, pool, and new-file entries deterministic", () => { + const items = [ + { + id: "a", + path: "tests/shared.test.ts", + project: "alpha", + pool: "forks", + duration: 10, + }, + { + id: "b", + path: "tests/shared.test.ts", + project: "beta", + pool: "forks", + duration: 10, + }, + { + id: "c", + path: "tests/shared.test.ts", + project: "beta", + pool: "threads", + duration: 10, + }, + { + id: "new", + path: "tests/new.test.ts", + project: "alpha", + pool: "forks", + duration: undefined, + }, + ]; + const specifications = items.map((item) => ({ + item, + key: `${item.path}\0${item.project}\0${item.pool}`, + duration: item.duration, + })); + const shards = allocateByDuration(specifications, 3); + const reversedShards = allocateByDuration([...specifications].reverse(), 3); + + expect(shards).toBeDefined(); + expect(reversedShards).toEqual(shards); + expect(shards!.flat()).toHaveLength(items.length); + expect(new Set(shards!.flat()).size).toBe(items.length); + }); +}); diff --git a/tests/ciTestSequencer.ts b/tests/ciTestSequencer.ts new file mode 100644 index 000000000..5e8c4fe5f --- /dev/null +++ b/tests/ciTestSequencer.ts @@ -0,0 +1,125 @@ +import { readFile } from "node:fs/promises"; +import { isAbsolute, relative, resolve, sep } from "node:path"; + +import { BaseSequencer, type TestSpecification } from "vitest/node"; +import { z } from "zod"; + +const reportSchema = z.object({ + success: z.literal(true), + numFailedTests: z.literal(0), + testResults: z.array(z.unknown()), +}); +const fileSchema = z.object({ + name: z.string(), + startTime: z.number(), + endTime: z.number(), +}); + +export function parseTestDurations( + contents: string, + root: string, +): Map | undefined { + let value: unknown; + try { + value = JSON.parse(contents); + } catch (error) { + if (error instanceof SyntaxError) return undefined; + throw error; + } + const report = reportSchema.safeParse(value); + if (!report.success) return undefined; + + const durations = new Map(); + for (const value of report.data.testResults) { + const result = fileSchema.safeParse(value); + if (!result.success) continue; + const { name, startTime, endTime } = result.data; + const duration = endTime - startTime; + const path = relative(root, resolve(root, name)); + if ( + !Number.isFinite(duration) || + duration <= 0 || + path === "" || + path === ".." || + path.startsWith(".." + sep) || + isAbsolute(path) + ) + continue; + const key = path.split(sep).join("/"); + durations.set(key, Math.max(durations.get(key) ?? 0, duration)); + } + return durations.size ? durations : undefined; +} + +export async function readTestDurations( + historyPath: string, + root: string, +): Promise | undefined> { + try { + return parseTestDurations(await readFile(historyPath, "utf8"), root); + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") + return undefined; + throw error; + } +} + +type TimedItem = { item: T; key: string; duration: number | undefined }; + +/** Distribute the longest files first, retaining every discovered specification. */ +export function allocateByDuration( + items: readonly TimedItem[], + shardCount: number, +): T[][] | undefined { + if (!Number.isInteger(shardCount) || shardCount < 1) + throw new RangeError("shardCount must be a positive integer"); + const known = items + .flatMap(({ duration }) => (duration === undefined ? [] : [duration])) + .sort((left, right) => left - right); + if (!known.length) return undefined; + const lower = known[Math.floor((known.length - 1) / 2)]!; + const median = lower + (known[Math.floor(known.length / 2)]! - lower) / 2; + const ordered = items.toSorted( + (left, right) => + (right.duration ?? median) - (left.duration ?? median) || + (left.key < right.key ? -1 : left.key > right.key ? 1 : 0), + ); + const shards = Array.from({ length: shardCount }, () => ({ + duration: 0, + items: [] as T[], + })); + for (const entry of ordered) { + const lightest = shards.reduce((left, right) => + right.duration < left.duration ? right : left, + ); + lightest.items.push(entry.item); + lightest.duration += entry.duration ?? median; + } + return shards.map(({ items }) => items); +} + +export class CiTestSequencer extends BaseSequencer { + override async shard( + files: TestSpecification[], + ): Promise { + const historyPath = process.env.REA_CI_TEST_HISTORY; + const shard = this.ctx.config.shard; + if (!historyPath || !shard) return super.shard(files); + const root = this.ctx.config.root; + const durations = await readTestDurations(historyPath, root); + if (!durations) return super.shard(files); + + const weighted = files.map((item) => { + const path = relative(root, resolve(root, item.moduleId)) + .split(sep) + .join("/"); + return { + item, + key: path + "\0" + item.project.name + "\0" + item.pool, + duration: durations.get(path), + }; + }); + const shards = allocateByDuration(weighted, shard.count); + return shards ? shards[shard.index - 1]! : super.shard(files); + } +} diff --git a/vitest.config.ts b/vitest.config.ts index eae295484..2171bf7a2 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -4,6 +4,8 @@ import { join } from "node:path"; import { defineConfig } from "vitest/config"; +import { CiTestSequencer } from "./tests/ciTestSequencer.js"; + // Playwright's registry does not derive a cache directory for Termux's // `android` platform. Match the production CLI fallback before workers import // browser modules. @@ -151,6 +153,9 @@ export default defineConfig({ env: { TMPDIR: CANONICAL_TEMPORARY_DIRECTORY }, maxWorkers: MAX_TEST_WORKERS, projects, + ...(process.env.REA_CI_TEST_HISTORY + ? { sequence: { sequencer: CiTestSequencer } } + : {}), retry: 0, reporters: ["default"], runner: "./tests/processOwnershipRunner.ts",