paperclip/scripts/__tests__/local-verify.test.mjs

import assert from "node:assert/strict";
import { execFileSync, spawn, spawnSync } from "node:child_process";
import { chmodSync, cpSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, utimesSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import test from "node:test";
import { fileURLToPath } from "node:url";
import { stripVTControlCharacters } from "node:util";
import { comparison, localEnvironment, snapshot } from "../local-verify/inputs.mjs";
import { planVerification } from "../local-verify/plan.mjs";

const repository = fileURLToPath(new URL("../../", import.meta.url));

function fixture(t) {
  const root = realpathSync(mkdtempSync(path.join(tmpdir(), "local-verify-")));
  t.after(() => rmSync(root, { recursive: true, force: true }));
  const write = (file, text) => {
    mkdirSync(path.dirname(path.join(root, file)), { recursive: true });
    writeFileSync(path.join(root, file), text);
  };
  for (const directory of ["scripts/ci", "scripts/local-verify", ".github/scripts"]) {
    cpSync(path.join(repository, directory), path.join(root, directory), { recursive: true });
  }
  for (const file of ["scripts/local-verify.mjs", "scripts/run-vitest-stable.mjs", "scripts/general-server-shard.mjs",
    "scripts/test-line-shard.mjs", "scripts/ci-profile/record.mjs", "turbo.json", ".github/ci-task-matrix.json"]) {
    write(file, readFileSync(path.join(repository, file), "utf8"));
  }
  for (const directory of ["server/src/__tests__", "server/scripts"]) mkdirSync(path.join(root, directory), { recursive: true });
  mkdirSync(path.join(root, "node_modules/.pnpm"), { recursive: true });
  for (const name of ["vitest", "oxc-parser", ".bin"]) {
    symlinkSync(path.join(repository, "node_modules", name), path.join(root, "node_modules", name), "dir");
  }
  for (const file of ["node_modules/.modules.yaml", "node_modules/.pnpm/lock.yaml"]) {
    write(file, readFileSync(path.join(repository, file), "utf8"));
  }
  write(".gitignore", "node_modules\nnode_modules/\n.turbo/\n.env\n");
  write("package.json", JSON.stringify({ name: "local-verification-fixture", private: true, type: "module", packageManager: "[email protected]",
    scripts: { verify: "node scripts/local-verify.mjs", "test:run": "vitest run" } }));
  write("vitest.config.mjs", 'export default {test:{projects:["ui","cli"]}};');
  for (const [directory, name] of [["ui", "@paperclipai/ui"], ["cli", "paperclipai"]]) {
    write(`${directory}/vitest.config.mjs`, `export default {test:{name:${JSON.stringify(name)},include:["src/**/*.test.ts"]}};`);
  }
  write("ui/src/leaf.ts", "export const value = 1;\n");
  write("ui/src/other.ts", "export const other = 1;\n");
  write("ui/src/consumer.test.ts", 'import {test,expect} from "vitest"; import {value} from "./leaf"; test("affected consumer",()=>{expect(value).toBeGreaterThan(0);expect(process.env.LOCAL_VERIFY_FLAVOR).not.toBe("fail");});');
  write("ui/src/unrelated.test.ts", 'import {test,expect} from "vitest"; import {other} from "./other"; test("unrelated full-suite control",()=>expect(other).toBe(123456));');
  write("cli/src/independent.test.ts", 'import {test,expect} from "vitest"; test("unchanged consumer project",()=>expect(1).toBe(1));');
  const git = (...args) => execFileSync("git", args, { cwd: root, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"] }).trim();
  git("init", "-q"); git("add", ".");
  git("-c", "user.name=Test", "-c", "[email protected]", "commit", "-qm", "baseline");
  const base = git("rev-parse", "HEAD");
  write("ui/src/leaf.ts", "export const value = 2;\n");
  const env = localEnvironment({ ...process.env, FORCE_COLOR: "0" });
  delete env.NODE_TEST_CONTEXT;
  delete env.DATABASE_URL;
  delete env.PAPERCLIP_TEST_POSTGRES_ADMIN_URL;
  for (const name of Object.keys(env)) if (/^PAPERCLIP_.*_TEST_DATABASE_URL$/.test(name)) delete env[name];
  const run = (args = [], extra = {}) => {
    const result = spawnSync(process.execPath, [path.join(root, "scripts/local-verify.mjs"), "--base", base, ...args], {
      cwd: root, env: { ...env, ...extra }, encoding: "utf8", timeout: 120_000, maxBuffer: 8 * 1024 * 1024,
    });
    assert.equal(result.error, undefined);
    return { ...result, text: stripVTControlCharacters(result.stdout + result.stderr) };
  };
  return { root, write, git, base, env, run };
}

test("real selected execution reuses passes and invalidates source, environment, untracked tests and broad inputs", { timeout: 180_000 }, async t => {
  const f = fixture(t);
  const first = f.run();
  assert.equal(first.status, 0, first.text);
  assert.match(first.text, /Test Files\s+1 passed/);
  assert.match(first.text, /1\/3 Vitest test files selected/);
  assert.doesNotMatch(first.text, /unrelated full-suite control/);
  const reused = f.run();
  assert.equal(reused.status, 0, reused.text);
  assert.match(reused.text, /reusing a passing local result/);
  assert.doesNotMatch(reused.text, /Test Files/);
  const last = JSON.parse(readFileSync(path.join(f.root, ".turbo/local-verify/last-run.json"), "utf8"));
  const passFile = path.join(f.root, ".turbo/local-verify", `${last.key}.json`);
  const corrupt = JSON.parse(readFileSync(passFile, "utf8"));
  corrupt.plan.selectedFiles = [];
  writeFileSync(passFile, JSON.stringify(corrupt));
  const recovered = f.run();
  assert.equal(recovered.status, 0, recovered.text);
  assert.doesNotMatch(recovered.text, /reusing a passing local result/);
  assert.match(recovered.text, /Reusing passed general-workspaces-a-reusable/);
  f.write("ui/src/leaf.ts", "export const value = 3;\n");
  const edited = f.run();
  assert.equal(edited.status, 0, edited.text);
  assert.match(edited.text, /Test Files\s+1 passed/);
  const environment = f.run([], { LOCAL_VERIFY_FLAVOR: "different" });
  assert.equal(environment.status, 0, environment.text);
  assert.match(environment.text, /Test Files\s+1 passed/);
  const environmentFailure = f.run([], { LOCAL_VERIFY_FLAVOR: "fail" });
  assert.equal(environmentFailure.status, 1, environmentFailure.text);
  assert.match(environmentFailure.text, /affected consumer/);
  const installed = readFileSync(path.join(f.root, "node_modules/.modules.yaml"), "utf8");
  f.write("node_modules/.modules.yaml", `${installed}\n# changed installation\n`);
  const dependencies = f.run();
  assert.equal(dependencies.status, 0, dependencies.text);
  assert.match(dependencies.text, /Test Files\s+1 passed/);
  rmSync(path.join(f.root, "node_modules/vitest"));
  const brokenInstall = f.run();
  assert.equal(brokenInstall.status, 2, brokenInstall.text);
  assert.doesNotMatch(brokenInstall.text, /reusing a passing local result/);
  symlinkSync(path.join(repository, "node_modules/vitest"), path.join(f.root, "node_modules/vitest"), "dir");
  f.write("ui/src/new.test.ts", 'import {test,expect} from "vitest"; test("new untracked regression",()=>expect(false).toBe(true));');
  const untracked = f.run();
  assert.equal(untracked.status, 1, untracked.text);
  assert.match(untracked.text, /new untracked regression/);
  const failedAgain = f.run();
  assert.equal(failedAgain.status, 1, failedAgain.text);
  assert.doesNotMatch(failedAgain.text, /reusing a passing local result/);
  rmSync(path.join(f.root, "ui/src/new.test.ts"));
  f.write("ui/vitest.config.mjs", 'export default {test:{name:"@paperclipai/ui",include:["src/**/*.test.ts"],testTimeout:10000}};');
  const config = f.run();
  assert.equal(config.status, 2, config.text);
  assert.match(config.text, /Broader verification required/);
  assert.doesNotMatch(config.text, /Test Files/);
  assert.match(config.text, /vitest.config.mjs/);
});

test("explicit full execution cannot reuse a passing subset; opaque checks stay fresh", { timeout: 180_000 }, async t => {
  const f = fixture(t);
  assert.equal(f.run().status, 0);
  const full = f.run(["--full"]);
  assert.equal(full.status, 1, full.text);
  assert.match(full.text, /unrelated full-suite control/);
  f.write("ui/src/consumer.test.ts", 'import {test,expect} from "vitest"; import {readFileSync} from "node:fs"; import {value} from "./leaf"; test("opaque fresh check",()=>expect(readFileSync(process.env.LOCAL_VERIFY_INPUT,"utf8")).toBe(String(value)));');
  f.git("add", "ui/src/consumer.test.ts");
  f.write(".turbo/external-input", "2");
  const input = { LOCAL_VERIFY_INPUT: path.join(f.root, ".turbo/external-input") };
  const opaque = f.run([], input);
  assert.equal(opaque.status, 0, opaque.text);
  assert.match(opaque.text, /Test Files\s+1 passed/);
  const fresh = f.run([], input);
  assert.equal(fresh.status, 0, fresh.text);
  assert.match(fresh.text, /Test Files\s+1 passed/);
  f.write(".turbo/external-input", "changed externally");
  const external = f.run([], input);
  assert.equal(external.status, 1, external.text);
  assert.match(external.text, /opaque fresh check/);
});

test("working tree comparison includes commits, staging, deletion, rename and untracked files", t => {
  const f = fixture(t);
  f.git("add", "ui/src/leaf.ts");
  f.git("-c", "user.name=Test", "-c", "[email protected]", "commit", "-qm", "committed change");
  f.git("mv", "ui/src/other.ts", "ui/src/renamed.ts");
  rmSync(path.join(f.root, "cli/src/independent.test.ts"));
  f.write("ui/src/added.ts", "export const added = true;");
  assert.deepEqual(comparison(f.root, f.base).changed, ["cli/src/independent.test.ts", "ui/src/added.ts", "ui/src/leaf.ts", "ui/src/other.ts", "ui/src/renamed.ts"]);
  assert.throws(() => comparison(f.root, "missing-ref"), /Cannot resolve a base/);
});

test("dependency installation, ignored dotenv and child environment enter the cache fingerprint", t => {
  const f = fixture(t), before = snapshot(f.root, f.env).hash;
  f.write(".env", "FEATURE=true\n");
  assert.notEqual(snapshot(f.root, f.env).hash, before);
  rmSync(path.join(f.root, ".env"));
  assert.equal(snapshot(f.root, f.env).hash, before);
  f.write("pnpm-lock.yaml", "lockfileVersion: '9.0'\n");
  assert.notEqual(snapshot(f.root, f.env).hash, before);
  assert.notEqual(snapshot(f.root, { ...f.env, NEW_TEST_FLAG: "1" }).hash, before);
  assert.equal(snapshot(f.root, { ...f.env, PAPERCLIP_TEST_POSTGRES_ADMIN_URL: "postgres://external.invalid/test" }).cacheable, false);
});

test("installed dependency code changes invalidate a real passing result", { timeout: 120_000 }, t => {
  const f = fixture(t);
  const installed = "node_modules/.pnpm/[email protected]/node_modules/clsx";
  f.write(`${installed}/package.json`, '{"name":"clsx","version":"1.0.0","type":"module","exports":"./index.mjs"}');
  f.write(`${installed}/index.mjs`, "export default () => 2;\n");
  symlinkSync(path.join(f.root, installed), path.join(f.root, "node_modules/clsx"), "dir");
  f.write("ui/src/consumer.test.ts", 'import {test,expect} from "vitest"; import clsx from "clsx"; import {value} from "./leaf"; test("installed dependency",()=>expect(clsx()).toBe(value));');
  const first = f.run();
  assert.equal(first.status, 0, first.text);
  const reused = f.run();
  assert.equal(reused.status, 0, reused.text);
  assert.match(reused.text, /reusing a passing local result/);
  const moduleFile = path.join(f.root, installed, "index.mjs"), original = lstatSync(moduleFile);
  f.write(`${installed}/index.mjs`, "export default () => 0;\n");
  utimesSync(moduleFile, original.atime, original.mtime);
  const changed = f.run();
  assert.equal(changed.status, 1, changed.text);
  assert.match(changed.text, /installed dependency/);
  assert.doesNotMatch(changed.text, /reusing a passing local result/);
});

test("repository skill directory links preserve reuse while target edits invalidate it", t => {
  const f = fixture(t);
  mkdirSync(path.join(f.root, ".claude/skills"), { recursive: true });
  symlinkSync("../../ui", path.join(f.root, ".claude/skills/local"), "dir");
  f.git("add", ".claude/skills/local");
  const before = snapshot(f.root, f.env);
  assert.equal(before.cacheable, true);
  f.write("ui/src/leaf.ts", "export const value = 3;\n");
  assert.notEqual(snapshot(f.root, f.env).hash, before.hash);
});

test("local chat selection schedules exactly six single-worker partitions", { timeout: 120_000 }, async t => {
  const f = fixture(t);
  f.write("ui/src/leaf.ts", "export const value = 1;\n");
  f.write("vitest.config.mjs", 'export default {test:{projects:["server"]}};');
  f.write("server/vitest.config.mjs", 'export default {test:{name:"@paperclipai/server",include:["src/**/*.test.ts"]}};');
  f.write("server/src/leaf.ts", "export const value = 1;\n");
  f.write("server/src/__tests__/chat-channels.integration.test.ts", 'import {test,expect} from "vitest"; import {value} from "../leaf"; test("chat selection",()=>expect(value).toBeGreaterThan(0));');
  f.write(".github/ci-task-matrix.json", JSON.stringify({ general_tests: [
    { task: "ci:server", group: "general-server-without-chat" }, { task: "ci:chat", group: "general-chat" },
  ] }));
  f.git("add", "."); f.git("-c", "user.name=Test", "-c", "[email protected]", "commit", "-qm", "chat baseline");
  const base = f.git("rev-parse", "HEAD");
  f.write("server/src/leaf.ts", "export const value = 2;\n");
  const plan = await planVerification({ root: f.root, env: f.env, diff: comparison(f.root, base) });
  assert.equal(plan.kind, "selected", plan.reason);
  const shards = plan.jobs.flatMap(job => JSON.parse(execFileSync(process.execPath,
    [path.join(f.root, "scripts/run-vitest-stable.mjs"), ...job.args, "--dry-run"],
    { cwd: f.root, env: f.env, encoding: "utf8" })).chatVitestShards);
  assert.deepEqual(shards, Array.from({ length: 6 }, (_, index) => ({ index, count: 6 })));
});

for (const [group, task, primary, directory] of [
  ["general-workspaces-b", "ci:workspaces-b", "@paperclipai/shared", "packages/shared"],
  ["general-workspaces-c", "ci:db", "@paperclipai/db", "packages/db"],
]) test(`${group} discovers and executes reusable and fresh selected partitions`, { timeout: 180_000 }, t => {
  const f = fixture(t);
  const inventory = JSON.parse(execFileSync(process.execPath, [path.join(f.root, "scripts/run-vitest-stable.mjs"),
    "--mode", "general", "--group", group, "--dry-run"], { cwd: f.root, env: f.env, encoding: "utf8" }));
  const projects = inventory.workspaceProjects.map((project, index) => {
    const folder = project === primary ? directory : `packages/fixture-${index}`;
    f.write(`${folder}/vitest.config.mjs`, `export default {test:{name:${JSON.stringify(project)},include:["src/**/*.test.ts"]}};`);
    f.write(`${folder}/src/independent.test.ts`, 'import {test,expect} from "vitest"; test("independent workspace",()=>expect(1).toBe(1));');
    return folder;
  });
  f.write("ui/src/leaf.ts", "export const value = 1;\n");
  f.write("vitest.config.mjs", `export default {test:{projects:${JSON.stringify(projects)}}};`);
  f.write(".github/ci-task-matrix.json", JSON.stringify({ general_tests: [{ task, group }] }));
  f.write(`${directory}/src/leaf.ts`, "export const value = 1;\n");
  f.write(`${directory}/src/reusable.test.ts`, 'import {test,expect} from "vitest"; import {value} from "./leaf"; test("workspace consumer",()=>expect(value).toBeGreaterThan(0));');
  f.write(`${directory}/src/fresh.test.ts`, 'import {test,expect} from "vitest"; import {readFileSync} from "node:fs"; import {value} from "./leaf"; test("fresh workspace consumer",()=>expect(readFileSync(process.env.LOCAL_VERIFY_INPUT,"utf8")).toBe(String(value)));');
  f.git("add", "."); f.git("-c", "user.name=Test", "-c", "[email protected]", "commit", "-qm", "workspace baseline");
  const base = f.git("rev-parse", "HEAD");
  f.write(`${directory}/src/leaf.ts`, "export const value = 2;\n");
  f.write(".turbo/external-input", "2");
  const env = { LOCAL_VERIFY_INPUT: path.join(f.root, ".turbo/external-input") };
  const first = f.run(["--base", base], env);
  assert.equal(first.status, 0, first.text);
  assert.equal((first.text.match(/Test Files\s+1 passed/g) ?? []).length, 2, first.text);
  const reused = f.run(["--base", base], env);
  assert.equal(reused.status, 0, reused.text);
  assert.ok(reused.text.includes(`Reusing passed ${group}-reusable`), reused.text);
  assert.equal((reused.text.match(/Test Files\s+1 passed/g) ?? []).length, 1, reused.text);
  f.write(".turbo/external-input", "changed externally");
  const changed = f.run(["--base", base], env);
  assert.equal(changed.status, 1, changed.text);
  assert.match(changed.text, /fresh workspace consumer/);
});

test("fingerprints preserve file boundaries when content resembles another input record", t => {
  const f = fixture(t), a = "ui/src/hash-a.ts", b = "ui/src/hash-b.ts";
  f.write(a, ""); f.write(b, "tail");
  const boundary = JSON.stringify(b) + String(lstatSync(path.join(f.root, b)).mode);
  f.write(a, `${boundary}payload`);
  const first = snapshot(f.root, f.env).hash;
  f.write(a, ""); f.write(b, `payload${boundary}tail`);
  assert.notEqual(snapshot(f.root, f.env).hash, first);
});

test("an edit between diff discovery and fingerprinting cannot certify a partial selection", { skip: process.platform === "win32" }, t => {
  const f = fixture(t);
  const executable = execFileSync("which", ["git"], { encoding: "utf8" }).trim();
  f.write(".turbo/bin/package.json", '{"type":"commonjs"}');
  f.write(".turbo/bin/git", `#!${process.execPath}
const {execFileSync}=require('node:child_process');
const fs=require('node:fs');
const args=process.argv.slice(2);
const result=execFileSync(${JSON.stringify(executable)},args,{stdio:['ignore','pipe','pipe']});
if(args[0]==='diff'&&!args.includes('--cached')&&!fs.existsSync('.turbo/diff-edited')){
  fs.writeFileSync('ui/src/other.ts','export const other = 2;\\n');
  fs.writeFileSync('.turbo/diff-edited','done');
}
process.stdout.write(result);
`);
  chmodSync(path.join(f.root, ".turbo/bin/git"), 0o755);
  const result = f.run([], { PATH: `${path.join(f.root, ".turbo/bin")}${path.delimiter}${f.env.PATH}` });
  assert.equal(result.status, 1, result.text);
  assert.match(result.text, /Inputs changed/);
  assert.doesNotMatch(result.text, /Local Vitest verification passed/);
});

test("documentation-only changes and empty diffs do not start a full suite", async t => {
  const f = fixture(t);
  f.write("ui/src/leaf.ts", "export const value = 1;\n");
  f.write("doc/DEVELOPING.md", "Guidance only\n");
  const result = f.run(["--explain"]);
  assert.equal(result.status, 0, result.text);
  assert.match(result.text, /0 Vitest test files selected/);
  assert.doesNotMatch(result.text, /Test Files/);
  const plan = await planVerification({ root: f.root, env: f.env,
    diff: { ...comparison(f.root, f.base), changed: [] } });
  assert.deepEqual(plan.jobs, []);
});

test("local server selection retains native suites and serialized ownership under a CI caller", { timeout: 120_000 }, t => {
  const f = fixture(t);
  f.write("ui/src/leaf.ts", "export const value = 1;\n");
  f.write("vitest.config.mjs", 'export default {test:{projects:["server"]}};');
  f.write("server/vitest.config.mjs", 'export default {test:{name:"@paperclipai/server",include:["src/**/*.test.ts"]}};');
  f.write("server/src/leaf.ts", "export const value = 1;\n");
  const native = "server/src/services/native-runtime/plugin-runner-probe.test.ts";
  const serialized = "server/src/__tests__/authz-owner.test.ts";
  for (const [file, imported] of [[native, "../../leaf"], [serialized, "../leaf"]]) {
    f.write(file, `import {test,expect} from "vitest"; import {value} from ${JSON.stringify(imported)}; test("local owner",()=>expect(value).toBeGreaterThan(0));`);
  }
  f.write(".github/ci-task-matrix.json", JSON.stringify({
    general_tests: [{ task: "ci:server", group: "general-server-without-chat" }],
    verify_serialized_server: [{ task: "ci:serialized" }],
  }));
  f.git("add", "."); f.git("-c", "user.name=Test", "-c", "[email protected]", "commit", "-qm", "server baseline");
  const base = f.git("rev-parse", "HEAD");
  f.write("server/src/leaf.ts", "export const value = 2;\n");
  const result = f.run(["--base", base], { GITHUB_WORKFLOW: "PR" });
  assert.equal(result.status, 0, result.text);
  assert.equal((result.text.match(/Test Files\s+1 passed/g) ?? []).length, 2);
  const record = JSON.parse(readFileSync(path.join(f.root, ".turbo/local-verify/last-run.json"), "utf8"));
  assert.deepEqual(record.plan.manifest.lanes["general-server-without-chat"].shards[0]["@paperclipai/server"], [native]);
  assert.deepEqual(record.plan.manifest.lanes.serialized.shards[0]["@paperclipai/server"], [serialized]);
});

test("source mutation and interruption do not publish reusable passes", { timeout: 120_000 }, async t => {
  const f = fixture(t);
  const runControlled = action => new Promise((resolve, reject) => {
    const child = spawn(process.execPath, [path.join(f.root, "scripts/local-verify.mjs"), "--base", f.base, "--no-cache"], {
      cwd: f.root, env: f.env, stdio: ["ignore", "pipe", "pipe"],
    });
    let text = "", acted = false;
    const timer = setTimeout(() => { child.kill("SIGTERM"); reject(new Error("Controlled verification did not finish")); }, 60_000);
    const collect = bytes => {
      text += bytes;
      if (!acted && text.includes("[verify] Running")) { acted = true; action(child); }
    };
    child.stdout.on("data", collect); child.stderr.on("data", collect);
    child.once("error", reject);
    child.once("close", (code, signal) => { clearTimeout(timer); resolve({ code, signal, text, acted }); });
  });
  const changed = await runControlled(() => f.write("ui/src/leaf.ts", "export const value = 4;\n"));
  assert.equal(changed.acted, true);
  assert.equal(changed.code, 1, changed.text);
  assert.match(changed.text, /Inputs changed during verification/);
  const beforeInterrupt = f.run();
  assert.equal(beforeInterrupt.status, 0, beforeInterrupt.text);
  const interrupted = await runControlled(child => child.kill("SIGTERM"));
  assert.equal(interrupted.acted, true);
  assert.equal(interrupted.code, 143, interrupted.text);
  const result = f.run();
  assert.equal(result.status, 0, result.text);
  assert.match(result.text, /Test Files\s+1 passed/);
  assert.doesNotMatch(result.text, /reusing a passing local result/);
});