rea/scripts/verify-commandcode-client.mjs

import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
import { homedir } from "node:os";
import { isAbsolute, join, relative } from "node:path";
import { fileURLToPath } from "node:url";
import Ajv2020 from "ajv/dist/2020.js";
import {
  TOOL_CONTRACTS,
  toolContract,
} from "../dist/contracts/toolContracts.js";
import { exec } from "./lib/verify-package-core.mjs";
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs";

assert(
  process.platform !== "win32",
  "This verifier requires a disposable POSIX account; Windows is unverified",
);
const account = process.env.REA_VERIFY_COMMANDCODE_ACCOUNT_HOME;
assert(
  account && isAbsolute(account) && account === homedir(),
  "Run as a disposable account whose actual OS home matches REA_VERIFY_COMMANDCODE_ACCOUNT_HOME; do not override HOME",
);
assert.equal(
  await readFile(join(account, ".rea-client-verification"), "utf8"),
  "Disposable REA client verification account\n",
);
const repo = fileURLToPath(new URL("..", import.meta.url));
const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;
const command = process.env.REA_VERIFY_COMMANDCODE_COMMAND ?? "cmd";
const mode = process.argv[2] ?? "call";
assert(
  ["chat", "call"].includes(mode),
  "Usage: verify-commandcode-client.mjs [chat|call]",
);
const verifier = createVerifierRun();
const lab = await mkdtemp(join(account, "rea-commandcode-client-"));
const workspace = join(lab, "workspace");
const target = join(workspace, "fixture α测试");
const profile = join(account, ".commandcode");
const configPath = join(profile, "mcp.json");
const skillDirectory = join(
  account,
  ".agents",
  "skills",
  "reverse-engineer-anything",
);
await Promise.all([
  mkdir(target, { recursive: true }),
  mkdir(profile, { recursive: true }),
  mkdir(join(lab, "tmp")),
]);
await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 });
await writeFile(
  join(target, "app.js"),
  'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n',
);
const environment = {
  PATH: process.env.PATH,
  NODE_OPTIONS: process.env.NODE_OPTIONS,
  USERPROFILE: account,
  TMPDIR: join(lab, "tmp"),
  CMD_LOCAL_ONLY: "1",
  // Native print mode checks for an account key even with a keyless BYOK model.
  // This synthetic value is never a real credential; local-only refuses hosted API calls.
  COMMAND_CODE_API_KEY: "local-fixture-only",
  COMMANDCODE_SKIP_UPDATES: "1",
  COMMANDCODE_DISABLE_CRON: "1",
  OTEL_SDK_DISABLED: "true",
  REA_PROCESS_RUN_ID: verifier.run_id,
};
const expectedNames = TOOL_CONTRACTS.map(
  (tool) => "mcp__rea__" + tool.name,
).sort();
const schemaValidator = new Ajv2020({ strict: false, validateFormats: false });
const textValues = (value) =>
  typeof value === "string"
    ? [value]
    : value !== null && typeof value === "object"
      ? Object.values(value).flatMap(textValues)
      : [];
let clientVersion;
let expectedSkill;
let inventoryChecked = false;
let skillLoaded = false;
let schemasChecked = 0;
let catalogArtifact;
let catalogDigest;
let catalogProjection;
let artifactProjectionSeen = false;
let evidenceSeen = false;
let evidenceBytes;
let finalSeen = false;
let stage = 0;
let failure;
const call = (name, args) => ({
  role: "assistant",
  content: null,
  tool_calls: [
    {
      id: `call_commandcode_${++stage}`,
      type: "function",
      function: { name, arguments: JSON.stringify(args) },
    },
  ],
});
const validateCatalog = async (text) => {
  const entries = [
    ...text.matchAll(
      /^### (mcp__rea__[A-Za-z0-9_]+)\n[\s\S]*?\nParameters:\n```json\n([\s\S]*?)\n```/gm,
    ),
  ];
  assert.deepEqual(
    entries.map((entry) => entry[1]).sort(),
    expectedNames,
    "Native exact-name search must return every REA input schema",
  );
  for (const entry of entries) {
    const schema = JSON.parse(entry[2]);
    assert(
      schemaValidator.validateSchema(schema),
      `${entry[1]}: invalid native schema: ${JSON.stringify(schemaValidator.errors)}`,
    );
  }
  schemasChecked = entries.length;
  catalogDigest = createHash("sha256").update(text).digest("hex");
  await writeFile(join(lab, "native-catalog.md"), text);
};
const fixture = await createOpenAiModelFixture({
  directory: lab,
  prefix: "commandcode",
  onRequest: async ({ body, tools }) => {
    assert(tools.some((tool) => tool.name === "search_tools"));
    const strings = textValues(body.messages);
    if (!inventoryChecked) {
      const names = [
        ...new Set(
          strings.flatMap(
            (text) => text.match(/mcp__rea__[A-Za-z0-9_]+/g) ?? [],
          ),
        ),
      ].sort();
      assert.deepEqual(
        names,
        expectedNames,
        "Default native deferred-tool prompt must advertise the complete REA inventory",
      );
      assert.equal(
        tools.filter((tool) => tool.name.startsWith("mcp__rea__")).length,
        0,
        "Keep native lazy schema delivery enabled",
      );
      assert(
        tools
          .find((tool) => tool.name === "activate_skill")
          ?.parameters.properties.name.enum.includes(
            "reverse-engineer-anything",
          ),
      );
      inventoryChecked = true;
    }
    if (mode === "chat") {
      assert.equal(
        body.messages.filter((message) => message.role === "tool").length,
        0,
      );
      finalSeen = true;
      return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };
    }
    if (stage === 0)
      return call("activate_skill", { name: "reverse-engineer-anything" });
    assert(
      strings.some((text) => text.includes(expectedSkill)),
      "The model must receive the complete activated skill body",
    );
    skillLoaded = true;
    const last = body.messages
      .filter((message) => message.role === "tool")
      .at(-1)?.content;
    assert.equal(typeof last, "string");
    if (stage === 1)
      return call("search_tools", {
        query: "select:" + expectedNames.join(","),
      });
    if (stage === 2) {
      const spill = last.match(
        /\[full output saved to: (.+?) — read it with read_file/,
      );
      if (spill) {
        catalogArtifact = spill[1];
        const child = relative(environment.TMPDIR, catalogArtifact);
        assert(
          isAbsolute(catalogArtifact) &&
            child &&
            !child.startsWith("..") &&
            !isAbsolute(child),
          "Native spill must remain inside this run's temporary directory",
        );
        const text = await readFile(catalogArtifact, "utf8");
        await validateCatalog(text);
        catalogProjection = { schemas: schemasChecked, sha256: catalogDigest };
        const query =
          "const fs=require('node:fs'),crypto=require('node:crypto');const text=fs.readFileSync(process.argv[1],'utf8');const schemas=[...text.matchAll(/^### (mcp__rea__[A-Za-z0-9_]+)\\n/gm)].length;console.log(JSON.stringify({rea_catalog:{schemas,sha256:crypto.createHash('sha256').update(text).digest('hex')}}));";
        return call("shell_command", {
          command: process.execPath,
          args: ["-e", query, catalogArtifact],
        });
      }
      await validateCatalog(last);
      return call("mcp__rea__analyze_javascript_application", {
        input_path: target,
        format: "directory",
      });
    }
    if (catalogArtifact && !artifactProjectionSeen) {
      const line = last
        .split("\n")
        .find((item) => item.startsWith('{"rea_catalog":'));
      assert(line, "Native shell must return the selected saved-catalog facts");
      assert.deepEqual(JSON.parse(line).rea_catalog, catalogProjection);
      artifactProjectionSeen = true;
      return call("mcp__rea__analyze_javascript_application", {
        input_path: target,
        format: "directory",
      });
    }
    const evidence = toolContract(
      "analyze_javascript_application",
    ).outputSchema.parse(JSON.parse(last));
    assert.equal(evidence.subject.local_path, target);
    const shapes = evidence.normalized_result.graph.nodes
      .flatMap((node) => node.observations)
      .map((observation) => observation.properties)
      .filter(
        (properties) =>
          properties.semantic_role === "export-return-shapes" &&
          properties.exported_name === "result",
      );
    assert(
      shapes.length > 0 &&
        shapes.every(
          (shape) => shape.return_shape_coverage?.projection_complete,
        ),
    );
    const fields = shapes.flatMap((shape) =>
      shape.static_return_shapes.flatMap((item) => item.fields),
    );
    assert(
      fields.some(
        (field) =>
          field.path === "/clientCompatibility" &&
          field.state === "literal" &&
          field.value === "REA_CLIENT_FIXTURE",
      ),
    );
    assert(
      fields.some(
        (field) =>
          field.path === "/count" &&
          field.state === "literal" &&
          field.value === 7,
      ),
    );
    evidenceBytes = Buffer.byteLength(last);
    await writeFile(join(lab, "native-evidence.json"), last);
    evidenceSeen = true;
    finalSeen = true;
    return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };
  },
});
await writeFile(
  join(profile, "providers.json"),
  JSON.stringify({
    provider: {
      "rea-local": {
        baseURL: fixture.baseUrl,
        apiKey: false,
        models: {
          "rea-client-fixture": { contextWindow: 1_000_000, maxOutput: 4096 },
        },
      },
    },
  }),
);
const original =
  JSON.stringify(
    {
      mcpServers: { other: { command: "unrelated-server", enabled: false } },
      preserved: "keep unrelated settings",
    },
    null,
    2,
  ) + "\n";
await writeFile(configPath, original);
const rea = (args) =>
  exec(
    process.execPath,
    [join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"],
    { env: environment, cwd: workspace, timeout: 60_000 },
  );
try {
  clientVersion = (
    await exec(command, ["--version"], {
      env: environment,
      cwd: workspace,
      timeout: 30_000,
    })
  ).stdout.trim();
  for (const args of [
    ["setup", "--client", "commandcode", "--dry-run"],
    ["setup", "--client", "commandcode", "--yes"],
    ["doctor", "--client", "commandcode", "--skill"],
  ]) {
    const response = await rea(args);
    await writeFile(
      join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`),
      response.stdout,
    );
    const result = JSON.parse(response.stdout);
    if (args[0] === "doctor") assert.equal(result.healthy, true);
    else
      assert.equal(
        result.status,
        args.includes("--dry-run") ? "planned" : "ready",
      );
    if (args.includes("--dry-run")) {
      assert(
        result.plannedActions.some((action) => action.target === configPath),
      );
      assert(
        result.plannedActions.every(
          (action) =>
            action.target === configPath ||
            action.target === skillDirectory ||
            action.target.startsWith(skillDirectory + "/"),
        ),
        "Refuse setup writes outside the disposable account's configuration and skill",
      );
    }
  }
  assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
  const configured = await readFile(configPath, "utf8");
  assert.equal(JSON.parse(configured).preserved, "keep unrelated settings");
  assert.equal(
    JSON.parse(configured).mcpServers.other.command,
    "unrelated-server",
  );
  const repeated = JSON.parse(
    (await rea(["setup", "--client", "commandcode", "--yes"])).stdout,
  );
  assert.deepEqual(repeated.appliedActions, []);
  assert.equal(await readFile(configPath, "utf8"), configured);
  assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
  expectedSkill = (await readFile(join(skillDirectory, "SKILL.md"), "utf8"))
    .replace(/^---\r?\n[\s\S]*?\r?\n---\r?\n/, "")
    .trim();
  const execution = exec(
    command,
    [
      "-p",
      mode === "chat"
        ? "Say hello briefly."
        : `Use the REA skill to analyze ${target}.`,
      "-m",
      "rea-local/rea-client-fixture",
      "--local-only",
      "--skip-onboarding",
      "--trust",
      "--output-format",
      "json",
      "--max-turns",
      "8",
      "--yolo",
    ],
    {
      env: environment,
      cwd: workspace,
      timeout: 120_000,
      maxBuffer: 10 * 1024 * 1024,
    },
  );
  execution.child.stdin.end();
  const response = await execution;
  await writeFile(join(lab, `${mode}.stdout`), response.stdout);
  await writeFile(join(lab, `${mode}.stderr`), response.stderr);
  assert.equal(fixture.failure, undefined);
  const events = response.stdout
    .trim()
    .split("\n")
    .map((line) => JSON.parse(line));
  assert(
    events.some(
      (event) =>
        event.type === "result" &&
        event.subtype === "success" &&
        event.finalText === "REA_CLIENT_COMPATIBILITY_OK",
    ),
  );
  assert(inventoryChecked && finalSeen);
  if (mode === "call")
    assert(
      skillLoaded &&
        schemasChecked === expectedNames.length &&
        evidenceSeen &&
        (!catalogArtifact || artifactProjectionSeen),
    );
} catch (cause) {
  failure = cause;
  if (typeof cause?.stdout === "string")
    await writeFile(join(lab, `${mode}.stdout`), cause.stdout);
  if (typeof cause?.stderr === "string")
    await writeFile(join(lab, `${mode}.stderr`), cause.stderr);
} finally {
  await fixture.close();
  const receipt = {
    client: "Command Code",
    clientVersion,
    mode,
    passed: failure === undefined,
    artifacts: lab,
    runtimeRoot,
    host: {
      platform: process.platform,
      architecture: process.arch,
      node: process.versions.node,
    },
    modelFixture:
      "loopback OpenAI completions/SSE, keyless BYOK, synthetic account-key gate and usage; no hosted authentication or live-model claim",
    discovery:
      "actual disposable OS account home; native user MCP configuration and default shared skill discovery; Windows unverified",
    interface:
      "default deferred schemas, complete exact-name search, native saved-catalog query when needed, actual JavaScript MCP call",
    inventoryChecked,
    catalogSize: inventoryChecked ? expectedNames.length : 0,
    schemasChecked,
    skillLoaded,
    catalogArtifact,
    catalogDigest,
    artifactProjectionSeen,
    evidenceSeen,
    evidenceBytes,
    finalSeen,
    requests: fixture.requests,
    probes: fixture.probes,
    fixtureFailure: fixture.failure,
    verifier: await completeVerifierRun(verifier),
  };
  await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2));
  console.log(JSON.stringify(receipt));
}
if (failure !== undefined) throw failure;