mirror of
https://github.com/morluto/rea.git
synced 2026-10-12 02:19:24 +00:00
test(clients): verify native Grok Build workflows (#1571)
This commit is contained in:
Ysopwnable
@@ -167,6 +167,26 @@ fix for that client/model limit. It preserves the complete catalog and schemas.
|
||||
Live model APIs, native Windows and default OS-home skill discovery remain
|
||||
unverified.
|
||||
|
||||
`verify:grok-client` requires installed Grok Build (verified with the official
|
||||
Linux x64 1.0.50 binary); select it with `REA_VERIFY_GROK_COMMAND`. This optional
|
||||
POSIX lane isolates `GROK_HOME`, an independent Git workspace and additional
|
||||
skill roots, disables foreign configuration discovery, guards setup writes,
|
||||
preserves unrelated registrations/backups and checks idempotence. Call mode
|
||||
loads the full skill through native `read_file`, queries all REA names and input
|
||||
schemas through native `search_tool`, and calls analysis through `use_tool`.
|
||||
If discovery reports `partial`, the native agent diagnoses REA registration and
|
||||
retries discovery. No startup-timeout override or fixed readiness delay is used.
|
||||
A larger Unicode-path fixture exercises native result offloading: the full
|
||||
retained Evidence is validated against its named schema, then native terminal
|
||||
queries recover the selected export facts, subject and full artifact digest in
|
||||
the next model request. This verifies useful artifact recovery without claiming
|
||||
that every offloaded graph fact enters model context. Native line-number and
|
||||
truncation envelopes remain in the request artifacts. Use `-- chat` for ordinary
|
||||
chat, or `REA_VERIFY_RUNTIME_ROOT` for a production-only installed package.
|
||||
The loopback OpenAI completions/SSE custom model declares a one-million-token
|
||||
context and synthetic usage. Live xAI APIs, native Windows and default OS-home
|
||||
skill discovery remain unverified.
|
||||
|
||||
`verify:omp-client` requires installed OMP (verified with the official Linux x64
|
||||
18.8.7 binary); select it with `REA_VERIFY_OMP_COMMAND`. This optional POSIX lane
|
||||
isolates the default agent, global config and XDG roots, guards setup targets,
|
||||
|
||||
@@ -182,6 +182,7 @@
|
||||
"verify:qwen-client": "npm run build:cached && node scripts/verify-qwen-client.mjs",
|
||||
"verify:gemini-client": "npm run build:cached && node scripts/verify-gemini-client.mjs",
|
||||
"verify:opencode-client": "npm run build:cached && node scripts/verify-opencode-client.mjs",
|
||||
"verify:grok-client": "npm run build:cached && node scripts/verify-grok-client.mjs",
|
||||
"verify:omp-client": "npm run build:cached && node scripts/verify-omp-client.mjs",
|
||||
"verify:copilot-client": "npm run build:cached && node scripts/verify-copilot-client.mjs",
|
||||
"verify:claude-client": "npm run build:cached && node scripts/verify-claude-client.mjs",
|
||||
|
||||
@@ -0,0 +1,539 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
|
||||
import { homedir, tmpdir } from "node:os";
|
||||
import { isAbsolute, join, relative } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import Ajv2020 from "ajv/dist/2020.js";
|
||||
import {
|
||||
TOOL_CONTRACTS,
|
||||
toolContract,
|
||||
} from "../dist/contracts/toolContracts.js";
|
||||
import { exec } from "./lib/verify-package-core.mjs";
|
||||
import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs";
|
||||
import { createOpenAiModelFixture } from "./verify/clients/model-fixture.mjs";
|
||||
|
||||
assert(
|
||||
process.platform !== "win32",
|
||||
"Native Grok verifier requires POSIX; Windows is unverified",
|
||||
);
|
||||
const repo = fileURLToPath(new URL("..", import.meta.url));
|
||||
const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo;
|
||||
const command = process.env.REA_VERIFY_GROK_COMMAND ?? "grok";
|
||||
const mode = process.argv[2] ?? "call";
|
||||
assert(
|
||||
["chat", "call"].includes(mode),
|
||||
"Usage: verify-grok-client.mjs [chat|call]",
|
||||
);
|
||||
const verifier = createVerifierRun();
|
||||
const lab = await mkdtemp(join(tmpdir(), "rea-grok-client-"));
|
||||
const account = join(lab, "account");
|
||||
const profile = join(lab, "selected profile");
|
||||
const workspace = join(lab, "workspace");
|
||||
const configPath = join(profile, "config.toml");
|
||||
const skillDirectory = join(
|
||||
account,
|
||||
".agents",
|
||||
"skills",
|
||||
"reverse-engineer-anything",
|
||||
);
|
||||
await Promise.all([mkdir(account), mkdir(profile), mkdir(workspace)]);
|
||||
await exec("git", ["init", "--quiet", workspace], { timeout: 30_000 });
|
||||
// Native compatibility discovery is independent of GROK_HOME. Disable those
|
||||
// sources and ignore the real shared skill root instead of changing HOME.
|
||||
const environment = {
|
||||
PATH: process.env.PATH,
|
||||
NODE_OPTIONS: process.env.NODE_OPTIONS,
|
||||
USERPROFILE: account,
|
||||
GROK_HOME: profile,
|
||||
GROK_DISABLE_AUTOUPDATER: "1",
|
||||
...Object.fromEntries(
|
||||
["CLAUDE", "CURSOR"].flatMap((vendor) =>
|
||||
["SKILLS", "RULES", "AGENTS", "MCPS", "HOOKS", "SESSIONS"].map((cell) => [
|
||||
`GROK_${vendor}_${cell}_ENABLED`,
|
||||
"false",
|
||||
]),
|
||||
),
|
||||
),
|
||||
XDG_DATA_HOME: join(lab, "data"),
|
||||
XDG_CONFIG_HOME: join(lab, "config"),
|
||||
XDG_CACHE_HOME: join(lab, "cache"),
|
||||
REA_PROCESS_RUN_ID: verifier.run_id,
|
||||
};
|
||||
const target = join(lab, "fixture α测试");
|
||||
await mkdir(target);
|
||||
await writeFile(
|
||||
join(target, "app.js"),
|
||||
'export function result() { return { clientCompatibility: "REA_CLIENT_FIXTURE", count: 7 }; }\n' +
|
||||
// A real analysis result large enough to exercise native MCP offloading.
|
||||
Array.from(
|
||||
{ length: 60 },
|
||||
(_, index) =>
|
||||
`export function extra${index}() { return { index: ${index}, tag: "GROK_LARGE_FIXTURE" }; }`,
|
||||
).join("\n"),
|
||||
);
|
||||
const expectedNames = TOOL_CONTRACTS.map((tool) => "rea__" + tool.name).sort();
|
||||
assert(
|
||||
expectedNames.length <= 255,
|
||||
"Native search_tool limit is uint8; this full-catalog query cannot cover more than 255 tools",
|
||||
);
|
||||
const schemaValidator = new Ajv2020({ strict: false, validateFormats: false });
|
||||
const shellQuote = (value) => "'" + value.replaceAll("'", "'\"'\"'") + "'";
|
||||
const containsText = (value, text) => {
|
||||
if (typeof value === "string") return value.includes(text);
|
||||
return (
|
||||
value !== null &&
|
||||
typeof value === "object" &&
|
||||
Object.values(value).some((item) => containsText(item, text))
|
||||
);
|
||||
};
|
||||
let clientVersion;
|
||||
let expectedSkill;
|
||||
let turn = 0;
|
||||
let skillRequested = false;
|
||||
let skillLoaded = false;
|
||||
let diagnosticsRequested = false;
|
||||
let diagnosticsVerified = false;
|
||||
let lastTool;
|
||||
let catalogChecked = false;
|
||||
let schemasChecked = 0;
|
||||
let analysisRequested = false;
|
||||
let evidenceSeen = false;
|
||||
let spillQueryRequested = false;
|
||||
let projectionSeen = false;
|
||||
let artifactDigest;
|
||||
let artifactBytes;
|
||||
let retainedArtifact;
|
||||
let expectedProjection;
|
||||
let finalSeen = false;
|
||||
let failure;
|
||||
const call = (name, args) => {
|
||||
lastTool = name;
|
||||
return {
|
||||
role: "assistant",
|
||||
content: null,
|
||||
tool_calls: [
|
||||
{
|
||||
id: `call_grok_${turn}`,
|
||||
type: "function",
|
||||
function: { name, arguments: JSON.stringify(args) },
|
||||
},
|
||||
],
|
||||
};
|
||||
};
|
||||
const verifyEvidence = (bytes) => {
|
||||
const evidence = toolContract(
|
||||
"analyze_javascript_application",
|
||||
).outputSchema.parse(JSON.parse(bytes.toString("utf8")));
|
||||
assert.equal(evidence.subject.local_path, target);
|
||||
const shapes = evidence.normalized_result.graph.nodes
|
||||
.flatMap((node) => node.observations)
|
||||
.map((observation) => observation.properties)
|
||||
.filter(
|
||||
(properties) =>
|
||||
properties.semantic_role === "export-return-shapes" &&
|
||||
properties.exported_name === "result",
|
||||
);
|
||||
assert(
|
||||
shapes.length > 0 &&
|
||||
shapes.every((shape) => shape.return_shape_coverage?.projection_complete),
|
||||
);
|
||||
const fields = shapes.flatMap((shape) =>
|
||||
shape.static_return_shapes.flatMap((item) => item.fields),
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(field) =>
|
||||
field.path === "/clientCompatibility" &&
|
||||
field.state === "literal" &&
|
||||
field.value === "REA_CLIENT_FIXTURE",
|
||||
),
|
||||
);
|
||||
assert(
|
||||
fields.some(
|
||||
(field) =>
|
||||
field.path === "/count" &&
|
||||
field.state === "literal" &&
|
||||
field.value === 7,
|
||||
),
|
||||
);
|
||||
artifactDigest = createHash("sha256").update(bytes).digest("hex");
|
||||
artifactBytes = bytes.length;
|
||||
evidenceSeen = true;
|
||||
return {
|
||||
evidence_id: evidence.evidence_id,
|
||||
subject: evidence.subject,
|
||||
operation: evidence.operation,
|
||||
artifact_sha256: artifactDigest,
|
||||
byte_length: artifactBytes,
|
||||
shapes,
|
||||
};
|
||||
};
|
||||
const diagnosticFile = join(lab, "native-doctor.json");
|
||||
const diagnosticCode =
|
||||
"const fs=require('node:fs');const d=JSON.parse(fs.readFileSync(process.argv[1],'utf8'));console.log(JSON.stringify({rea_client_diagnostic:{healthy:d.healthy,registrations:d.identity.registrations.filter(r=>r.client==='grok_build'),skill:d.identity.skill.state}}));";
|
||||
// Query the saved result through a native agent command. Only the facts needed
|
||||
// for this analyst question enter context; the complete named-schema Evidence
|
||||
// and its digest remain available in the native artifact.
|
||||
const recoveryCode =
|
||||
"const fs=require('node:fs'),crypto=require('node:crypto');const bytes=fs.readFileSync(process.argv[1]),e=JSON.parse(bytes);const shapes=e.normalized_result.graph.nodes.flatMap(n=>n.observations).map(o=>o.properties).filter(p=>p.semantic_role==='export-return-shapes'&&p.exported_name==='result');console.log(JSON.stringify({rea_recovered:{evidence_id:e.evidence_id,subject:e.subject,operation:e.operation,artifact_sha256:crypto.createHash('sha256').update(bytes).digest('hex'),byte_length:bytes.length,shapes}}));";
|
||||
const fixture = await createOpenAiModelFixture({
|
||||
directory: lab,
|
||||
prefix: "grok",
|
||||
onRequest: async ({ body, tools }) => {
|
||||
// Native title/summary requests cannot execute the analyst workflow.
|
||||
if (!tools.some((tool) => tool.name === "search_tool"))
|
||||
return { role: "assistant", content: "REA fixture auxiliary response" };
|
||||
turn++;
|
||||
assert(
|
||||
containsText(body.messages, skillDirectory),
|
||||
"Grok must discover the isolated installed skill",
|
||||
);
|
||||
assert(
|
||||
!containsText(body.messages, join(homedir(), ".agents", "skills")),
|
||||
"Do not inherit real personal skills",
|
||||
);
|
||||
if (mode === "chat") {
|
||||
finalSeen = true;
|
||||
return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };
|
||||
}
|
||||
const replies = body.messages.filter((message) => message.role === "tool");
|
||||
if (!skillRequested) {
|
||||
skillRequested = true;
|
||||
return call("read_file", {
|
||||
target_file: join(skillDirectory, "SKILL.md"),
|
||||
});
|
||||
}
|
||||
assert(
|
||||
expectedSkill !== undefined && containsText(replies, expectedSkill),
|
||||
"Native read_file must return the full skill with its documented line-number rendering",
|
||||
);
|
||||
skillLoaded = true;
|
||||
const last = replies.at(-1)?.content;
|
||||
assert.equal(typeof last, "string");
|
||||
if (!catalogChecked) {
|
||||
const discovery =
|
||||
lastTool === "search_tool" ? JSON.parse(last) : undefined;
|
||||
if (discovery?.status === "partial" && !diagnosticsRequested) {
|
||||
diagnosticsRequested = true;
|
||||
const diagnostic = [
|
||||
process.execPath,
|
||||
join(runtimeRoot, "scripts/rea.mjs"),
|
||||
"doctor",
|
||||
"--client",
|
||||
"grok_build",
|
||||
"--skill",
|
||||
"--json",
|
||||
]
|
||||
.map(shellQuote)
|
||||
.join(" ");
|
||||
const projection = [
|
||||
process.execPath,
|
||||
"-e",
|
||||
diagnosticCode,
|
||||
diagnosticFile,
|
||||
]
|
||||
.map(shellQuote)
|
||||
.join(" ");
|
||||
return call("run_terminal_command", {
|
||||
command: `${diagnostic} > ${shellQuote(diagnosticFile)} && ${projection}`,
|
||||
description:
|
||||
"Diagnose REA registration while native MCP discovery is pending",
|
||||
});
|
||||
}
|
||||
if (lastTool === "run_terminal_command") {
|
||||
const line = last
|
||||
.split("\n")
|
||||
.find((item) => item.startsWith('{"rea_client_diagnostic":'));
|
||||
assert(
|
||||
line,
|
||||
"Native REA diagnostics failed or did not return the selected status",
|
||||
);
|
||||
const diagnostic = JSON.parse(line).rea_client_diagnostic;
|
||||
assert.equal(diagnostic.healthy, true);
|
||||
assert.equal(diagnostic.skill, "aligned");
|
||||
assert(
|
||||
diagnostic.registrations.some(
|
||||
(registration) =>
|
||||
registration.state === "aligned" &&
|
||||
registration.config_path === configPath,
|
||||
),
|
||||
);
|
||||
diagnosticsVerified = true;
|
||||
}
|
||||
if (discovery?.status !== "ready")
|
||||
return call("search_tool", { query: "rea", limit: 255 });
|
||||
const inventory = discovery.results
|
||||
.filter((result) => result.server === "rea")
|
||||
.flatMap((result) => result.tools);
|
||||
assert.deepEqual(
|
||||
inventory.map((tool) => tool.tool_name).sort(),
|
||||
expectedNames,
|
||||
"Native search_tool must discover the complete REA catalog",
|
||||
);
|
||||
for (const tool of inventory) {
|
||||
assert(tool.input_schema, `${tool.tool_name}: missing input schema`);
|
||||
assert(
|
||||
schemaValidator.validateSchema(tool.input_schema),
|
||||
`${tool.tool_name}: invalid advertised schema: ${JSON.stringify(schemaValidator.errors)}`,
|
||||
);
|
||||
schemasChecked++;
|
||||
}
|
||||
await writeFile(join(lab, "native-discovery.json"), last);
|
||||
catalogChecked = true;
|
||||
analysisRequested = true;
|
||||
return call("use_tool", {
|
||||
tool_name: "rea__analyze_javascript_application",
|
||||
tool_input: { input_path: target, format: "directory" },
|
||||
});
|
||||
}
|
||||
if (!spillQueryRequested) {
|
||||
const marker = "Full output written to: ";
|
||||
const start = last.indexOf(marker);
|
||||
if (start < 0) {
|
||||
verifyEvidence(Buffer.from(last, "utf8"));
|
||||
finalSeen = true;
|
||||
return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };
|
||||
}
|
||||
// The native hint explicitly calls for a query rather than read_file on
|
||||
// the saved JSON's very long line. Preserve the original hint in requests.
|
||||
const file = last
|
||||
.slice(start + marker.length)
|
||||
.split(". The full output ")[0];
|
||||
assert(isAbsolute(file));
|
||||
const childPath = relative(join(profile, "sessions"), file);
|
||||
assert(
|
||||
childPath !== "" &&
|
||||
!childPath.startsWith("..") &&
|
||||
!isAbsolute(childPath) &&
|
||||
file.endsWith(".json"),
|
||||
"Unexpected native MCP spill path",
|
||||
);
|
||||
retainedArtifact = file;
|
||||
expectedProjection = verifyEvidence(await readFile(file));
|
||||
spillQueryRequested = true;
|
||||
return call("run_terminal_command", {
|
||||
command: [process.execPath, "-e", recoveryCode, file]
|
||||
.map(shellQuote)
|
||||
.join(" "),
|
||||
description:
|
||||
"Query full saved REA Evidence while preserving artifact identity",
|
||||
});
|
||||
}
|
||||
const line = last
|
||||
.split("\n")
|
||||
.find((item) => item.startsWith('{"rea_recovered":'));
|
||||
assert(
|
||||
line,
|
||||
"The next model request must contain native artifact query output",
|
||||
);
|
||||
const projection = JSON.parse(line).rea_recovered;
|
||||
assert.deepEqual(
|
||||
projection,
|
||||
expectedProjection,
|
||||
"Native query must recover the observed facts and complete artifact digest",
|
||||
);
|
||||
projectionSeen = true;
|
||||
finalSeen = true;
|
||||
return { role: "assistant", content: "REA_CLIENT_COMPATIBILITY_OK" };
|
||||
},
|
||||
});
|
||||
const q = JSON.stringify;
|
||||
const original = `# Keep unrelated settings and server state.
|
||||
disabled_mcp_servers = ["rea", "other"]
|
||||
[cli]
|
||||
auto_update = false
|
||||
[features]
|
||||
telemetry = "off"
|
||||
[skills]
|
||||
paths = [${q(join(account, ".agents", "skills"))}]
|
||||
ignore = [${q(join(homedir(), ".agents", "skills"))}]
|
||||
[model.rea-client-fixture]
|
||||
model = "rea-client-fixture"
|
||||
base_url = ${q(fixture.baseUrl)}
|
||||
api_key = "local-fixture-key"
|
||||
api_backend = "chat_completions"
|
||||
context_window = 1000000
|
||||
max_completion_tokens = 4096
|
||||
[mcp_servers.other]
|
||||
command = "unrelated-server"
|
||||
enabled = false
|
||||
`;
|
||||
await writeFile(configPath, original);
|
||||
const rea = async (args) =>
|
||||
exec(
|
||||
process.execPath,
|
||||
[join(runtimeRoot, "scripts/rea.mjs"), ...args, "--json"],
|
||||
{ env: environment, cwd: workspace, timeout: 60_000 },
|
||||
);
|
||||
try {
|
||||
clientVersion = (
|
||||
await exec(command, ["--version"], { env: environment, timeout: 30_000 })
|
||||
).stdout.trim();
|
||||
for (const args of [
|
||||
["setup", "--client", "grok_build", "--dry-run"],
|
||||
["setup", "--client", "grok_build", "--yes"],
|
||||
["doctor", "--client", "grok_build", "--skill"],
|
||||
]) {
|
||||
const result = await rea(args);
|
||||
await writeFile(
|
||||
join(lab, `${args[0]}${args.includes("--dry-run") ? "-plan" : ""}.json`),
|
||||
result.stdout,
|
||||
);
|
||||
const parsed = JSON.parse(result.stdout);
|
||||
if (args[0] === "doctor") assert.equal(parsed.healthy, true);
|
||||
else {
|
||||
assert.equal(
|
||||
parsed.status,
|
||||
args.includes("--dry-run") ? "planned" : "ready",
|
||||
);
|
||||
if (args.includes("--dry-run")) {
|
||||
assert(
|
||||
parsed.plannedActions.some((action) => action.target === configPath),
|
||||
);
|
||||
assert(
|
||||
parsed.plannedActions.some(
|
||||
(action) => action.target === skillDirectory,
|
||||
),
|
||||
);
|
||||
assert(
|
||||
parsed.plannedActions.every(
|
||||
(action) =>
|
||||
action.target === configPath ||
|
||||
action.target === skillDirectory ||
|
||||
action.target.startsWith(skillDirectory + "/"),
|
||||
),
|
||||
"Refuse writes outside the owned profile/skill",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
|
||||
const configured = await readFile(configPath, "utf8");
|
||||
assert(
|
||||
configured.includes("unrelated-server") &&
|
||||
configured.includes("# Keep unrelated settings"),
|
||||
);
|
||||
const repeated = JSON.parse(
|
||||
(await rea(["setup", "--client", "grok_build", "--yes"])).stdout,
|
||||
);
|
||||
assert.deepEqual(repeated.appliedActions, []);
|
||||
assert.equal(await readFile(configPath, "utf8"), configured);
|
||||
assert.equal(await readFile(`${configPath}.rea.backup`, "utf8"), original);
|
||||
const skill = await readFile(join(skillDirectory, "SKILL.md"), "utf8");
|
||||
expectedSkill = skill
|
||||
.trimEnd()
|
||||
.split("\n")
|
||||
.map((line, index) =>
|
||||
index === 0 || (index + 1) % 10 === 0 ? `${index + 1}→${line}` : line,
|
||||
)
|
||||
.join("\n");
|
||||
const execution = exec(
|
||||
command,
|
||||
[
|
||||
"-p",
|
||||
mode === "chat"
|
||||
? "Say hello briefly."
|
||||
: `Use the REA skill to analyze ${target}.`,
|
||||
"-m",
|
||||
"rea-client-fixture",
|
||||
"--output-format",
|
||||
"streaming-messages-json",
|
||||
"--always-approve",
|
||||
"--no-subagents",
|
||||
],
|
||||
{
|
||||
env: environment,
|
||||
cwd: workspace,
|
||||
timeout: 120_000,
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
},
|
||||
);
|
||||
execution.child.stdin.end();
|
||||
const response = await execution;
|
||||
await writeFile(join(lab, `${mode}.stdout`), response.stdout);
|
||||
await writeFile(join(lab, `${mode}.stderr`), response.stderr);
|
||||
assert.equal(fixture.failure, undefined);
|
||||
const events = response.stdout
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(line));
|
||||
const init = events.find(
|
||||
(event) => event.type === "system" && event.subtype === "init",
|
||||
);
|
||||
assert(init?.mcp_servers?.some((server) => server.name === "rea"));
|
||||
assert(
|
||||
init.mcp_servers.every((server) => ["rea", "other"].includes(server.name)),
|
||||
"Do not inherit foreign MCP configuration",
|
||||
);
|
||||
assert(
|
||||
events.some(
|
||||
(event) =>
|
||||
event.type === "result" &&
|
||||
event.subtype === "success" &&
|
||||
event.is_error === false &&
|
||||
event.result === "REA_CLIENT_COMPATIBILITY_OK",
|
||||
),
|
||||
);
|
||||
assert(finalSeen);
|
||||
if (mode === "call")
|
||||
assert(
|
||||
skillLoaded &&
|
||||
catalogChecked &&
|
||||
analysisRequested &&
|
||||
evidenceSeen &&
|
||||
(!spillQueryRequested || projectionSeen) &&
|
||||
(!diagnosticsRequested || diagnosticsVerified),
|
||||
);
|
||||
} catch (cause) {
|
||||
failure = cause;
|
||||
if (typeof cause?.stdout === "string")
|
||||
await writeFile(join(lab, `${mode}.stdout`), cause.stdout);
|
||||
if (typeof cause?.stderr === "string")
|
||||
await writeFile(join(lab, `${mode}.stderr`), cause.stderr);
|
||||
} finally {
|
||||
await fixture.close();
|
||||
const lineage = await completeVerifierRun(verifier);
|
||||
const receipt = {
|
||||
client: "Grok Build",
|
||||
clientVersion,
|
||||
mode,
|
||||
passed: failure === undefined,
|
||||
artifacts: lab,
|
||||
runtimeRoot,
|
||||
host: {
|
||||
platform: process.platform,
|
||||
architecture: process.arch,
|
||||
node: process.versions.node,
|
||||
},
|
||||
modelFixture:
|
||||
"loopback OpenAI completions/SSE, declared one-million-token custom model and synthetic usage; no live model claim",
|
||||
discovery:
|
||||
"native GROK_HOME, isolated additional skill directory, disabled foreign compatibility sources; default OS-home skill discovery and Windows unverified",
|
||||
interface:
|
||||
"default native search_tool/use_tool with a complete explicit inventory query; asynchronous partial discovery may trigger native REA diagnosis; large MCP outputs are queried via native terminal",
|
||||
catalogChecked,
|
||||
catalogSize: catalogChecked ? expectedNames.length : 0,
|
||||
schemasChecked,
|
||||
skillLoaded,
|
||||
diagnosticsRequested,
|
||||
diagnosticsVerified,
|
||||
analysisRequested,
|
||||
evidenceSeen,
|
||||
spillQueryRequested,
|
||||
projectionSeen,
|
||||
retainedArtifact,
|
||||
artifactBytes,
|
||||
artifactDigest,
|
||||
finalSeen,
|
||||
evidenceScope:
|
||||
"complete saved Evidence validated with its named schema; the model receives the selected export facts, subject and full artifact digest, not every offloaded graph fact",
|
||||
requests: fixture.requests,
|
||||
probes: fixture.probes,
|
||||
fixtureFailure: fixture.failure,
|
||||
verifier: lineage,
|
||||
};
|
||||
await writeFile(join(lab, "receipt.json"), JSON.stringify(receipt, null, 2));
|
||||
console.log(JSON.stringify(receipt));
|
||||
}
|
||||
if (failure !== undefined) throw failure;
|
||||
Reference in New Issue
Block a user