mirror of
https://github.com/feder-cr/Jobs_Applier_AI_Agent.git
synced 2026-11-07 10:36:05 -10:00
tests: the MCP servers in the engine smoke and in the real-VM run, and the e2e contract that knows them
The engine smoke declares the engine's fake MCP server and a program that is not installed: through the image's relay the first connects as dot with its secret in its environment and on no command line, its tool runs under its permission and its instructions reach the prompt, the second fails saying what the relay found, and a config without them stops the server. The real-VM run's new step m runs mcp-server-time through uvx: its tool asks, is approved always and runs as dot; a second server waits for its secret, set through the CLI. Steps n and p look for that secret as for the key.
This commit is contained in:
1 parent
2afd73b0fe
commit
e6ebfc79f8
8 files changed
+155
-26
No files matched your search
@@ -139,11 +139,13 @@ reset_config() {
|
||||
echo '{"computer.exec":"allow"}' > /tmp/perms.json; chmod 0644 /tmp/perms.json
|
||||
echo '{}' > /tmp/models.json; chmod 0644 /tmp/models.json # the config's models (the summary role)
|
||||
echo openai/gpt-4o-mini > /tmp/model.txt; chmod 0644 /tmp/model.txt # the config's model: its limits are the stand-in's
|
||||
echo '{}' > /tmp/mcp.json; chmod 0644 /tmp/mcp.json # the config's MCP servers
|
||||
echo '{}' > /tmp/mcp-secrets.json; chmod 0600 /tmp/mcp-secrets.json # their secrets, pushed with the key
|
||||
}
|
||||
push() {
|
||||
api -o /dev/null -w '%{http_code}' -X POST -H 'content-type: application/json' -d "{\"openrouter_api_key\":\"$KEY\"}" "$A/secrets"
|
||||
api -o /dev/null -w '%{http_code}' -X POST -H 'content-type: application/json' -d "{\"openrouter_api_key\":\"$KEY\",\"mcp_secrets\":$(cat /tmp/mcp-secrets.json)}" "$A/secrets"
|
||||
echo -n " "
|
||||
api -o /dev/null -w '%{http_code}' -X PUT -H 'content-type: application/json' -d '{"name":"smoke","model":{"provider":"openrouter","id":"'"$(cat /tmp/model.txt)"'"},"permissions":'"$(cat /tmp/perms.json)"',"models":'"$(cat /tmp/models.json)"',"limits":{"max_steps_per_task":60,"max_cost_per_task_usd":1}}' "$A/config"
|
||||
api -o /dev/null -w '%{http_code}' -X PUT -H 'content-type: application/json' -d '{"name":"smoke","model":{"provider":"openrouter","id":"'"$(cat /tmp/model.txt)"'"},"permissions":'"$(cat /tmp/perms.json)"',"models":'"$(cat /tmp/models.json)"',"limits":{"max_steps_per_task":60,"max_cost_per_task_usd":1},"mcp_servers":'"$(cat /tmp/mcp.json)"'}' "$A/config"
|
||||
}
|
||||
# What the fake host runs on every agent.started: the same push, as a script.
|
||||
write_push_script() { { declare -f api push; echo "H=(-sS -H 'Authorization: Bearer $TOKEN'); A=$A; KEY=$KEY; push"; } > /tmp/push.sh; }
|
||||
|
||||
@@ -89,6 +89,11 @@ install -D -m 0644 "$ENGINE_TESTS/../nanobot/dots/invisible_playwright_mcp.json"
|
||||
FAKE_MCP=/usr/local/lib/smoke-fake/invisible-playwright-mcp
|
||||
printf '#!/bin/sh\nexec /opt/invisible-dots-engine/bin/python -I -B %s/fakes/fake_mcp_server.py "$@"\n' "$FAKE_MCP_DIR" > "$FAKE_MCP"
|
||||
chmod 0755 "$FAKE_MCP"
|
||||
# --- the stand-in for an MCP server the person declares: the engine's test fake of any server, run as dot the same way ---
|
||||
install -D -m 0644 "$ENGINE_TESTS/fakes/fake_tool_server.py" "$FAKE_MCP_DIR/fakes/fake_tool_server.py"
|
||||
FAKE_TOOLS=/usr/local/lib/smoke-fake/fake-tools
|
||||
printf '#!/bin/sh\nexec /opt/invisible-dots-engine/bin/python -I -B %s/fakes/fake_tool_server.py "$@"\n' "$FAKE_MCP_DIR" > "$FAKE_TOOLS"
|
||||
chmod 0755 "$FAKE_TOOLS"
|
||||
# --- dot-agentd as dotagentd, the engine as dotengine with the stand-in as its browser program ---
|
||||
MCP_COMMAND=$FAKE_MCP
|
||||
start_guest_daemons
|
||||
@@ -437,6 +442,27 @@ TABLE_TOOLS=$(grep -c '^ "[a-z_]*": ToolEntry(' "$ENGINE_TESTS/../nanobot
|
||||
check "GET /tools through dot-agentd lists the $TABLE_TOOLS tools of the table, each with a description" "[ $TABLE_TOOLS -gt 0 ] && api $A/tools | jq -e '(.tools|length)==$TABLE_TOOLS and all(.tools[]; (.description|length)>0 and (.permission|length)>0)' >/dev/null"
|
||||
check "GET /tools offers what the model was offered" "[ \"\$(api $A/tools | jq -c '[.tools[]|select(.offered)|.name]|sort')\" = '[\"exec\",\"exec_session\",\"list_exec_sessions\"]' ]"
|
||||
check "GET /tools names the permission each tool exercises" "api $A/tools | jq -e '(.tools|map({(.name):.permission})|add) | .exec==\"computer.exec\" and .read_file==\"files.read\" and .write_file==\"files.write\" and .cron==\"automations\"' >/dev/null"
|
||||
# --- the MCP servers the person declares (architecture 8.3): through the image's relay, as dot, a secret by the
|
||||
# relay's environment; one whose program is not installed says why ---
|
||||
MCP_SECRET=smoke-mcp-secret-7
|
||||
echo '{"computer.exec":"allow","mcp.tools":"allow"}' > /tmp/perms.json
|
||||
echo '{"tools":{"command":"'"$FAKE_TOOLS"'","env":{"MODE":"smoke"},"secrets":["TOKEN"],"timeout_s":30},"missing":{"command":"/usr/local/bin/not-installed","timeout_s":30}}' > /tmp/mcp.json
|
||||
echo '{"tools":{"TOKEN":"'"$MCP_SECRET"'"}}' > /tmp/mcp-secrets.json
|
||||
check "the host pushes two MCP servers and a secret of one (204 204)" "[ \"\$(push)\" = '204 204' ]"
|
||||
mcp_state() { api $A/tools | jq -r --arg n "$1" '.mcp_servers[]|select(.name==$n)|.state'; }
|
||||
wait_mcp_state() { for _ in $(seq 1 60); do [ "$(mcp_state "$1")" = "$2" ] && return 0; sleep 1; done; return 1; }
|
||||
check "GET /tools: the declared server connects through the relay, with its tools under mcp.tools" "wait_mcp_state tools connected && api $A/tools | jq -e '[.tools[]|select(.permission==\"mcp.tools\")|.name]|sort == [\"mcp_tools_echo\",\"mcp_tools_env\",\"mcp_tools_exit\",\"mcp_tools_header\",\"mcp_tools_picture\"] and ([.tools[]|select(.permission==\"mcp.tools\")|.offered]|all)' >/dev/null"
|
||||
check "GET /tools: the server whose program is not installed is failed, saying what the relay found" "wait_mcp_state missing failed && api $A/tools | jq -e '.mcp_servers[]|select(.name==\"missing\")|.error|test(\"not-installed\")' >/dev/null"
|
||||
check "the declared server runs as dot, its secret in its environment and on no command line" "pgrep -u dot -f fake_tool_server.py >/dev/null && for p in \$(pgrep -u dot -f fake_tool_server.py); do tr '\\0' '\\n' < /proc/\$p/environ; done | grep -qx 'TOKEN=$MCP_SECRET' && ! cat /proc/[0-9]*/cmdline 2>/dev/null | tr '\\0' ' ' | grep -q '$MCP_SECRET'"
|
||||
ev msg-mcp user.message '{"text":"RUN-TOOL mcp_tools_env {\"name\":\"MODE\"}"}' >/dev/null
|
||||
check "a call of the server's tool runs under mcp.tools and answers with the server's environment" "wait_event $STREAM '.type==\"tool.called\" and .data.tool==\"mcp_tools_env\" and .data.ok==true and .data.permission==\"mcp.tools\"' && wait_event $STREAM '.type==\"message.assistant\" and .data.in_reply_to==\"msg-mcp\" and (.data.text|test(\"smoke\"))'"
|
||||
check "the prompt of that turn carries the server's instructions, and why the other is not connected" "tail -n 2 /tmp/fake-full.jsonl | grep -q 'Call echo to repeat a text.' && tail -n 2 /tmp/fake-full.jsonl | grep -q 'Not connected: '"
|
||||
check "the secret reached no request the model was sent" "! grep -q '$MCP_SECRET' /tmp/fake-full.jsonl"
|
||||
echo '{"computer.exec":"allow"}' > /tmp/perms.json
|
||||
echo '{}' > /tmp/mcp.json
|
||||
echo '{}' > /tmp/mcp-secrets.json
|
||||
check "the host pushes the allow-exec config with no server again (204 204)" "[ \"\$(push)\" = '204 204' ]"
|
||||
check "a server the config no longer declares is stopped, its tools gone" "for _ in \$(seq 1 30); do pgrep -u dot -f fake_tool_server.py >/dev/null || break; sleep 1; done; ! pgrep -u dot -f fake_tool_server.py >/dev/null && api $A/tools | jq -e '(.mcp_servers==[]) and ([.tools[]|select(.permission|startswith(\"mcp.\"))]==[])' >/dev/null"
|
||||
# The host reads the Dot's files through the TCP port, which is limited to /home/dot with every symbolic link followed.
|
||||
# A link a page could make the Dot stage must not show the proxy password in /proc/<pid>/environ of the browser server
|
||||
# (it runs as dot), nor the Dot's token. The engine's socket takes any path dot can open.
|
||||
|
||||
+9
-7
@@ -49,9 +49,10 @@ tested by `tests/repo/e2e.test.ts` on every push:
|
||||
| j | the server restarts and adopts the running VM (same pid, browser still open); `invisible-dots computer stop` reaches STOPPED with QEMU gone, through the guest's own poweroff (`computer.stopped` says `forced: false`, in well under the 60 s after which QEMU is killed); the open browser was closed on the way down (`browser.identity.closed`); `start` reaches READY with no browser server running; the identity is launched again and its `.stealth-identity.json` is unchanged (by SHA-256); `title.txt`, the note, a `grep` that finds it, and the conversation are all still there; the guest's whole system journal, read inside the guest, holds no OpenRouter key |
|
||||
| k | with `computer.exec: ask`, a task stops at an approval; the **computer is killed** (SIGKILL of QEMU) while it waits; the control plane records `computer.stopped` with `reason: exited` and starts the computer again by itself (the Dot still has a task), a second `agent.started` follows, the engine resumes from its database with the same approval pending and the host re-pushes key and config; approved with a note, the task completes and a counter file proves the command ran exactly once |
|
||||
| l | a task runs a slow `exec`; while it is in flight the computer is killed the same way; after the restart the Dot's event log has that call once, as a `tool.called` with `interrupted: true`, and once the command's time is over no copy of it is running and its marker file was never written: the command died with its computer and was not run again (architecture 8.7) |
|
||||
| m | the key is in none of the Dot's event and approval rows (read back decompressed through the API) and, once the Dot is stopped, in none of its files: the overlay disk, the seed, the serial log and QEMU's log |
|
||||
| n | `DELETE /api/dots/:id` removes the Dot, its QEMU process and `vms/<id>` |
|
||||
| o | the key is in no event row read back after the delete, and in no file of the run's logs, `logs/` or the embedded database's directory |
|
||||
| m | two MCP servers in the config, both `uvx mcp-server-time`: `time`, whose permission asks, and `keyed`, which names the secret `TIME_TOKEN`. `keyed` is `failed` because its secret is not set; a task's call of `mcp_time_get_current_time` stops at an approval for `mcp.time`, approved always, and runs; `GET .../tools` lists `time` connected with its tools offered, and its process runs as `dot`. The secret is set through `invisible-dots secret mcp` (stdin, not echoed), `keyed` connects at the next turn, `invisible-dots mcp` says so, and the secret is in the server's environment and on no command line in the guest |
|
||||
| n | the key and the MCP secret are in none of the Dot's event and approval rows (read back decompressed through the API) and, once the Dot is stopped, in none of its files: the overlay disk, the seed, the serial log and QEMU's log |
|
||||
| o | `DELETE /api/dots/:id` removes the Dot, its QEMU process and `vms/<id>` |
|
||||
| p | the key and the MCP secret are in no event row read back after the delete, and in no file of the run's logs, `logs/` or the embedded database's directory |
|
||||
|
||||
Steps k and l end the computer, not the engine alone. The engine runs as
|
||||
`dotengine`, dot-agentd as `dotagentd` and everything the model runs as `dot`, and no
|
||||
@@ -71,10 +72,11 @@ of the key. They look for its first 12 characters; the journal check of step j
|
||||
looks inside the guest for the `sk-or-` prefix every OpenRouter key has (the run
|
||||
refuses a key without it), without giving the model any part of the key, and is
|
||||
only accepted when the guest proves it read the system journal. Large rows are
|
||||
compressed inside the database's files, which is why step m reads them through
|
||||
the API as well as step o scanning the files. The guest's journal compresses
|
||||
large entries too, which is why step j reads it inside the guest besides step m
|
||||
scanning the disk.
|
||||
compressed inside the database's files, which is why step n reads them through
|
||||
the API as well as step p scanning the files. The guest's journal compresses
|
||||
large entries too, which is why step j reads it inside the guest besides step n
|
||||
scanning the disk. The MCP secret of step m is looked for whole: it is made for
|
||||
the run and never shown to the model.
|
||||
|
||||
A file the Dot wrote is checked by having the Dot hash it in the guest
|
||||
(`printf '%s' "$(cat <file>)" | sha256sum`, so a trailing newline does not
|
||||
|
||||
+1
-1
@@ -102,7 +102,7 @@ export class Product {
|
||||
|
||||
/**
|
||||
* One `invisible-dots` call. Its arguments and output go to the CLI log; stdin
|
||||
* (only ever the key) does not. With `stream`, output is also appended to
|
||||
* (only ever a secret: the key, an MCP server's) does not. With `stream`, output is also appended to
|
||||
* that file as it arrives, for a long command whose progress is followed.
|
||||
*/
|
||||
cli = async (args: string[], options: { stdin?: string; timeoutMs?: number; stream?: string } = {}): Promise<CliResult> => {
|
||||
|
||||
@@ -447,6 +447,8 @@ export interface DotYamlOptions {
|
||||
permissions?: Record<string, "allow" | "ask" | "deny">;
|
||||
/** The computer's size; 2 CPUs and 4gb when not given. */
|
||||
computer?: { cpu: number; memory: string };
|
||||
/** MCP servers the Dot runs on its computer, by name (architecture section 7). */
|
||||
mcpServers?: Record<string, { command: string; args?: string[]; secrets?: string[] }>;
|
||||
}
|
||||
|
||||
/** The Dot's config (architecture section 7) as YAML. */
|
||||
@@ -464,6 +466,17 @@ export function dotYaml(options: DotYamlOptions): string {
|
||||
` memory: ${options.computer?.memory ?? "4gb"}`,
|
||||
" idle_timeout: 0",
|
||||
...(permissions.length > 0 ? ["permissions:", ...permissions.map(([permission, decision]) => ` ${permission}: ${decision}`)] : []),
|
||||
...(options.mcpServers
|
||||
? [
|
||||
"mcp_servers:",
|
||||
...Object.entries(options.mcpServers).flatMap(([name, server]) => [
|
||||
` ${name}:`,
|
||||
` command: ${server.command}`,
|
||||
...(server.args ? [` args: ${JSON.stringify(server.args)}`] : []),
|
||||
...(server.secrets ? [` secrets: ${JSON.stringify(server.secrets)}`] : []),
|
||||
]),
|
||||
]
|
||||
: []),
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
+63
-12
@@ -115,6 +115,8 @@ const TITLE = "Example Domain";
|
||||
const WORKSPACE = "/home/dot/workspace";
|
||||
const MEMORY_NOTE = "example-title.md";
|
||||
const BROWSERS = "/home/dot/browsers";
|
||||
/** A secret of an MCP server, set in step m through the CLI; steps n and p look for it as for the key. */
|
||||
const MCP_SECRET = `mcp-e2e-${randomBytes(12).toString("hex")}`;
|
||||
|
||||
// The product, driven from the outside (driver.ts)
|
||||
|
||||
@@ -189,7 +191,7 @@ async function waitReply(dotId: string, messageId: string): Promise<StoredEvent>
|
||||
}
|
||||
|
||||
/** Rewrites the Dot's config (it is pushed to the guest at once): every step names the whole config it needs. */
|
||||
async function configure(options: Pick<DotYamlOptions, "permissions"> = {}): Promise<void> {
|
||||
async function configure(options: Pick<DotYamlOptions, "permissions" | "mcpServers"> = {}): Promise<void> {
|
||||
await api("PATCH", route(ROUTES.dot, { id: state.dotId! }), { config: dotYaml({ name: DOT_NAME, model: MODEL, ...options }) });
|
||||
}
|
||||
|
||||
@@ -898,14 +900,63 @@ async function main(): Promise<void> {
|
||||
return `VM killed during exec; tool.called with interrupted: true for the call, which was not run again; the model answered ${JSON.stringify(done.summary)}`;
|
||||
});
|
||||
|
||||
await step("m", "the key is in none of the Dot's own files and rows", async () => {
|
||||
await step("m", "MCP servers: one runs as dot and asks before its tool, one waits for its secret, set through the CLI", async () => {
|
||||
const dotId = state.dotId!;
|
||||
const needle = Buffer.from(key.slice(0, 12), "utf8");
|
||||
// mcp-server-time through uvx, which the image has: the engine starts it at the config push, from PyPI the first time.
|
||||
const time = { command: "uvx", args: ["mcp-server-time", "--local-timezone", "Europe/Rome"] };
|
||||
await configure({ permissions: { "mcp.time": "ask" }, mcpServers: { time, keyed: { ...time, secrets: ["TIME_TOKEN"] } } });
|
||||
type ToolTable = { tools: { name: string; permission: string; offered: boolean }[]; mcp_servers: { name: string; state: string; error: string | null; tools: number }[] };
|
||||
const tableRoute = `/api/dots/${dotId}/tools`;
|
||||
const keyed = await waitFor("the keyed server to wait for its secret", 3 * MINUTE, async () => {
|
||||
const table = await api<ToolTable>("GET", tableRoute);
|
||||
return table.mcp_servers.find((s) => s.name === "keyed" && s.state === "failed");
|
||||
}, 2000);
|
||||
assert(String(keyed.error).includes("TIME_TOKEN is not set"), `the keyed server failed with ${JSON.stringify(keyed.error)}`);
|
||||
|
||||
// A tool of the server asks (mcp.time: ask); always allowed, the next call runs without asking.
|
||||
const task = await queueTask("Use the time MCP server's get_current_time tool to get the current time in Asia/Tokyo, then answer with only the time as HH:MM.");
|
||||
const approval = await waitApproval(task.id, "mcp_time_get_current_time");
|
||||
assert(approval.permission === "mcp.time", `the approval is for ${approval.permission}`);
|
||||
await approve(approval.id, undefined, { always: true });
|
||||
await waitTask(task.id);
|
||||
const all = await events(dotId);
|
||||
const call = toolCalls(all, task.id, "mcp_time_get_current_time").find((e) => e.data.ok === true);
|
||||
assert(call && call.data.permission === "mcp.time", `no successful mcp_time_get_current_time under mcp.time (tools: ${describeTools(all, task.id)})`);
|
||||
const table = await api<ToolTable>("GET", tableRoute);
|
||||
const server = table.mcp_servers.find((s) => s.name === "time");
|
||||
assert(server?.state === "connected" && server.tools > 0, `the time server is ${JSON.stringify(server)}`);
|
||||
assert(table.tools.some((t) => t.name === "mcp_time_get_current_time" && t.permission === "mcp.time" && t.offered), "GET tools does not list the server's tool as offered");
|
||||
// It runs as dot, like every program of the model; only the engine's relay, which starts it, is another user's.
|
||||
const owner = await guestExec(dotId, "ps -eo user=,args= | grep 'mcp-server-tim[e]' | grep -v 'dot-agentd relay' | awk '{print $1}' | sort -u");
|
||||
assert(owner.stdout.trim() === "dot", `mcp-server-time runs as ${JSON.stringify(owner.stdout.trim())}, not dot`);
|
||||
|
||||
// The secret, from stdin through the CLI; the server starts with it.
|
||||
const set = await cli(["secret", "mcp", "--dot", DOT_NAME, "keyed", "TIME_TOKEN", "--json"], { stdin: `${MCP_SECRET}\n` });
|
||||
assert(set.code === 0, `invisible-dots secret mcp exited with ${set.code}: ${set.stderr.trim()}`);
|
||||
assert(!set.stdout.includes(MCP_SECRET), "the CLI echoed the secret");
|
||||
const started = await waitFor("the keyed server to start with its secret", 3 * MINUTE, async () => {
|
||||
await runTask("Answer with only OK.");
|
||||
const now = await api<ToolTable>("GET", tableRoute);
|
||||
return now.mcp_servers.find((s) => s.name === "keyed" && s.state === "connected");
|
||||
}, 5000);
|
||||
const listed = await cli(["mcp", DOT_NAME]);
|
||||
assert(listed.code === 0 && /keyed\s+connected/.test(listed.stdout), `invisible-dots mcp says: ${listed.stdout}`);
|
||||
// The secret is in the server's environment, which only its own user reads, and on no command line.
|
||||
const environ = await guestExec(dotId, `for p in $(pgrep -u dot -f mcp-server-time); do tr '\\0' '\\n' < /proc/$p/environ; done | grep -c '^TIME_TOKEN=${MCP_SECRET}$' || true`);
|
||||
assert(Number(environ.stdout.trim()) >= 1, "no mcp-server-time process holds TIME_TOKEN in its environment");
|
||||
const cmdlines = await guestExec(dotId, `cat /proc/[0-9]*/cmdline 2>/dev/null | tr '\\0' ' ' | grep -c '${MCP_SECRET}' || true`);
|
||||
assert(cmdlines.stdout.trim() === "0", "the secret is on a command line");
|
||||
return `time connected with ${server.tools} tools, its tool asked under mcp.time and ran once always allowed (${String(call.data.target ?? "no target")}); keyed waited for TIME_TOKEN, then connected with ${started.tools} tools once it was set through the CLI; the secret is in its environment and on no command line`;
|
||||
});
|
||||
|
||||
await step("n", "the key and the MCP secret are in none of the Dot's own files and rows", async () => {
|
||||
const dotId = state.dotId!;
|
||||
const needles = [Buffer.from(key.slice(0, 12), "utf8"), Buffer.from(MCP_SECRET, "utf8")];
|
||||
// The database stores large jsonb values compressed (TOAST), so a key in a long event would not show in its files
|
||||
// (step o): the rows are read back decompressed through the API, while the Dot's approvals still exist.
|
||||
// (step p): the rows are read back decompressed through the API, while the Dot's approvals still exist.
|
||||
const rows = [...(await events(dotId)), ...(await api<{ approvals: unknown[] }>("GET", route(ROUTES.approvals))).approvals];
|
||||
assert(rowsHolding(rows, needle) === 0, "found in an event or approval row of the database");
|
||||
// Stopped first, so the guest has flushed its disk and QEMU has closed its files; scanned before step n deletes them.
|
||||
for (const needle of needles) assert(rowsHolding(rows, needle) === 0, "found in an event or approval row of the database");
|
||||
// Stopped first, so the guest has flushed its disk and QEMU has closed its files; scanned before step o deletes them.
|
||||
await stopComputer(dotId, state.pid!);
|
||||
await copyFile(join(HOME, "vms", dotId, "serial.log"), join(LOG_DIR, "serial.log")).catch(() => undefined);
|
||||
const files = [...(await filesUnder(join(HOME, "vms", dotId))), join(HOME, "logs", `qemu-${dotId}.log`)];
|
||||
@@ -914,7 +965,7 @@ async function main(): Promise<void> {
|
||||
assert(files.includes(join(HOME, "vms", dotId, name)), `vms/${dotId}/${name} is not there to be scanned`);
|
||||
}
|
||||
assert(existsSync(join(HOME, "logs", `qemu-${dotId}.log`)), `logs/qemu-${dotId}.log is not there to be scanned`);
|
||||
const found = await filesHolding(files, needle);
|
||||
const found = (await Promise.all(needles.map((needle) => filesHolding(files, needle)))).flat();
|
||||
// Only "found" or "not found", and where: never the key or any part of it.
|
||||
assert(found.length === 0, `found in ${found.length} file(s): ${found.join(", ")}`);
|
||||
const bytes = (await Promise.all(files.map((file) => stat(file)))).reduce((sum, s) => sum + s.size, 0);
|
||||
@@ -924,7 +975,7 @@ async function main(): Promise<void> {
|
||||
);
|
||||
});
|
||||
|
||||
await step("n", "delete the Dot", async () => {
|
||||
await step("o", "delete the Dot", async () => {
|
||||
const dotId = state.dotId!;
|
||||
const pid = state.pid!;
|
||||
await api("DELETE", route(ROUTES.dot, { id: dotId }));
|
||||
@@ -935,11 +986,11 @@ async function main(): Promise<void> {
|
||||
return `pid ${pid} gone, vms/${dotId} removed`;
|
||||
});
|
||||
|
||||
await step("o", "the key appears in no log and no database file", async () => {
|
||||
const needle = Buffer.from(key.slice(0, 12), "utf8");
|
||||
await step("p", "the key and the MCP secret appear in no log and no database file", async () => {
|
||||
const needles = [Buffer.from(key.slice(0, 12), "utf8"), Buffer.from(MCP_SECRET, "utf8")];
|
||||
// Events outlive their Dot: read back once more, decompressed, after the delete.
|
||||
const rows = await events(state.dotId!);
|
||||
assert(rowsHolding(rows, needle) === 0, "found in an event row of the database");
|
||||
for (const needle of needles) assert(rowsHolding(rows, needle) === 0, "found in an event row of the database");
|
||||
await stopServer();
|
||||
const scanned = [
|
||||
...(await filesUnder(LOG_DIR)),
|
||||
@@ -947,7 +998,7 @@ async function main(): Promise<void> {
|
||||
// The embedded database's files, for whatever is stored uncompressed.
|
||||
...(await filesUnder(join(HOME, "db"))),
|
||||
];
|
||||
const found = await filesHolding(scanned, needle);
|
||||
const found = (await Promise.all(needles.map((needle) => filesHolding(scanned, needle)))).flat();
|
||||
// Only "found" or "not found", and where: never the key or any part of it.
|
||||
assert(found.length === 0, `found in ${found.length} file(s): ${found.join(", ")}`);
|
||||
return `not found in ${rows.length} event rows, nor in ${scanned.length} files (run logs, ${join(HOME, "logs")}, db)`;
|
||||
|
||||
+20
-4
@@ -26,6 +26,7 @@ import { USAGE } from "@invisible-dots/cli";
|
||||
import {
|
||||
type AgentStateAnswer as SharedAgentStateAnswer,
|
||||
HOST_EVENT_TYPES,
|
||||
mcpServerOf,
|
||||
OUTBOUND_EVENT_TYPES,
|
||||
PERMISSIONS,
|
||||
parseDotConfig,
|
||||
@@ -144,17 +145,32 @@ describe("the e2e run's contract with the product", () => {
|
||||
...run.matchAll(/^\s+\["([a-z_]+)", "[a-z.]+"\],?$/gm),
|
||||
].map((m) => m[1]!);
|
||||
expect(tools.length).toBeGreaterThan(10);
|
||||
for (const tool of tools) expect(Object.keys(TOOLS), `run.ts names the tool ${tool}`).toContain(tool);
|
||||
// A tool of an MCP server is the server's, not the table's: its server is one the run declares (`mcpServers`).
|
||||
const declared = (server: string) => new RegExp(`mcpServers: \\{[^}]*\\b${server}\\b`).test(run);
|
||||
for (const tool of tools) {
|
||||
const server = /^mcp_([a-z0-9-]+)_/.exec(tool)?.[1];
|
||||
if (server !== undefined) expect(declared(server), `run.ts names ${tool} of an MCP server it does not declare`).toBe(true);
|
||||
else expect(Object.keys(TOOLS), `run.ts names the tool ${tool}`).toContain(tool);
|
||||
}
|
||||
// A dotted name in quotes is an event type or a permission, unless it is one of the few hosts and files the run writes.
|
||||
const other = new Set(["example.com", "title.txt", "approval.txt", "approval-counter.txt", "exec-marker.txt", "api.token", "qemu.json", "seed.iso", "serial.log", "disk.qcow2", "screenshot.png", "frame.jpg", "summary.json", "summary.txt", "events.txt", "server.log", "image-build.log", "cli.log", "dot.yaml"]);
|
||||
const dotted = [...run.matchAll(/"([a-z]+(?:\.[a-z]+)+)"/g)].map((m) => m[1]!).filter((name) => !other.has(name));
|
||||
expect(dotted.length).toBeGreaterThan(10);
|
||||
const mcpPermissionOfDeclared = (name: string) => {
|
||||
const server = mcpServerOf(name);
|
||||
return server !== null && declared(server);
|
||||
};
|
||||
for (const name of new Set(dotted)) {
|
||||
expect((EVENTS as readonly string[]).includes(name) || (PERMISSIONS as readonly string[]).includes(name), `run.ts names ${name}: not in EVENTS and not a permission`).toBe(true);
|
||||
expect(
|
||||
(EVENTS as readonly string[]).includes(name) || (PERMISSIONS as readonly string[]).includes(name) || mcpPermissionOfDeclared(name),
|
||||
`run.ts names ${name}: not in EVENTS, not a permission, not the permission of an MCP server it declares`,
|
||||
).toBe(true);
|
||||
}
|
||||
// Every permission the run sets is one of the Dot's.
|
||||
// Every permission the run sets is one of the Dot's, or the permission of an MCP server it declares.
|
||||
for (const m of run.matchAll(/permissions: \{([^}]*)\}/g)) {
|
||||
for (const key of m[1]!.matchAll(/"([a-z.]+)":/g)) expect(PERMISSIONS as readonly string[], key[1]).toContain(key[1]);
|
||||
for (const key of m[1]!.matchAll(/"([a-z0-9.-]+)":/g)) {
|
||||
expect((PERMISSIONS as readonly string[]).includes(key[1]!) || mcpPermissionOfDeclared(key[1]!), key[1]).toBe(true);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -22,6 +22,10 @@ import {
|
||||
GUEST_PATHS,
|
||||
IDENTITY_ERROR_STATUS,
|
||||
INBOUND_EVENT_TYPES,
|
||||
MCP_SECRET_PATTERN,
|
||||
MCP_SECRET_RULE,
|
||||
MCP_SERVER_NAME_PATTERN,
|
||||
MCP_SERVER_STATES,
|
||||
MODEL_ROLES,
|
||||
OPENROUTER_KEY_PATTERN,
|
||||
OPENROUTER_KEY_RULE,
|
||||
@@ -124,6 +128,7 @@ describe("the vendored nanobot fork", () => {
|
||||
expect(tuple("OUTBOUND_EVENT_TYPES")).toEqual([...OUTBOUND_EVENT_TYPES]);
|
||||
expect(tuple("AGENT_STATES")).toEqual([...AGENT_STATES]);
|
||||
expect(tuple("MODEL_ROLES")).toEqual([...MODEL_ROLES]);
|
||||
expect(tuple("MCP_SERVER_STATES")).toEqual([...MCP_SERVER_STATES]);
|
||||
|
||||
const routesMatch = /^AGENT_ROUTES = \{([^}]*)\}/m.exec(protocol);
|
||||
expect(routesMatch, "AGENT_ROUTES").not.toBeNull();
|
||||
@@ -184,6 +189,20 @@ describe("the vendored nanobot fork", () => {
|
||||
expect(secrets).not.toMatch(/re\.compile\("/);
|
||||
});
|
||||
|
||||
it("names an MCP server and checks its secrets by the rules packages/shared names: the engine's copy", () => {
|
||||
// The host checks a server's name and a secret's value when the person enters them, the engine again on PUT /config
|
||||
// and POST /secrets, and a tool's server is read back from its name by the same rule.
|
||||
const protocol = lf(readFileSync(join(fork, "nanobot/dots/protocol.py"), "utf8"));
|
||||
expect(/^MCP_SERVER_NAME_PATTERN = "([^"]+)"/m.exec(protocol)?.[1]).toBe(MCP_SERVER_NAME_PATTERN);
|
||||
expect(/^MCP_SECRET_PATTERN = "([^"]+)"/m.exec(protocol)?.[1]).toBe(MCP_SECRET_PATTERN);
|
||||
expect(/^MCP_SECRET_RULE = "([^"]+)"/m.exec(protocol)?.[1]).toBe(MCP_SECRET_RULE);
|
||||
|
||||
const secrets = lf(readFileSync(join(fork, "nanobot/dots/secrets.py"), "utf8"));
|
||||
expect(secrets).toContain("re.compile(MCP_SERVER_NAME_PATTERN)");
|
||||
expect(secrets).toContain("re.compile(MCP_SECRET_PATTERN)");
|
||||
expect(secrets).toContain("{MCP_SECRET_RULE}");
|
||||
});
|
||||
|
||||
it("stays out of this repository's npm workspace, TypeScript project and test run", () => {
|
||||
const pkg = JSON.parse(readFileSync(join(repo, "package.json"), "utf8")) as { workspaces: string[] };
|
||||
for (const pattern of pkg.workspaces) expect(pattern.startsWith("invisible_engine_dots"), pattern).toBe(false);
|
||||
|
||||
Reference in new issue
Block a user