Merge branch 'guest-fixes' into fixes-batch

This commit is contained in:
feder-cr committed 2026-10-11 17:50:35 +02:00
commit ac26cec1bf
45 files changed
+950 -287

No files matched your search

+3 -2
View File
@@ -37,7 +37,7 @@ jobs:
with:
node-version: "24"
cache: npm
- uses: actions/setup-go@v5
- uses: actions/setup-go@v6
with:
go-version-file: guest/dot-agentd/go.mod
- name: KVM for the builder VM, and QEMU
@@ -50,7 +50,8 @@ jobs:
- run: npm ci --no-audit --no-fund
- run: npm run build --workspace @invisible-dots/cli
# The runtime ISO, which `image build` makes first, carries dot-agentd.
- run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go -C guest/dot-agentd build -trimpath -o bin/dot-agentd ./cmd/dot-agentd
- working-directory: guest/dot-agentd
run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false -ldflags="-s -w" -o bin/dot-agentd ./cmd/dot-agentd
- name: the image (downloaded instead when its release exists already)
id: build
run: |
+11 -3
View File
@@ -39,6 +39,14 @@ jobs:
with:
node-version: "24"
cache: npm
# Linux's vitest builds the real dot-agentd and drives it with the host's GuestClient
# (apps/vm-manager/test/agentd.linux.test.ts, which vitest.config.ts collects on Linux only).
- if: runner.os == 'Linux'
uses: actions/setup-go@v6
with:
go-version-file: guest/dot-agentd/go.mod
# The module has no dependencies, so there is no go.sum to key a cache on.
cache: false
- run: npm ci --no-audit --no-fund
- run: npm run typecheck
- name: the pre-push hook parses
@@ -169,7 +177,7 @@ jobs:
working-directory: guest/dot-agentd
steps:
- uses: actions/checkout@v4
- uses: actions/setup-go@v5
- uses: actions/setup-go@v6
with:
go-version-file: guest/dot-agentd/go.mod
# The module has no dependencies, so there is no go.sum to key a cache on.
@@ -186,7 +194,7 @@ jobs:
- name: refuse a run with too few tests or an unexpected skip
if: always()
run: node ../../.github/scripts/test-guard.mjs go-test.json --suite go --floors ../../.github/test-floors.json
- run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o bin/dot-agentd ./cmd/dot-agentd
- run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false -ldflags="-s -w" -o bin/dot-agentd ./cmd/dot-agentd
- run: file bin/dot-agentd | tee /dev/stderr | grep -q "ELF 64-bit LSB executable, x86-64.*statically linked"
engine:
@@ -217,7 +225,7 @@ jobs:
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
# Builds dot-agentd in golang:1.26, the engine's environment in ubuntu:24.04
# Builds dot-agentd in golang:1.26.8, the engine's environment in ubuntu:24.04
# the way provision.sh builds it, and runs the Dot. It exits non-zero on a
# failed or a skipped check and prints "SMOKE: N passed, F failed, S skipped".
- name: run the smoke on this checkout
+1
View File
@@ -79,6 +79,7 @@ export * from "./host.js";
export * from "./logger.js";
export * from "./ports.js";
export * from "./powershell.js";
export * from "./process-file.js";
export * from "./qemu-args.js";
export * from "./runner.js";
export * from "./seed.js";
+51
View File
@@ -0,0 +1,51 @@
/**
* The pid file of a Dot's VM (architecture section 3.2): what lets a control plane that restarted find the QEMU it
* left running. The VmManager decides from it whether that QEMU still runs and may be killed (`processIsOurs`).
*/
import { readFile, writeFile } from "node:fs/promises";
import { replaceFile } from "@invisible-dots/shared";
/**
* The QEMU process of a Dot, the guest port it forwards, and the host's uptime when it was spawned. A host uptime
* lower than that one means the host restarted since, so the process is gone whatever now runs under that pid.
*/
export interface ProcessFile {
pid: number;
guest_port: number;
host_uptime_s?: number;
}
export function parseProcessFile(text: string): ProcessFile | undefined {
try {
const value = JSON.parse(text) as Partial<ProcessFile>;
const pid = value.pid;
const port = value.guest_port;
if (Number.isSafeInteger(pid) && pid! > 0 && Number.isInteger(port) && port! >= 1 && port! <= 65535) {
const record: ProcessFile = { pid: pid!, guest_port: port! };
if (typeof value.host_uptime_s === "number" && Number.isFinite(value.host_uptime_s)) record.host_uptime_s = value.host_uptime_s;
return record;
}
} catch {
// Falls through: not JSON is as unreadable as JSON of the wrong shape.
}
return undefined;
}
/** The pid file at `path`: undefined when there is none, "unreadable" when it is not one this code wrote. */
export async function readProcessFile(path: string): Promise<ProcessFile | "unreadable" | undefined> {
let text: string;
try {
text = await readFile(path, "utf8");
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined;
throw error;
}
return parseProcessFile(text) ?? "unreadable";
}
/** Written whole or not at all (a temporary file renamed over it), so a reader never sees half a pid file. */
export async function writeProcessFile(path: string, record: ProcessFile): Promise<void> {
const temporary = `${path}.tmp`;
await writeFile(temporary, `${JSON.stringify(record)}\n`, { mode: 0o600 });
await replaceFile(temporary, path);
}
+19 -3
View File
@@ -3,7 +3,7 @@
* one function that builds it, on every host; only the accelerator differs,
* and it arrives here as a value from host.ts.
*/
import { GUEST_PORT, vmName } from "@invisible-dots/shared";
import { CONFIG_BOUNDS, GUEST_PORT, parseSizeMiB, vmName } from "@invisible-dots/shared";
import { VmManagerError } from "./errors.js";
import type { Accelerator } from "./host.js";
@@ -57,6 +57,21 @@ function integerInRange(label: string, value: number, min: number, max: number):
return value;
}
/**
* The cpus and memory every VM of this project may have, Dots and the image builder's VM alike: the range a Dot's
* config allows (CONFIG_BOUNDS of packages/shared, which owns it), in the units QEMU takes.
*/
export const VM_RESOURCE_BOUNDS = {
cpus: { min: CONFIG_BOUNDS.cpu.min, max: CONFIG_BOUNDS.cpu.max },
memoryMiB: { min: parseSizeMiB(CONFIG_BOUNDS.memory.min), max: parseSizeMiB(CONFIG_BOUNDS.memory.max) },
} as const;
/** Refuses cpus or memory outside VM_RESOURCE_BOUNDS, naming the value. */
export function checkVmResources(spec: { cpus: number; memoryMiB: number }): void {
integerInRange("cpus", spec.cpus, VM_RESOURCE_BOUNDS.cpus.min, VM_RESOURCE_BOUNDS.cpus.max);
integerInRange("memoryMiB", spec.memoryMiB, VM_RESOURCE_BOUNDS.memoryMiB.min, VM_RESOURCE_BOUNDS.memoryMiB.max);
}
/**
* The CPU model of every VM of this project (section 3.4): the host's own
* CPU, without the virtualization extensions. A Dot never runs a hypervisor,
@@ -79,6 +94,7 @@ export const CPU_MODEL = "host,-vmx,-svm";
* changes both.
*/
export function machineArgs(spec: { accelerator: Accelerator; cpus: number; memoryMiB: number }): string[] {
checkVmResources(spec);
return [
"-machine",
"q35",
@@ -87,9 +103,9 @@ export function machineArgs(spec: { accelerator: Accelerator; cpus: number; memo
"-cpu",
CPU_MODEL,
"-smp",
String(integerInRange("cpus", spec.cpus, 1, 16)),
String(spec.cpus),
"-m",
String(integerInRange("memoryMiB", spec.memoryMiB, 256, 64 * 1024)),
String(spec.memoryMiB),
];
}
+18 -72
View File
@@ -42,7 +42,8 @@ import { GuestClient, waitForGuestHealth, type GuestClientOptions, type WaitForG
import { accelerator as hostAccelerator, findQemu, type Accelerator, type QemuInstallation } from "./host.js";
import { stderrLogger, type Logger } from "./logger.js";
import { pickFreePort, portListens } from "./ports.js";
import { CPU_MODEL, qemuArgs, qemuPathArg } from "./qemu-args.js";
import { readProcessFile, writeProcessFile, type ProcessFile } from "./process-file.js";
import { checkVmResources, CPU_MODEL, qemuArgs, qemuPathArg } from "./qemu-args.js";
import {
fileSize,
NodeCommandRunner,
@@ -182,12 +183,6 @@ function assertDotId(dotId: string): void {
}
}
function assertIntInRange(label: string, value: number, min: number, max: number): void {
if (!Number.isInteger(value) || value < min || value > max) {
throw new VmManagerError(`${label} must be an integer from ${min} to ${max}, got ${value}`);
}
}
async function exists(path: string): Promise<boolean> {
try {
await access(path);
@@ -218,18 +213,6 @@ const CPU_MODEL_FAILURES = [/CPU model 'host' requires/i, /unable to find CPU mo
/** How much of the end of a log goes into an error: enough for QEMU's last words and the guest's last boot lines. */
const LOG_TAIL_BYTES = 4 * 1024;
/**
* The pid file (section 3.2): the QEMU process of a Dot, the guest port it
* forwards, and the host's uptime when it was spawned. A host uptime lower
* than that one means the host restarted since, so the process is gone
* whatever now runs under that pid.
*/
export interface ProcessFile {
pid: number;
guest_port: number;
host_uptime_s?: number;
}
/** seed-instance.json, or undefined for one this code did not write: the next seed then takes the content's own id. */
function parsePreviousSeed(text: string): PreviousSeed | undefined {
try {
@@ -241,22 +224,6 @@ function parsePreviousSeed(text: string): PreviousSeed | undefined {
return undefined;
}
function parseProcessFile(text: string): ProcessFile | undefined {
try {
const value = JSON.parse(text) as Partial<ProcessFile>;
const pid = value.pid;
const port = value.guest_port;
if (Number.isSafeInteger(pid) && pid! > 0 && Number.isInteger(port) && port! >= 1 && port! <= 65535) {
const record: ProcessFile = { pid: pid!, guest_port: port! };
if (typeof value.host_uptime_s === "number" && Number.isFinite(value.host_uptime_s)) record.host_uptime_s = value.host_uptime_s;
return record;
}
} catch {
// Falls through: not JSON is as unreadable as JSON of the wrong shape.
}
return undefined;
}
/** A QEMU this process spawned: its exit is set the moment Node reports it. */
interface OwnedProcess {
pid: number;
@@ -389,8 +356,7 @@ export class VmManager {
private validateSpec(spec: VmSpec): void {
assertDotId(spec.dotId);
assertIntInRange("cpus", spec.cpus, 1, 16);
assertIntInRange("memoryMiB", spec.memoryMiB, 256, 64 * 1024);
checkVmResources(spec);
if (!Number.isSafeInteger(spec.diskBytes) || spec.diskBytes <= 0) {
throw new VmManagerError(`diskBytes must be a positive integer, got ${spec.diskBytes}`);
}
@@ -400,6 +366,16 @@ export class VmManager {
}
}
/** Both images of the spec exist: a VM is never created or started on a path that leads nowhere. */
private async assertImages(spec: VmSpec): Promise<void> {
for (const [label, path] of [
["golden image", spec.goldenImage],
["runtime ISO", spec.runtimeImage],
] as const) {
if (!(await exists(path))) throw new VmManagerError(`${label} ${path} does not exist: build it with "invisible-dots image build" first`);
}
}
private async qemuImg(args: string[]) {
const { img } = await this.qemu();
return this.runner.run(img, args, { timeoutMs: this.options.qemuImgTimeoutMs ?? 300_000 });
@@ -426,12 +402,7 @@ export class VmManager {
async create(spec: VmSpec): Promise<CreateVmResult> {
this.validateSpec(spec);
return this.withLock(spec.dotId, async () => {
for (const [label, path] of [
["golden image", spec.goldenImage],
["runtime ISO", spec.runtimeImage],
] as const) {
if (!(await exists(path))) throw new VmManagerError(`${label} ${path} does not exist: build it with "invisible-dots image build" first`);
}
await this.assertImages(spec);
const current = await this.probe(spec.dotId);
if (current.state !== "STOPPED") {
throw new VmStateError(spec.dotId, `the VM is ${current.state}; stop it before creating its disk and seed again`);
@@ -452,26 +423,6 @@ export class VmManager {
});
}
/** The pid file: undefined when there is none, "unreadable" when it is not one this code wrote. */
private async readProcessFile(dotId: string): Promise<ProcessFile | "unreadable" | undefined> {
let text: string;
try {
text = await readFile(this.paths.processFilePath(dotId), "utf8");
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined;
throw error;
}
return parseProcessFile(text) ?? "unreadable";
}
/** Written whole or not at all (a temporary file renamed over it), so a reader never sees half a pid file. */
private async writeProcessFile(dotId: string, record: ProcessFile): Promise<void> {
const path = this.paths.processFilePath(dotId);
const temporary = `${path}.tmp`;
await writeFile(temporary, `${JSON.stringify(record)}\n`, { mode: 0o600 });
await replaceFile(temporary, path);
}
/** Forget a QEMU that is gone: its pid file, and its ChildProcess if this process held it. */
private async forget(dotId: string): Promise<void> {
this.owned.delete(dotId);
@@ -492,7 +443,7 @@ export class VmManager {
}
private async qemuRuns(dotId: string): Promise<boolean> {
const record = await this.readProcessFile(dotId);
const record = await readProcessFile(this.paths.processFilePath(dotId));
return record !== undefined && record !== "unreadable" && this.runs(dotId, record);
}
@@ -517,7 +468,7 @@ export class VmManager {
/** Read a VM's state from its pid file and its process (section 3.4). */
private async probe(dotId: string): Promise<Probe> {
const record = await this.readProcessFile(dotId);
const record = await readProcessFile(this.paths.processFilePath(dotId));
if (record === undefined) return STOPPED;
const file = this.paths.processFilePath(dotId);
if (record === "unreadable") {
@@ -590,12 +541,7 @@ export class VmManager {
await this.forget(dotId);
const diskPath = this.paths.diskPath(dotId);
if (!(await exists(diskPath))) throw new VmStateError(dotId, `${diskPath} does not exist; create the VM first`);
for (const [label, path] of [
["golden image", spec.goldenImage],
["runtime ISO", spec.runtimeImage],
] as const) {
if (!(await exists(path))) throw new VmManagerError(`${label} ${path} does not exist: build it with "invisible-dots image build" first`);
}
await this.assertImages(spec);
await this.writeSeed(spec);
const qemu = await this.qemu();
const accel = this.accelerator;
@@ -641,7 +587,7 @@ export class VmManager {
const record: ProcessFile = { pid: child.pid, guest_port: guestPort, host_uptime_s: hostUptimeSeconds() };
try {
// Before anything can fail: from here on a restarted control plane finds this QEMU.
await this.writeProcessFile(dotId, record);
await writeProcessFile(this.paths.processFilePath(dotId), record);
} catch (error) {
await this.killOurs(dotId, record);
await this.forget(dotId);
+179
View File
@@ -0,0 +1,179 @@
/**
* The host's GuestClient against the real dot-agentd, not a stand-in: the daemon built with the image builder's one
* build, run as its README's development mode describes (`--run-as=`: the model's work as the daemon's own user;
* `--listen 127.0.0.1:0`), with a guest config holding the Dot's token, and a stand-in for the engine on the unix
* socket the daemon proxies `/v1/agent` to. What the two sides each test alone with a fake of the other, this tests
* across the seam: the proof, the token, the routes, the bodies, and an event stream passed on as it is written.
*
* Linux only (vitest.config.ts leaves `*.linux.test.ts` out elsewhere): the daemon is a Linux program, its exec route
* runs /bin/bash and the engine's socket is a unix socket. It needs Go, which CI's node job sets up from go.mod.
*/
import { execFile, spawn, type ChildProcess } from "node:child_process";
import { mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises";
import { createServer, get, type Server, type ServerResponse } from "node:http";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { AGENTD_ROUTES, GUEST_UNPROVEN, type OutboundEvent } from "@invisible-dots/shared";
import { agentdBuildCommand } from "@invisible-dots/image-builder";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { GuestClient, GuestRequestError } from "../src/index.js";
const REPO = resolve(fileURLToPath(new URL(".", import.meta.url)), "../../..");
const TOKEN = "integration-token-0123456789";
let dir: string;
let home: string;
let daemon: ChildProcess | undefined;
let agent: Server | undefined;
let port: number;
/** What reached the stand-in engine through the daemon's proxy, as `METHOD /path?query`. */
const agentSeen: string[] = [];
/** The open event streams the stand-in holds, to write the next event into. */
const streams: ServerResponse[] = [];
function event(seq: number): OutboundEvent {
return { seq, id: `evt_${seq}`, type: "automation.next_run", ts: "2026-10-11T10:00:00.000Z", data: { next_run_at_ms: seq } };
}
function sse(res: ServerResponse, seq: number): void {
res.write(`id: ${seq}\ndata: ${JSON.stringify(event(seq))}\n\n`);
}
/** The engine's side of the proxy: /health, and /events/stream, which sends one event and keeps the stream open. */
function startAgent(socket: string): Promise<Server> {
const server = createServer((req, res) => {
const url = new URL(req.url!, "http://agent");
agentSeen.push(`${req.method} ${url.pathname}${url.search}`);
if (url.pathname === "/health") {
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ status: "ok", state: "idle", openrouter_configured: true, browser: { identities: 0, open: 0 }, checks: {} }));
return;
}
if (url.pathname === "/events/stream") {
res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" });
sse(res, Number(url.searchParams.get("after") ?? 0) + 1);
streams.push(res);
return;
}
res.writeHead(404, { "content-type": "application/json" }).end(JSON.stringify({ error: "not_found" }));
});
return new Promise((done, fail) => {
server.once("error", fail);
server.listen(socket, () => done(server));
});
}
/** Starts the daemon and answers the TCP port it listens on, from its log line `listening on="tcp 127.0.0.1:<port>"`. */
function startDaemon(binary: string, args: string[]): Promise<{ child: ChildProcess; port: number }> {
const child = spawn(binary, args, { stdio: ["ignore", "ignore", "pipe"] });
return new Promise((done, fail) => {
let log = "";
child.stderr!.setEncoding("utf8");
child.stderr!.on("data", (chunk: string) => {
log += chunk;
const listening = /listening on="tcp 127\.0\.0\.1:(\d+)"/.exec(log);
if (listening) done({ child, port: Number(listening[1]) });
});
child.once("error", fail);
child.once("exit", (code, signal) => fail(new Error(`dot-agentd exited (${code ?? signal}) before it listened:\n${log}`)));
});
}
beforeAll(async () => {
dir = await mkdtemp(join(tmpdir(), "idots-agentd-"));
home = join(dir, "home");
await mkdir(home);
const binary = join(dir, "dot-agentd");
const build = agentdBuildCommand(REPO, binary);
await promisify(execFile)(build.command, build.args, { env: { ...process.env, ...build.env } });
const config = join(dir, "config.json");
await writeFile(config, JSON.stringify({ dotId: "dot_integration", token: TOKEN }), { mode: 0o600 });
const agentSocket = join(dir, "agent.sock");
agent = await startAgent(agentSocket);
const started = await startDaemon(binary, [
"--config", config,
"--home", home,
"--run-dir", join(dir, "run"),
"--agent-socket", agentSocket,
"--listen", "127.0.0.1:0",
"--run-as=",
]);
daemon = started.child;
port = started.port;
// The build: a cold Go cache compiles the standard library for linux/amd64 first, and a host whose Go is older
// than go.mod's toolchain downloads that one.
}, 600_000);
afterAll(async () => {
for (const res of streams) res.destroy();
if (daemon && daemon.exitCode === null && daemon.signalCode === null) {
const exited = new Promise((done) => daemon!.once("exit", done));
daemon.kill("SIGTERM");
await exited;
}
await new Promise<void>((done) => (agent ? agent.close(() => done()) : done()));
if (dir) await rm(dir, { recursive: true, force: true });
});
describe("GuestClient against dot-agentd", () => {
it("is given the proof of the token before it sends the token", async () => {
await expect(new GuestClient(port, TOKEN).prove()).resolves.toBeUndefined();
const impostor = new GuestClient(port, "another-dot-token");
await expect(impostor.health()).rejects.toMatchObject({ code: GUEST_UNPROVEN });
});
it("is refused without the token: 401 with the Bearer challenge", async () => {
const answer = await new Promise<{ status: number; challenge: string | undefined; body: string }>((done, fail) => {
get({ host: "127.0.0.1", port, path: AGENTD_ROUTES.health }, (res) => {
let body = "";
res.setEncoding("utf8");
res.on("data", (chunk: string) => (body += chunk));
res.on("end", () => done({ status: res.statusCode ?? 0, challenge: res.headers["www-authenticate"], body }));
}).once("error", fail);
});
expect(answer.status).toBe(401);
expect(answer.challenge).toBe('Bearer realm="dot-agentd"');
expect(JSON.parse(answer.body)).toMatchObject({ error: "unauthorized" });
});
it("reads health with the engine's own answer passed through", async () => {
const health = await new GuestClient(port, TOKEN).health();
expect(health).toMatchObject({ agentd: "ok", agent: { status: "ok", state: "idle", openrouter_configured: true } });
expect(agentSeen).toContain("GET /health");
});
it("runs a command in home and reads its exit code and both streams", async () => {
const client = new GuestClient(port, TOKEN);
expect(await client.exec({ command: "printf out; printf err >&2; exit 3" })).toMatchObject({ exit_code: 3, stdout: "out", stderr: "err", timed_out: false });
expect((await client.exec({ command: "pwd" })).stdout.trim()).toBe(await realpath(home));
});
it("writes, reads and lists a file under home, and refuses one outside it", async () => {
const client = new GuestClient(port, TOKEN);
await client.writeFile("notes/hello.txt", "hello from the host");
expect((await client.readFile("~/notes/hello.txt")).toString("utf8")).toBe("hello from the host");
const listing = await client.listFiles("notes");
expect(listing.entries).toEqual([expect.objectContaining({ name: "hello.txt", type: "file", size: 19 })]);
const outside = await client.readFile("/etc/hostname").catch((error: unknown) => error);
expect(outside).toBeInstanceOf(GuestRequestError);
expect(outside).toMatchObject({ status: 403, code: "outside_home" });
});
it("proxies the engine's routes, and its event stream as it is written", async () => {
const client = new GuestClient(port, TOKEN);
expect(await client.agentHealth()).toMatchObject({ status: "ok", state: "idle" });
const controller = new AbortController();
const got: number[] = [];
for await (const evt of client.events({ after: 4, signal: controller.signal })) {
got.push(evt.seq);
// The first event arrived while the engine still holds the stream open: the proxy did not wait for its end.
// The next one is written only now.
if (got.length === 1) sse(streams.at(-1)!, 6);
else controller.abort();
}
expect(got).toEqual([5, 6]);
expect(agentSeen).toContain("GET /events/stream?after=4");
});
});
+15 -9
View File
@@ -282,21 +282,27 @@ QEMU on the same disk would corrupt it.
`invisible-playwright-mcp`, transitive packages included, by
`guest/image-builder/builder/mcp-requirements.lock`, every package at an
exact version with the SHA-256 of its files. The builder installs it with
`uv pip install --require-hashes`, so nothing is resolved from the index at
build time; the lock is the one place the two top-level versions are
`uv pip install --require-hashes --only-binary :all:`, so nothing is
resolved from the index at build time and no package's build script runs;
the lock is the one place the two top-level versions are
written, and it is part of the inputs digest, so a changed transitive
package is a new image. apt packages are not pinned by version; apt checks
their signatures.
package is a new image. The apt packages come from the Ubuntu archive as it
was at the moment `apt_snapshot` of `pins.json` names (snapshot.ubuntu.com,
`apt-get --snapshot`), so they and their dependencies are the same on any day
of the build; apt checks their signatures.
- The **runtime disk** (`runtime-<version>.iso`, attached read-only to every VM
and mounted at `/opt/invisible-dots`) carries our code: the engine's source,
the `dot-agentd` binary and the systemd units. A new
version of our code is a new ISO and a VM restart, not a new golden image and
not a rebuilt overlay.
Neither image is ever published by this project: the host builds both from
public sources (`invisible-dots image build`, code in `guest/image-builder/`),
with the same QEMU it runs Dots with and the same machine: the builder VM's
command line is built from the vm-manager's `machineArgs()`, drive and device
The host builds the runtime disk from this checkout, and the golden image from
public sources (`invisible-dots image build`, code in `guest/image-builder/`)
unless CI has published the golden image of exactly the same inputs: the
release `golden-<inputs digest>` of `.github/workflows/golden-image.yml`,
which `image build` downloads instead (`guest/image-builder/src/prebuilt.ts`;
section 11.3). Either way the golden image is provisioned with the same QEMU a
host runs Dots with and the same machine: the builder VM's command line is built from the vm-manager's `machineArgs()`, drive and device
functions, the ones `qemuArgs()` uses, plus only what a builder needs, so a
golden image is provisioned on the machine type, accelerator and CPU model a
Dot boots it on. The seed and runtime ISO labels are each defined once, in
@@ -2947,7 +2953,7 @@ system and the apt packages the golden image installs on it (`apt_packages` of
`guest/image-builder/pins.json`: Xvfb, XFCE and their libraries, over the Linux kernel,
bash and the rest of Ubuntu's base), most of it GPL or LGPL software under its own
licenses and source offers, downloaded from Ubuntu's archive on the machine that builds
the image and published by no one here. Whoever copies a golden image to another
the image, which is CI when the image is the published release of the paragraph above. Whoever copies a golden image to another
machine takes those licenses with it (the paragraph above).
### 11.4 setup --all
+5 -2
View File
@@ -60,11 +60,14 @@ an environment only by its owner.
## Build
```sh
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o bin/dot-agentd ./cmd/dot-agentd
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false -ldflags="-s -w" -o bin/dot-agentd ./cmd/dot-agentd
```
The result is a static linux/amd64 binary with no dependencies outside the
Go standard library. The unix-only pieces (socket umask, process groups,
Go standard library, and the same bytes from the same source with the Go that
`go.mod`'s `toolchain` names: no checkout path, commit or "modified" mark is
stamped into it. This is the one build: `agentdBuildCommand()` of the image
builder, CI and the smoke run it with these flags. The unix-only pieces (socket umask, process groups,
`statfs`) sit behind build tags, so `go vet ./...` also works on other systems.
## Test
+4
View File
@@ -1,3 +1,7 @@
module github.com/feder-cr/dots/guest/dot-agentd
go 1.25.0
// The Go that builds dot-agentd: CI's (setup-go reads this line), the smoke's (the golang image of this version),
// and a host's whose own Go is older (go downloads this one). The same source and this Go give the same bytes.
toolchain go1.26.8
+39 -7
View File
@@ -13,6 +13,7 @@ import (
"path/filepath"
"strconv"
"strings"
"sync"
"syscall"
"testing"
"time"
@@ -93,24 +94,55 @@ func TestProcFollowsAResizeAndForwardsSignals(t *testing.T) {
resize := make(chan ProcTTY, 1)
signals := make(chan syscall.Signal, 1)
resize <- ProcTTY{Cols: 50, Rows: 10}
// The program says "resized" once its terminal has the new size, and the signal is sent only then: it must reach
// a program that is running, and it ends the program, which would otherwise sleep for 30 s.
out := &signalWriter{want: "resized", seen: make(chan struct{})}
go func() {
time.Sleep(500 * time.Millisecond)
<-out.seen
signals <- syscall.SIGTERM
}()
code, out, _ := relay(t, sock, RelayOptions{
Argv: []string{"/bin/sh", "-c", `sleep 0.3; stty size; sleep 30`},
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
code, err := RunRelay(ctx, RelayOptions{
Socket: sock,
Argv: []string{"/bin/sh", "-c", `while [ "$(stty size)" != "10 50" ]; do sleep 0.05; done; echo resized; sleep 30`},
TTY: &ProcTTY{Cols: 80, Rows: 24},
Resize: resize,
Signals: signals,
}, "")
if !strings.Contains(out, "10 50") {
t.Errorf("the resize did not reach the terminal: %q", out)
}, nil, out, io.Discard)
if err != nil {
t.Fatalf("the relay failed (the resize never reached the terminal?): %v; output %q", err, out.String())
}
if code != 128+int(syscall.SIGTERM) {
t.Errorf("code %d, want %d", code, 128+int(syscall.SIGTERM))
t.Errorf("code %d, want %d; output %q", code, 128+int(syscall.SIGTERM), out.String())
}
}
// signalWriter keeps what is written to it and closes seen once it holds want.
type signalWriter struct {
mu sync.Mutex
buf bytes.Buffer
want string
seen chan struct{}
once sync.Once
}
func (w *signalWriter) Write(p []byte) (int, error) {
w.mu.Lock()
defer w.mu.Unlock()
n, err := w.buf.Write(p)
if strings.Contains(w.buf.String(), w.want) {
w.once.Do(func() { close(w.seen) })
}
return n, err
}
func (w *signalWriter) String() string {
w.mu.Lock()
defer w.mu.Unlock()
return w.buf.String()
}
func TestProcDiesWithItsCaller(t *testing.T) {
sock, _ := localSocket(t)
procDiesWithItsCaller(t, sock, filepath.Join(t.TempDir(), "pid"))
+13 -8
View File
@@ -2,8 +2,10 @@
`@invisible-dots/image-builder` builds the two guest images of architecture
section 3.3 on the host, with the same code on Linux and Windows. It is what
`invisible-dots image build` runs. Neither image is ever published: every host
builds its own from public sources.
`invisible-dots image build` runs. Every host builds its runtime ISO from its
checkout; the golden image it downloads when CI published the one of exactly
its inputs (the release `golden-<inputs digest>`, `src/prebuilt.ts`), and
builds it from public sources otherwise.
Both images land in `INVISIBLE_DOTS_HOME/images`, each with a manifest next to
it (`golden-<v>.json`, `runtime-<v>.json`) that records what went in and the
@@ -33,13 +35,14 @@ image's SHA-256; `invisible-dots doctor` checks an image against it with
same machine, CPU model and devices as a Dot (its command line is built
from the vm-manager's `machineArgs()`). `provision.sh` installs the
apt packages (Xvfb, a minimal XFCE, the browser's libraries, ImageMagick
for dot-agentd's screenshots), uv, then removes the cloud image's services a
Dot never uses (snapd, unattended-upgrades, apport, the LXD stubs, Ubuntu Pro,
for dot-agentd's screenshots) from the Ubuntu archive as it was at
`pins.json`'s `apt_snapshot` (`apt-get --snapshot`), uv, then removes the
cloud image's services a Dot never uses (snapd, unattended-upgrades, apport, the LXD stubs, Ubuntu Pro,
the release upgrader, the SSH server, pollinate, which would tell Canonical of
every new Dot), then builds the Dot's browser with
`build-browser-env.sh <lock> ~dot/.local/share/invisible-dots/mcp`, as user
`dot`: a virtual environment filled with
`uv pip install --require-hashes -r mcp-requirements.lock`, its
`uv pip install --require-hashes --only-binary :all: -r mcp-requirements.lock`, its
`invisible-playwright-mcp` linked into `~/.local/bin`, and
`invisible-playwright fetch` run from that environment, so the cached browser
engine is the one the MCP server expects (the library keeps its own GeoIP
@@ -101,11 +104,13 @@ named in the error.
| on the ISO | from |
|---|---|
| `install.sh` | `runtime/install.sh` |
| `vmproxy.py` | `runtime/vmproxy.py`, which `install.sh` runs to set up the Dot's VM proxy |
| `VERSION` | the version |
| `bin/dot-agentd` | `guest/dot-agentd/bin/` (`CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/dot-agentd ./cmd/dot-agentd`), refused unless it is a linux/amd64 ELF |
| `bin/dot-agentd` | `guest/dot-agentd/bin/` (`CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false -ldflags="-s -w" -o bin/dot-agentd ./cmd/dot-agentd`), refused unless it is a linux/amd64 ELF |
| `bin/dot-desktop` | `runtime/dot-desktop.sh` |
| `engine/nanobot/...` | `invisible_engine_dots/nanobot/`: every `*.py` and the `*.md` under `templates/` |
| `engine/requirements.lock` | `builder/engine-requirements.lock`, byte for byte: the engine refuses to start when it differs from the venv's copy |
| `bin/dot-install` | `runtime/dot-install.sh`, the one command `dot` may run as root |
| `engine/nanobot/...` | `invisible_engine_dots/nanobot/`: every `*.py`, the `*.json` data and the `*.md` under `templates/` |
| `engine/skills/<name>/SKILL.md` | `invisible_engine_dots/skills/`, the built-in skills |
| `engine/LICENSE`, `engine/UPSTREAM.md` | `invisible_engine_dots/` |
| `units/*.service` | `units/` |
@@ -33,7 +33,9 @@ install -m 0644 "$lock" "$dot_lock"
as_dot mkdir -p "$bin_dir" "$(dirname "$env_dir")"
as_dot uv venv --quiet --python /usr/bin/python3 "$env_dir"
as_dot uv pip install --python "$env_dir/bin/python" --require-hashes -r "$dot_lock"
# Wheels only, as for the engine: no build script of any package runs. --no-cache: the image does not need
# uv's download cache in dot's home.
as_dot uv pip install --no-cache --python "$env_dir/bin/python" --require-hashes --only-binary :all: -r "$dot_lock"
[ -x "$env_dir/bin/invisible-playwright" ] || { echo "build-browser-env: no invisible-playwright in $env_dir/bin" >&2; exit 1; }
[ -x "$env_dir/bin/invisible-playwright-mcp" ] || { echo "build-browser-env: no invisible-playwright-mcp in $env_dir/bin" >&2; exit 1; }
as_dot ln -sfn "$env_dir/bin/invisible-playwright-mcp" "$bin_dir/invisible-playwright-mcp"
+8 -3
View File
@@ -30,13 +30,18 @@ trap 'on_error "$LINENO" "$BASH_COMMAND"' ERR
export DEBIAN_FRONTEND=noninteractive
export NEEDRESTART_MODE=a
step "installing packages: $APT_PACKAGES"
step "installing packages from the Ubuntu archive as of $APT_SNAPSHOT: $APT_PACKAGES"
# The cloud image may still be running its own apt jobs at first boot: wait for the lock.
apt_wait=(-o DPkg::Lock::Timeout=600)
apt-get "${apt_wait[@]}" update
# The packages and every dependency come from the archive as it was at the pinned moment (snapshot.ubuntu.com,
# which apt reaches for the archive this image is configured with), not as it is on the day of the build: the same
# pins make the same image. --error-on=any: a list that cannot be fetched fails the build instead of leaving apt
# with whatever lists the cloud image had.
apt_snapshot=(--snapshot "$APT_SNAPSHOT")
apt-get "${apt_wait[@]}" "${apt_snapshot[@]}" --error-on=any update
# shellcheck disable=SC2086
# unsafe-io: no fsync per package, the builder VM is thrown away if the build fails anyway.
apt-get "${apt_wait[@]}" -o Dpkg::Options::=--force-unsafe-io install -y --no-install-recommends $APT_PACKAGES
apt-get "${apt_wait[@]}" "${apt_snapshot[@]}" -o Dpkg::Options::=--force-unsafe-io install -y --no-install-recommends $APT_PACKAGES
step "removing what a Dot never uses"
# Services of the cloud image that would run in every Dot for nothing: snaps, automatic upgrades (apt in the
+1
View File
@@ -11,6 +11,7 @@
"url": "https://github.com/heiher/hev-socks5-tunnel/releases/download/2.18.0/hev-socks5-tunnel-linux-x86_64",
"sha256": "1903b23869cbd12b7b57045ca161c0f90314d9c4ef2835e7a9e4a3179bf7d925"
},
"apt_snapshot": "20261010T000000Z",
"apt_packages": [
"xvfb",
"xfce4-session",
+7 -2
View File
@@ -14,8 +14,13 @@ usage="usage: sudo dot-install <package>..."
[ "$(id -u)" -eq 0 ] || { echo "dot-install: run it with sudo ($usage)" >&2; exit 2; }
[ $# -ge 1 ] || { echo "$usage" >&2; exit 2; }
for name in "$@"; do
# A Debian package name: lower case letters, digits, "+", "." and "-", starting with a letter or a digit.
[[ "$name" =~ ^[a-z0-9][a-z0-9+.-]+$ ]] || { echo "dot-install: \"$name\" is not a package name ($usage)" >&2; exit 2; }
# A Debian package name (policy 5.6.1): lower case letters, digits, "+", "." and "-", at least two characters,
# starting with a letter or a digit; that leaves out every option, version (=), release (/) and architecture (:).
# And not ending in "-": `apt-get install` reads a trailing "-" as "remove this package" when no package has the
# whole word as its name (apt-get(8)), so `foo-` would remove foo as root; no package of the archive ends in "-".
# A trailing "+" stays: it is apt's "install" mark, so `foo+` installs foo, and a name that ends in it (g++,
# ncbi-blast+) is matched whole first, so nothing but an install can come of it.
[[ "$name" =~ ^[a-z0-9][a-z0-9+.-]*[a-z0-9+.]$ ]] || { echo "dot-install: \"$name\" is not a package name ($usage)" >&2; exit 2; }
done
export DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a
+5 -37
View File
@@ -57,43 +57,11 @@ install -d -o dotengine -g dotengine -m 0700 /home/dotengine/state
proxy_dir=/run/dot-vmproxy
if [ -n "$(python3 -c 'import json;print(json.load(open("/etc/invisible-dots/config.json")).get("proxy") or "")' 2>/dev/null)" ]; then
install -d -o root -g root -m 0700 "$proxy_dir"
# Writes hev.yml (0600) and env (the proxy's address and the uplink, for the routes); answers 1 for a value that
# is not socks5://[user:password@]host:port.
if ! python3 - "$proxy_dir" <<'PY'
import json, os, socket, subprocess, sys
from urllib.parse import unquote, urlsplit
out = sys.argv[1]
url = urlsplit(json.load(open("/etc/invisible-dots/config.json"))["proxy"].strip())
if url.scheme != "socks5" or not url.hostname or not url.port:
sys.exit(1)
address = socket.getaddrinfo(url.hostname, url.port, socket.AF_INET, socket.SOCK_STREAM)[0][4][0]
route = subprocess.run(["ip", "-4", "route", "show", "default"], capture_output=True, text=True).stdout.split()
gateway, device = route[route.index("via") + 1], route[route.index("dev") + 1]
q = lambda value: json.dumps(value)
lines = [
"tunnel:", " name: tun0", " mtu: 8500", " ipv4: 198.18.0.1", " post-up-script: /run/dot-vmproxy/up.sh",
"socks5:", f" address: {q(address)}", f" port: {url.port}", " udp: 'udp'",
]
if url.username:
lines += [f" username: {q(unquote(url.username))}", f" password: {q(unquote(url.password or ''))}"]
lines += ["mapdns:", " address: 198.18.0.2", " port: 53", " network: 100.64.0.0", " netmask: 255.192.0.0", " cache-size: 10000"]
def write(name, text, mode):
fd = os.open(os.path.join(out, name), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode)
with os.fdopen(fd, "wb") as f:
f.write(text.encode())
write("hev.yml", "\n".join(lines) + "\n", 0o600)
write("up.sh", "#!/bin/sh\n"
f"ip route replace {address}/32 via {gateway} dev {device}\n"
"ip route replace default dev tun0 metric 1\n"
"resolvectl dns tun0 198.18.0.2 && resolvectl domain tun0 '~.' && resolvectl default-route tun0 yes\n"
f"resolvectl default-route {device} no || true\n", 0o700)
write("allow.nft", "table inet dot_vmproxy {\n chain out {\n type filter hook output priority 0; policy drop;\n"
" oifname { \"lo\", \"tun0\" } accept\n ct state established,related accept\n"
f" ip daddr {address} tcp dport {url.port} accept\n ip daddr {address} meta l4proto udp accept\n"
" ip daddr 10.0.2.0/24 accept\n udp dport 67 accept\n }\n}\n", 0o600)
PY
then
log "error: the Dot's proxy is not socks5://[user:password@]host:port; nothing goes out until it is fixed"
# vmproxy.py (on this disk) writes the tunnel's settings, its routes and the firewall; when it cannot (the value is
# not socks5://[user:password@]host:port, its host does not resolve, the VM has no default route), it says why
# and writes nothing, and the firewall below lets nothing out.
if ! reason=$(python3 "$runtime/vmproxy.py" /etc/invisible-dots/config.json "$proxy_dir" 2>&1); then
log "error: the Dot's proxy cannot be set up: $reason; nothing goes out until it is fixed"
printf 'table inet dot_vmproxy {\n chain out {\n type filter hook output priority 0; policy drop;\n oifname "lo" accept\n ct state established,related accept\n ip daddr 10.0.2.0/24 accept\n }\n}\n' > "$proxy_dir/allow.nft"
fi
sysctl -q -w net.ipv6.conf.all.disable_ipv6=1 net.ipv6.conf.default.disable_ipv6=1
+110
View File
@@ -0,0 +1,110 @@
"""The Dot's VM proxy, as install.sh sets it up (it runs this as root on every boot when config.json has a proxy).
python3 vmproxy.py <config.json> <output directory>
Writes, into the output directory, hev.yml (0600: the tunnel's settings, password included), up.sh (0700: the routes
and the DNS the tunnel sets once tun0 is up) and allow.nft (0600: the firewall that lets nothing leave but through the
tunnel). Prints nothing on success. Otherwise it writes nothing, prints the reason on stderr and exits with:
3 the proxy is not socks5://[user:password@]host:port
4 the proxy's host does not resolve to an IPv4 address
5 the VM has no default IPv4 route through a gateway (the tunnel needs it to reach the proxy)
Whatever the reason, install.sh then puts a firewall in place that lets nothing out: the Dot never goes out directly
when it has a proxy.
"""
import json
import os
import socket
import subprocess
import sys
from urllib.parse import unquote, urlsplit
INVALID, UNRESOLVED, NO_ROUTE = 3, 4, 5
class Refused(Exception):
def __init__(self, code, reason):
super().__init__(reason)
self.code = code
def proxy_url(config_path):
with open(config_path) as f:
raw = json.load(f).get("proxy") or ""
try:
url = urlsplit(raw.strip())
port = url.port
except ValueError as error:
raise Refused(INVALID, f"the proxy is not socks5://[user:password@]host:port ({error})")
if url.scheme != "socks5" or not url.hostname or not port:
raise Refused(INVALID, "the proxy is not socks5://[user:password@]host:port")
return url
def resolve(host, port):
try:
return socket.getaddrinfo(host, port, socket.AF_INET, socket.SOCK_STREAM)[0][4][0]
except OSError as error:
raise Refused(UNRESOLVED, f"the proxy's host {host} does not resolve to an IPv4 address ({error})")
def default_route():
"""The gateway and the device of the default IPv4 route."""
shown = subprocess.run(["ip", "-4", "route", "show", "default"], capture_output=True, text=True, check=True).stdout
words = shown.split()
if "via" not in words or "dev" not in words:
seen = shown.strip() or "none"
raise Refused(NO_ROUTE, f"the VM has no default IPv4 route through a gateway (ip route shows: {seen})")
return words[words.index("via") + 1], words[words.index("dev") + 1]
def files(url, address, gateway, device):
q = json.dumps
lines = [
"tunnel:", " name: tun0", " mtu: 8500", " ipv4: 198.18.0.1", " post-up-script: /run/dot-vmproxy/up.sh",
"socks5:", f" address: {q(address)}", f" port: {url.port}", " udp: 'udp'",
]
if url.username:
lines += [f" username: {q(unquote(url.username))}", f" password: {q(unquote(url.password or ''))}"]
lines += ["mapdns:", " address: 198.18.0.2", " port: 53", " network: 100.64.0.0", " netmask: 255.192.0.0", " cache-size: 10000"]
up = (
"#!/bin/sh\n"
f"ip route replace {address}/32 via {gateway} dev {device}\n"
"ip route replace default dev tun0 metric 1\n"
"resolvectl dns tun0 198.18.0.2 && resolvectl domain tun0 '~.' && resolvectl default-route tun0 yes\n"
f"resolvectl default-route {device} no || true\n"
)
allow = (
"table inet dot_vmproxy {\n chain out {\n type filter hook output priority 0; policy drop;\n"
" oifname { \"lo\", \"tun0\" } accept\n ct state established,related accept\n"
f" ip daddr {address} tcp dport {url.port} accept\n ip daddr {address} meta l4proto udp accept\n"
" ip daddr 10.0.2.0/24 accept\n udp dport 67 accept\n }\n}\n"
)
return [("hev.yml", "\n".join(lines) + "\n", 0o600), ("up.sh", up, 0o700), ("allow.nft", allow, 0o600)]
def write(out, name, text, mode):
fd = os.open(os.path.join(out, name), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode)
with os.fdopen(fd, "wb") as f:
f.write(text.encode())
def main(config_path, out):
try:
url = proxy_url(config_path)
address = resolve(url.hostname, url.port)
gateway, device = default_route()
except Refused as refused:
print(refused, file=sys.stderr)
return refused.code
for name, text, mode in files(url, address, gateway, device):
write(out, name, text, mode)
return 0
if __name__ == "__main__":
if len(sys.argv) != 3:
print("usage: vmproxy.py <config.json> <output directory>", file=sys.stderr)
sys.exit(2)
sys.exit(main(sys.argv[1], sys.argv[2]))
+5 -3
View File
@@ -1,7 +1,7 @@
/**
* The files of this package that end up inside a guest: the builder's
* cloud-init and provisioner, and the runtime ISO's install hook, desktop
* script and systemd units. They are kept as plain files so they can be read,
* cloud-init and provisioner, and the runtime ISO's install hook, VM proxy
* setup, desktop script and systemd units. They are kept as plain files so they can be read,
* diffed and checked with shell tools, and read from disk at build time.
*/
import { readFile } from "node:fs/promises";
@@ -30,6 +30,8 @@ export const BUILDER_BROWSER_BUILD = "builder/build-browser-env.sh";
export const RUNTIME_INSTALL = "runtime/install.sh";
export const RUNTIME_DESKTOP = "runtime/dot-desktop.sh";
export const RUNTIME_DOT_INSTALL = "runtime/dot-install.sh";
/** Writes the VM proxy's tunnel settings, routes and firewall; install.sh runs it when the Dot has a proxy. */
export const RUNTIME_VMPROXY = "runtime/vmproxy.py";
/** The guest units, in the order install.sh enables them. */
export const GUEST_UNITS = ["dot-desktop.service", "dot-agentd.service", "invisible-dots-agent.service"] as const;
@@ -38,7 +40,7 @@ export function unitAsset(name: (typeof GUEST_UNITS)[number]): string {
}
/** Every guest file, for checks that apply to all of them. */
export const GUEST_ASSETS: readonly string[] = [BUILDER_USER_DATA, BUILDER_PROVISION, BUILDER_PYTHON_LOCK, BUILDER_ENGINE_LOCK, BUILDER_ENGINE_BUILD, BUILDER_BROWSER_BUILD, RUNTIME_INSTALL, RUNTIME_DESKTOP, RUNTIME_DOT_INSTALL, ...GUEST_UNITS.map(unitAsset)];
export const GUEST_ASSETS: readonly string[] = [BUILDER_USER_DATA, BUILDER_PROVISION, BUILDER_PYTHON_LOCK, BUILDER_ENGINE_LOCK, BUILDER_ENGINE_BUILD, BUILDER_BROWSER_BUILD, RUNTIME_INSTALL, RUNTIME_DESKTOP, RUNTIME_DOT_INSTALL, RUNTIME_VMPROXY, ...GUEST_UNITS.map(unitAsset)];
/**
* Reads a guest file and refuses one a Windows checkout turned into CRLF:
+16 -35
View File
@@ -8,11 +8,11 @@ import { createHash } from "node:crypto";
import { chmod, copyFile, mkdir, readFile, rm, stat } from "node:fs/promises";
import { basename, join } from "node:path";
import { writeIso } from "@invisible-dots/iso";
import { hostPaths, replaceFile, type HostPaths } from "@invisible-dots/shared";
import { hostPaths, type HostPaths } from "@invisible-dots/shared";
import { BUILDER_BROWSER_BUILD, BUILDER_ENGINE_BUILD, BUILDER_ENGINE_LOCK, BUILDER_PROVISION, BUILDER_PYTHON_LOCK, BUILDER_USER_DATA, defaultAssetRoot, readGuestAsset } from "./assets.js";
import { fetchVerified, sha256File, type Fetch, type FetchVerifiedOptions } from "./download.js";
import { acquireLock, type Lock } from "./lock.js";
import { manifestPathFor, writeManifest, type GoldenManifest } from "./manifest.js";
import { manifestPathFor, type GoldenManifest } from "./manifest.js";
import { BASE_IMAGE, downloadFileName, GUEST_PINS, type BaseImagePin, type GuestPins, type PinnedDownload } from "./pins.js";
import { waitForExit, type ProcessRunner } from "./process.js";
import { PREBUILT_RELEASES, pullPrebuiltGolden } from "./prebuilt.js";
@@ -21,7 +21,8 @@ import { builderQemuArgs, type Accelerator, type QemuPrograms } from "./qemu.js"
import { SEED_VOLUME_ID } from "@invisible-dots/vm-manager";
import { builderSeedEntries } from "./seed.js";
import { buildVerdict, followSerialLog, installedComponents } from "./serial.js";
import { checkVersion, findImageByDigest, inputsDigest, versionFor } from "./versions.js";
import { inputsDigest } from "./versions.js";
import { publishImage, resolveImageVersion } from "./publish.js";
export const GOLDEN_DEFAULTS = {
/** Each Dot's overlay is larger; cloud-init grows the filesystem when a Dot boots. */
@@ -112,28 +113,16 @@ export async function buildGoldenImage(options: GoldenBuildOptions): Promise<Gol
// source is not: it travels on the runtime disk (section 3.3).
const digest = inputsDigest([JSON.stringify(base), JSON.stringify(pins), userData, provision, pythonLock, diskSize, engineLock, engineBuild, browserBuild]);
await mkdir(paths.imagesDir, { recursive: true });
let version: string;
if (options.version === undefined) {
const existing = await findImageByDigest(paths.imagesDir, "golden", digest);
if (existing) {
log(`${basename(existing.path)} already holds these inputs; nothing to do`);
return { version: existing.version, image: existing.path, manifest: manifestPathFor(existing.path), created: false };
}
version = versionFor(now(), digest);
} else {
version = checkVersion(options.version);
}
const image = paths.goldenImagePath(version);
const manifest = manifestPathFor(image);
if (await exists(image)) {
if (await exists(manifest)) {
log(`${basename(image)} already exists; nothing to do`);
return { version, image, manifest, created: false };
}
throw new GoldenBuildError(`${image} exists without its manifest ${manifest}: it is not one this builder finished; remove it by hand`);
}
const { version, image, manifest, existing } = await resolveImageVersion({
kind: "golden",
imagesDir: paths.imagesDir,
digest,
version: options.version,
now,
imagePath: (v) => paths.goldenImagePath(v),
log,
});
if (existing) return { version, image, manifest, created: false };
const lock = await acquireLock(join(paths.imagesDir, ".golden-build.lock"), "golden image build");
try {
@@ -264,18 +253,14 @@ async function buildLocked(options: GoldenBuildOptions & Resolved, target: Targe
"invisible-playwright-mcp": target.python.mcpVersion,
"invisible-playwright": target.python.playwrightVersion,
"mcp-requirements.lock": createHash("sha256").update(target.pythonLock).digest("hex"),
apt_snapshot: pins.apt_snapshot,
apt_packages: [...pins.apt_packages],
},
engine: { lock_sha256: createHash("sha256").update(target.engineLock).digest("hex") },
installed: installedComponents(consoleText),
builder: { accelerator: options.accelerator },
};
// The manifest goes first: the control plane picks images by file name,
// so the image must not appear before the record that describes it.
await writeManifest(target.manifest, manifest);
await replaceFile(partial, target.image);
// Immutable once a VM uses it (section 3.3); QEMU opens a backing file read-only anyway.
await chmod(target.image, 0o444);
await publishImage(partial, target.image, target.manifest, manifest);
await rm(workDir, { recursive: true, force: true });
log(`done: ${target.image} (sha256 ${sha256})`);
return { version: target.version, image: target.image, manifest: target.manifest, created: true };
@@ -349,7 +334,3 @@ function duration(ms: number): string {
function lastLines(text: string, count: number): string {
return text.split(/\r?\n/).filter((line) => line.trim() !== "").slice(-count).join("\n");
}
async function exists(path: string): Promise<boolean> {
return (await stat(path).catch(() => undefined)) !== undefined;
}
+1
View File
@@ -26,6 +26,7 @@ export { normalizePackageName, parseHashedLock, parsePythonLock, type PythonLock
export type { ProcessRunner, RunningProcess } from "./process.js";
export { builderQemuArgs, type Accelerator, type QemuPrograms } from "./qemu.js";
export {
AGENTD_BUILD_SHELL,
agentdBuildCommand,
assertLinuxAmd64Elf,
type BuildCommand,
+2
View File
@@ -38,6 +38,8 @@ export interface GoldenManifest {
"invisible-playwright": string;
/** SHA-256 of builder/mcp-requirements.lock: the whole Python environment, transitive packages included. */
"mcp-requirements.lock": string;
/** The moment of the Ubuntu archive apt_packages came from (pins.json). */
apt_snapshot: string;
apt_packages: string[];
};
/** The engine's Python environment (builder/engine-requirements.lock): the lock the runtime disk's copy must equal. */
+10
View File
@@ -38,6 +38,11 @@ export interface GuestPins {
uv: PinnedDownload;
/** hev-socks5-tunnel, the static binary the guest routes the whole VM through when the Dot has a VM proxy. */
tunnel: PinnedDownload;
/**
* The moment of the Ubuntu archive the packages come from (snapshot.ubuntu.com, `YYYYMMDDTHHMMSSZ`): apt resolves
* apt_packages and their dependencies as the archive was then, so the same pins give the same packages on any day.
*/
apt_snapshot: string;
apt_packages: string[];
}
@@ -49,6 +54,8 @@ const SHA256 = /^[0-9a-f]{64}$/;
*/
const PLAIN_WORD = /^[A-Za-z0-9][A-Za-z0-9.+_-]*$/;
const FILE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
/** A time of snapshot.ubuntu.com, as `apt-get --snapshot` takes it. */
const APT_SNAPSHOT = /^\d{8}T\d{6}Z$/;
function field(object: Record<string, unknown>, key: string, where: string): string {
const value = object[key];
@@ -119,9 +126,12 @@ export function parseGuestPins(value: unknown): GuestPins {
if ("python_packages" in o) throw new Error(`${where}: python_packages moved to builder/mcp-requirements.lock, which pins them with their hashes`);
const apt = o.apt_packages;
if (!Array.isArray(apt) || apt.length === 0) throw new Error(`${where}: apt_packages must be a non-empty array`);
const snapshot = field(o, "apt_snapshot", where);
if (!APT_SNAPSHOT.test(snapshot)) throw new Error(`${where}: apt_snapshot "${snapshot}" is not a snapshot time like 20261010T000000Z`);
return {
uv: parseDownload(o.uv, `${where} uv`),
tunnel: parseDownload(o.tunnel, `${where} tunnel`),
apt_snapshot: snapshot,
apt_packages: apt.map((name, i) => plainWord(typeof name === "string" ? name : "", `${where} apt_packages[${i}]`)),
};
}
+5 -7
View File
@@ -14,11 +14,11 @@
*/
import { createHash } from "node:crypto";
import { createReadStream } from "node:fs";
import { chmod, mkdir, open, rm } from "node:fs/promises";
import { mkdir, open, rm } from "node:fs/promises";
import { join } from "node:path";
import { replaceFile } from "@invisible-dots/shared";
import { DownloadError, fetchTextWithRetries, fetchVerified, type FetchVerifiedOptions } from "./download.js";
import { writeManifest, type GoldenManifest } from "./manifest.js";
import type { GoldenManifest } from "./manifest.js";
import { publishImage } from "./publish.js";
import { checkVersion } from "./versions.js";
/** Where the releases of the repository are downloaded from. */
@@ -139,10 +139,8 @@ export async function pullPrebuiltGolden(options: PullOptions): Promise<PulledIm
throw new Error(`the prebuilt golden image hashes to ${sha256}, its manifest says ${manifest.sha256}`);
}
const manifestPath = options.manifestPath(image);
// The manifest first, as for a built image: the control plane picks images by file name.
await writeManifest(manifestPath, manifest);
await replaceFile(partial, image);
await chmod(image, 0o444);
// Published as a built image is: the manifest first, the image read-only.
await publishImage(partial, image, manifestPath, manifest);
await rm(workDir, { recursive: true, force: true });
log(`done: ${image} (sha256 ${sha256})`);
return { version, image, manifest: manifestPath };
+6 -13
View File
@@ -1,22 +1,15 @@
/**
* The process boundary of the builder: the two calls it makes into the host.
* It is the shape of the vm-manager's CommandRunner, declared here rather
* than imported so this package's logic loads without the vm-manager; the
* real runner is the vm-manager's (see host.ts), and tests pass a fake.
* The process boundary of the builder: the two calls it makes into the host, in the vm-manager's terms (its
* CommandRunner's `run` and the process `startProcess` gives). The real runner joins those two (the CLI's host.ts),
* and tests pass a fake.
*/
import type { CommandResult, StartedProcess } from "@invisible-dots/vm-manager";
export interface RunningProcess {
readonly pid: number | undefined;
/** The last few KiB of stderr, for error messages when the process dies. */
stderrTail(): string;
/** Called once when the process exits or fails to start; `code` is null when killed by a signal. */
onExit(listener: (code: number | null, signal: NodeJS.Signals | null, error?: Error) => void): void;
kill(signal?: NodeJS.Signals): void;
}
export type RunningProcess = StartedProcess;
export interface ProcessRunner {
/** Runs to completion; rejects unless the command exits 0. Argument arrays only, never a shell. */
run(command: string, args: readonly string[], options?: { timeoutMs?: number }): Promise<{ stdout: string; stderr: string }>;
run(command: string, args: readonly string[], options?: { timeoutMs?: number }): Promise<CommandResult>;
/** `cwd` is the child's working directory: never inherited, see the vm-manager's startProcess. */
spawn(command: string, args: readonly string[], options: { cwd: string }): RunningProcess;
}
+75
View File
@@ -0,0 +1,75 @@
/**
* What every way of getting an image shares (the golden image built or downloaded, the runtime ISO): which version
* a build makes, or that an image of these inputs is there already, and how a finished image takes its name.
*/
import { chmod, mkdir, stat } from "node:fs/promises";
import { basename } from "node:path";
import { replaceFile, type ImageKind } from "@invisible-dots/shared";
import { manifestPathFor, writeManifest, type ImageManifest } from "./manifest.js";
import { checkVersion, findImageByDigest, versionFor } from "./versions.js";
export interface ImageTarget {
version: string;
image: string;
manifest: string;
/** True when the image is there already, complete with its manifest: there is nothing to build. */
existing: boolean;
}
export interface ResolveImageOptions {
kind: ImageKind;
imagesDir: string;
/** The digest of the image's inputs. */
digest: string;
/** An explicit version, used as given; undefined is `<UTC time>-<digest>`, or the image that has this digest. */
version: string | undefined;
now: () => Date;
/** The path of the image of a version. */
imagePath: (version: string) => string;
log: (line: string) => void;
}
async function present(path: string): Promise<boolean> {
return (await stat(path).catch(() => undefined)) !== undefined;
}
/**
* The image a build makes: with no explicit version, the image of the images directory that has these inputs if
* there is one, otherwise a new version. An image file without its manifest is one no build finished (the manifest
* is written first), and is refused rather than built over.
*/
export async function resolveImageVersion(options: ResolveImageOptions): Promise<ImageTarget> {
await mkdir(options.imagesDir, { recursive: true });
let version: string;
if (options.version === undefined) {
const found = await findImageByDigest(options.imagesDir, options.kind, options.digest);
if (found) {
options.log(`${basename(found.path)} already holds these inputs; nothing to do`);
return { version: found.version, image: found.path, manifest: manifestPathFor(found.path), existing: true };
}
version = versionFor(options.now(), options.digest);
} else {
version = checkVersion(options.version);
}
const image = options.imagePath(version);
const manifest = manifestPathFor(image);
if (await present(image)) {
if (await present(manifest)) {
options.log(`${basename(image)} already exists; nothing to do`);
return { version, image, manifest, existing: true };
}
throw new Error(`${image} exists without its manifest ${manifest}: it is not one this builder finished; remove it by hand`);
}
return { version, image, manifest, existing: false };
}
/**
* Puts a finished image (`partial`, on the same filesystem) under its name. The manifest goes first: the control
* plane picks images by file name, so the image must never appear before the record that describes it. The image is
* read-only from then on: immutable once a VM uses it (architecture section 3.3).
*/
export async function publishImage(partial: string, image: string, manifestPath: string, manifest: ImageManifest): Promise<void> {
await writeManifest(manifestPath, manifest);
await replaceFile(partial, image);
await chmod(image, 0o444);
}
+1 -2
View File
@@ -27,9 +27,8 @@ export interface BuilderVmSpec {
serialLog: string;
}
/** The builder VM's command line; its cpus and memory are held to a Dot's bounds by `machineArgs()`. */
export function builderQemuArgs(spec: BuilderVmSpec): string[] {
if (!Number.isInteger(spec.cpus) || spec.cpus < 1) throw new Error(`cpus must be a positive integer, got ${spec.cpus}`);
if (!Number.isInteger(spec.memoryMib) || spec.memoryMib < 1024) throw new Error(`memory must be at least 1024 MiB, got ${spec.memoryMib}`);
return [
"-name",
"invisible-dots-image-builder",
+41 -40
View File
@@ -4,6 +4,7 @@
* /opt/invisible-dots:
*
* /install.sh the hook each Dot's seed runs on every boot
* /vmproxy.py install.sh's setup of the Dot's VM proxy (tunnel, routes, firewall)
* /VERSION this runtime's version
* /bin/dot-agentd the computer daemon (linux/amd64)
* /bin/dot-desktop ExecStart of dot-desktop.service
@@ -22,11 +23,11 @@
* permission bits having to survive a Windows host.
*/
import { createHash } from "node:crypto";
import { chmod, mkdir, open, readdir, rm, stat } from "node:fs/promises";
import { open, readdir, rm, stat } from "node:fs/promises";
import { basename, join } from "node:path";
import { fileURLToPath } from "node:url";
import { writeIso, type IsoEntry } from "@invisible-dots/iso";
import { hostPaths, replaceFile, type HostPaths } from "@invisible-dots/shared";
import { hostPaths, type HostPaths } from "@invisible-dots/shared";
import { RUNTIME_ISO_LABEL } from "@invisible-dots/vm-manager";
import {
defaultAssetRoot,
@@ -35,12 +36,14 @@ import {
RUNTIME_DESKTOP,
RUNTIME_DOT_INSTALL,
RUNTIME_INSTALL,
RUNTIME_VMPROXY,
unitAsset,
} from "./assets.js";
import { sha256File } from "./download.js";
import { acquireLock } from "./lock.js";
import { manifestPathFor, writeManifest, type RuntimeFile, type RuntimeManifest } from "./manifest.js";
import { checkVersion, findImageByDigest, inputsDigest, versionFor } from "./versions.js";
import type { RuntimeFile, RuntimeManifest } from "./manifest.js";
import { publishImage, resolveImageVersion } from "./publish.js";
import { inputsDigest } from "./versions.js";
export interface RuntimeInputs {
/** guest/dot-agentd/bin/dot-agentd, built for linux/amd64 */
@@ -65,15 +68,28 @@ export interface BuildCommand {
}
/**
* The command that builds `defaultRuntimeInputs(repoRoot).agentdBinary`: a static linux/amd64 executable whatever
* the host is (`assertLinuxAmd64Elf` refuses anything else), with the paths trimmed so the same source gives the same
* bytes from any checkout. `go -C` enters the module itself, so the caller's working directory does not matter.
* The one build of dot-agentd, the same wherever it runs (this function, CI, the smoke, the READMEs): a static
* linux/amd64 executable whatever the host is (`assertLinuxAmd64Elf` refuses anything else), with the same bytes from
* the same source and toolchain (go.mod's `toolchain`) in any checkout: `-trimpath` leaves out the checkout's path,
* `-buildvcs=false` the commit and whether the tree had changes, which Go otherwise stamps into a binary built in a
* git checkout, so that every commit, and every edit of any file, would make another runtime disk; `-s -w` leave out
* the symbol table and the debug information.
*/
export function agentdBuildCommand(repoRoot: string): BuildCommand {
const AGENTD_BUILD_ENV = { CGO_ENABLED: "0", GOOS: "linux", GOARCH: "amd64" } as const;
const AGENTD_BUILD_FLAGS = ["-trimpath", "-buildvcs=false", "-ldflags=-s -w"] as const;
/** The same build as a shell line, run in guest/dot-agentd: what the documentation and the workflows quote. */
export const AGENTD_BUILD_SHELL = 'CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false -ldflags="-s -w" -o bin/dot-agentd ./cmd/dot-agentd';
/**
* The command that builds dot-agentd into `output`, by default `defaultRuntimeInputs(repoRoot).agentdBinary`, the
* file the runtime disk takes. `go -C` enters the module itself, so the caller's working directory does not matter.
*/
export function agentdBuildCommand(repoRoot: string, output: string = defaultRuntimeInputs(repoRoot).agentdBinary): BuildCommand {
return {
command: "go",
args: ["-C", join(repoRoot, "guest", "dot-agentd"), "build", "-trimpath", "-o", defaultRuntimeInputs(repoRoot).agentdBinary, "./cmd/dot-agentd"],
env: { CGO_ENABLED: "0", GOOS: "linux", GOARCH: "amd64" },
args: ["-C", join(repoRoot, "guest", "dot-agentd"), "build", ...AGENTD_BUILD_FLAGS, "-o", output, "./cmd/dot-agentd"],
env: { ...AGENTD_BUILD_ENV },
};
}
@@ -112,8 +128,7 @@ export async function assertLinuxAmd64Elf(path: string): Promise<void> {
const machine = header.readUInt16LE(18);
if (!elf || !is64 || !little || machine !== 0x3e) {
throw new Error(
`${path} is not a linux/amd64 executable; build it with ` +
`"go build" and GOOS=linux GOARCH=amd64 CGO_ENABLED=0 in guest/dot-agentd`,
`${path} is not a linux/amd64 executable; build it in guest/dot-agentd with ${AGENTD_BUILD_SHELL}`,
);
}
}
@@ -192,14 +207,14 @@ export async function runtimeFiles(inputs: RuntimeInputs, assetRoot: string = de
const agentd = await stat(inputs.agentdBinary).catch(() => undefined);
if (!agentd?.isFile()) {
throw new Error(
`dot-agentd binary ${inputs.agentdBinary} not found: build it first ` +
`(in guest/dot-agentd: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/dot-agentd ./cmd/dot-agentd)`,
`dot-agentd binary ${inputs.agentdBinary} not found: build it first (in guest/dot-agentd: ${AGENTD_BUILD_SHELL})`,
);
}
await assertLinuxAmd64Elf(inputs.agentdBinary);
const files = [
stageBytes("install.sh", await readGuestAsset(assetRoot, RUNTIME_INSTALL)),
stageBytes("vmproxy.py", await readGuestAsset(assetRoot, RUNTIME_VMPROXY)),
stageBytes("bin/dot-desktop", await readGuestAsset(assetRoot, RUNTIME_DESKTOP)),
stageBytes("bin/dot-install", await readGuestAsset(assetRoot, RUNTIME_DOT_INSTALL)),
await stageFile("bin/dot-agentd", inputs.agentdBinary),
@@ -216,28 +231,17 @@ export async function buildRuntimeIso(options: RuntimeBuildOptions = {}): Promis
const files = await runtimeFiles(options.inputs ?? defaultRuntimeInputs(), options.assetRoot ?? defaultAssetRoot());
const digest = inputsDigest(files.flatMap((file) => [file.path, file.sha256]));
await mkdir(paths.imagesDir, { recursive: true });
let version: string;
if (options.version === undefined) {
const existing = await findImageByDigest(paths.imagesDir, "runtime", digest);
if (existing) {
log(`${basename(existing.path)} already holds this code; nothing to do`);
return { version: existing.version, iso: existing.path, manifest: manifestPathFor(existing.path), created: false };
}
version = versionFor(now(), digest);
} else {
version = checkVersion(options.version);
}
const iso = paths.runtimeIsoPath(version);
const manifestPath = manifestPathFor(iso);
if ((await stat(iso).catch(() => undefined)) !== undefined) {
if ((await stat(manifestPath).catch(() => undefined)) !== undefined) {
log(`${basename(iso)} already exists; nothing to do`);
return { version, iso, manifest: manifestPath, created: false };
}
throw new Error(`${iso} exists without its manifest ${manifestPath}: it is not one this builder finished; remove it by hand`);
}
const target = await resolveImageVersion({
kind: "runtime",
imagesDir: paths.imagesDir,
digest,
version: options.version,
now,
imagePath: (v) => paths.runtimeIsoPath(v),
log,
});
const { version, image: iso, manifest: manifestPath } = target;
if (target.existing) return { version, iso, manifest: manifestPath, created: false };
const lock = await acquireLock(join(paths.imagesDir, ".runtime-build.lock"), "runtime ISO build");
try {
@@ -263,10 +267,7 @@ export async function buildRuntimeIso(options: RuntimeBuildOptions = {}): Promis
content_digest: digest,
files: all.map((file): RuntimeFile => ({ path: file.path, sha256: file.sha256, size_bytes: file.size })),
};
// Manifest first, then the image appears under the name the control plane looks for.
await writeManifest(manifestPath, manifest);
await replaceFile(partial, iso);
await chmod(iso, 0o444);
await publishImage(partial, iso, manifestPath, manifest);
log(`done: ${iso} (sha256 ${manifest.sha256})`);
return { version, iso, manifest: manifestPath, created: true };
} catch (error) {
+3 -1
View File
@@ -2,7 +2,8 @@
* The builder VM's seed disk. It is one ISO labelled `cidata`: cloud-init's
* NoCloud datasource reads user-data and meta-data from it, and the
* provisioner mounts it again to reach its own files, so the builder needs
* one CD-ROM and no second payload disk.
* this one read-only disk (a virtio disk, as every ISO of this project is
* attached: the vm-manager's isoDriveArg) and no second payload disk.
*/
import type { IsoEntry } from "@invisible-dots/iso";
import { downloadFileName, type GuestPins } from "./pins.js";
@@ -53,6 +54,7 @@ export function pinsEnv(pins: GuestPins, python: PythonLock): string {
["ENGINE_LOCK", SEED_ENGINE_LOCK],
["ENGINE_BUILD", SEED_ENGINE_BUILD],
["BROWSER_BUILD", SEED_BROWSER_BUILD],
["APT_SNAPSHOT", pins.apt_snapshot],
["APT_PACKAGES", pins.apt_packages.join(" ")],
];
return values
+2
View File
@@ -58,6 +58,7 @@ beforeEach(async () => {
pins = {
uv: { version: "0.12.22", url: http.url(`/uv/${UV_FILE}`), shasums_url: http.url(`/uv/${UV_FILE}.sha256`), shasums_entry: UV_FILE, sha256: sha256(UV) },
tunnel: { version: "2.18.0", url: http.url(`/tunnel/${TUNNEL_FILE}`), sha256: sha256(TUNNEL) },
apt_snapshot: "20261010T000000Z",
apt_packages: ["xvfb", "imagemagick"],
};
logs = [];
@@ -133,6 +134,7 @@ describe("buildGoldenImage", () => {
uv: { version: "0.12.22", sha256: sha256(UV) },
"invisible-playwright-mcp": "0.71.1",
"invisible-playwright": "0.32.0",
apt_snapshot: "20261010T000000Z",
apt_packages: ["xvfb", "imagemagick"],
},
// The engine's lock, which the runtime disk's copy must equal.
+60 -4
View File
@@ -1,10 +1,10 @@
// Checks of the files that run inside the guest. A mistake in them shows up
// only inside a booted VM, minutes into a build or on every Dot's boot, so
// the cheap ones are caught here.
import { execFileSync } from "node:child_process";
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { execFileSync, spawnSync } from "node:child_process";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { delimiter, join } from "node:path";
import { GUEST_PATHS } from "@invisible-dots/shared";
import { describe, expect, it } from "vitest";
import { BUILDER_PYTHON_LOCK, defaultAssetRoot, GUEST_ASSETS, GUEST_UNITS, unitAsset } from "../src/assets.js";
@@ -115,6 +115,62 @@ describe("the sudo grants (architecture section 4.1)", () => {
});
});
/**
* dot-install as sudo runs it, with `id` answering root and `apt-get` a stand-in that records its arguments: what
* reaches apt, or that nothing does.
*/
describe.skipIf(!bashUsable)("dot-install, the one command dot may run as root", () => {
function dotInstall(...names: string[]): { status: number; stderr: string; apt: string[] } {
const stubs = mkdtempSync(join(tmpdir(), "idots-dot-install-"));
try {
const calls = join(stubs, "apt-calls");
writeFileSync(join(stubs, "id"), "#!/bin/sh\necho 0\n", { mode: 0o755 });
writeFileSync(join(stubs, "apt-get"), `#!/bin/sh\necho "$*" >> '${calls.replace(/\\/g, "/")}'\n`, { mode: 0o755 });
// Windows spells the variable Path, and an environment holding both spellings is ambiguous.
const pathKey = Object.keys(process.env).find((key) => key.toUpperCase() === "PATH") ?? "PATH";
const run = spawnSync("bash", [join(root, "runtime", "dot-install.sh"), ...names], {
env: { ...process.env, [pathKey]: `${stubs}${delimiter}${process.env[pathKey] ?? ""}` },
encoding: "utf8",
timeout: 20_000,
});
const apt = existsSync(calls) ? readFileSync(calls, "utf8").trim().split("\n") : [];
return { status: run.status ?? -1, stderr: run.stderr, apt };
} finally {
rmSync(stubs, { recursive: true, force: true });
}
}
const lock = "-o DPkg::Lock::Timeout=300";
it("installs package names, those that end in + included", () => {
expect(dotInstall("nodejs", "libgtk-3-0t64", "python3.12", "g++", "ncbi-blast+")).toEqual({
status: 0,
stderr: "",
apt: [`${lock} update -qq`, `${lock} install -y nodejs libgtk-3-0t64 python3.12 g++ ncbi-blast+`],
});
});
it("passes a trailing + on, which apt reads as install", () => {
expect(dotInstall("foo+").apt).toEqual([`${lock} update -qq`, `${lock} install -y foo+`]);
});
it.each([
["a trailing -, which apt reads as remove", "coreutils-"],
["an option", "-o"],
["a long option", "--allow-remove-essential"],
["a version", "foo=1.0"],
["a release", "foo/noble"],
["an architecture", "foo:i386"],
["upper case", "Foo"],
["one character", "a"],
["an empty word", ""],
])("refuses %s, and nothing reaches apt", (_what, name) => {
const run = dotInstall("nodejs", name);
expect(run.status).toBe(2);
expect(run.stderr).toContain(`"${name}" is not a package name`);
expect(run.apt).toEqual([]);
});
});
describe("the Dot's browser", () => {
const provision = text("builder/provision.sh");
const build = text("builder/build-browser-env.sh");
@@ -123,7 +179,7 @@ describe("the Dot's browser", () => {
expect(provision).toContain('bash "$payload/$BROWSER_BUILD" "$payload/$PYTHON_LOCK" "$mcp_env"');
expect(provision).toContain("mcp_env=/home/dot/.local/share/invisible-dots/mcp\n");
expect(build).toContain('as_dot uv venv --quiet --python /usr/bin/python3 "$env_dir"');
expect(build).toContain('as_dot uv pip install --python "$env_dir/bin/python" --require-hashes -r "$dot_lock"');
expect(build).toContain('as_dot uv pip install --no-cache --python "$env_dir/bin/python" --require-hashes --only-binary :all: -r "$dot_lock"');
expect(build).not.toMatch(/uv tool install/);
expect(build).toContain('as_dot ln -sfn "$env_dir/bin/invisible-playwright-mcp" "$bin_dir/invisible-playwright-mcp"');
expect(build).toContain('as_dot "$env_dir/bin/invisible-playwright" fetch');
+7
View File
@@ -58,6 +58,13 @@ describe("pin validation", () => {
expect(() => parseGuestPins({ ...good, apt_packages: ["xvfb", "x; rm -rf /"] })).toThrow(/apt_packages\[1\]/);
});
it("takes the packages from a pinned snapshot of the archive, and refuses one that is not a snapshot time", () => {
expect(GUEST_PINS.apt_snapshot).toMatch(/^\d{8}T\d{6}Z$/);
expect(() => parseGuestPins({ ...good, apt_snapshot: "latest" })).toThrow(/apt_snapshot/);
const { apt_snapshot: _snapshot, ...unpinned } = good;
expect(() => parseGuestPins(unpinned)).toThrow(/apt_snapshot/);
});
it("refuses python_packages, which would be a second copy of the lock's versions", () => {
expect(() => parseGuestPins({ ...good, python_packages: { "invisible-playwright-mcp": "0.70.2" } })).toThrow(/mcp-requirements\.lock/);
});
+18 -1
View File
@@ -8,7 +8,7 @@ import { fileBytes, listFiles, parseIso } from "../../../packages/iso/test/iso-r
import { defaultAssetRoot, GUEST_UNITS } from "../src/assets.js";
import { readManifest, verifyImage, type RuntimeManifest } from "../src/manifest.js";
import { RUNTIME_ISO_LABEL } from "@invisible-dots/vm-manager";
import { agentdBuildCommand, assertLinuxAmd64Elf, buildRuntimeIso, defaultRuntimeInputs, runtimeFiles, type RuntimeInputs } from "../src/runtime.js";
import { AGENTD_BUILD_SHELL, agentdBuildCommand, assertLinuxAmd64Elf, buildRuntimeIso, defaultRuntimeInputs, runtimeFiles, type RuntimeInputs } from "../src/runtime.js";
import { sha256 } from "./http-fixture.js";
/** The first bytes of an ELF executable for `machine` (0x3e is x86-64, 0xb7 arm64). */
@@ -73,11 +73,26 @@ describe("agentdBuildCommand", () => {
join("repo", "root", "guest", "dot-agentd"),
"build",
"-trimpath",
"-buildvcs=false",
"-ldflags=-s -w",
"-o",
defaultRuntimeInputs(join("repo", "root")).agentdBinary,
"./cmd/dot-agentd",
]);
});
it("builds into another file when asked, with the same flags", () => {
const build = agentdBuildCommand(join("repo", "root"), join("elsewhere", "dot-agentd"));
expect(build.args).toEqual([...agentdBuildCommand(join("repo", "root")).args.slice(0, -3), "-o", join("elsewhere", "dot-agentd"), "./cmd/dot-agentd"]);
});
it("is the build the shell line quotes: its environment, then go build with its flags, run in guest/dot-agentd", () => {
const build = agentdBuildCommand(join("repo", "root"), "bin/dot-agentd");
const env = Object.entries(build.env).map(([name, value]) => `${name}=${value}`);
// The shell line's one quoted word, -ldflags="-s -w", is the single argument -ldflags=-s -w.
const args = build.args.slice(2).map((arg) => (arg.includes(" ") ? arg.replace(/=(.*)$/, '="$1"') : arg));
expect(AGENTD_BUILD_SHELL).toBe([...env, "go", ...args].join(" "));
});
});
describe("buildRuntimeIso", () => {
@@ -108,6 +123,7 @@ describe("buildRuntimeIso", () => {
"engine/nanobot/templates/agent/tool_contract.md",
"engine/skills/web-forms/SKILL.md",
"install.sh",
"vmproxy.py",
...GUEST_UNITS.map((unit) => `units/${unit}`),
].sort(),
);
@@ -115,6 +131,7 @@ describe("buildRuntimeIso", () => {
expect(read("VERSION").toString()).toBe(`${result.version}\n`);
expect(read("bin/dot-agentd")).toEqual(elf(0x3e));
expect(read("install.sh")).toEqual(await readFile(join(defaultAssetRoot(), "runtime", "install.sh")));
expect(read("vmproxy.py")).toEqual(await readFile(join(defaultAssetRoot(), "runtime", "vmproxy.py")));
expect(read("bin/dot-desktop")).toEqual(await readFile(join(defaultAssetRoot(), "runtime", "dot-desktop.sh")));
expect(read("units/dot-agentd.service")).toEqual(await readFile(join(defaultAssetRoot(), "units", "dot-agentd.service")));
// The engine's source as it is.
+1
View File
@@ -32,6 +32,7 @@ describe("pins.env", () => {
expect(text).toContain("ENGINE_LOCK='engine-requirements.lock'\n");
expect(text).toContain("ENGINE_BUILD='build-engine-env.sh'\n");
expect(text).toContain("BROWSER_BUILD='build-browser-env.sh'\n");
expect(text).toContain(`APT_SNAPSHOT='${GUEST_PINS.apt_snapshot}'\n`);
expect(text).toContain(`APT_PACKAGES='${GUEST_PINS.apt_packages.join(" ")}'\n`);
const provision = (await readGuestAsset(defaultAssetRoot(), BUILDER_PROVISION)).toString("utf8");
+22 -4
View File
@@ -43,13 +43,31 @@ describe("installedComponents", () => {
});
describe("followSerialLog", () => {
it("reports progress lines as they are appended, including a file that appears late", async () => {
it("reports nothing and stops cleanly while the file does not exist", async () => {
const seen: string[] = [];
const follower = followSerialLog(join(dir, "serial.log"), (step) => seen.push(step), 10);
await follower.stop();
expect(seen).toEqual([]);
expect(follower.tail(1)).toEqual([]);
});
it("reports progress lines as they are appended, from a file that appears after it started", async () => {
const path = join(dir, "serial.log");
const seen: string[] = [];
const follower = followSerialLog(path, (step) => seen.push(step), 10);
await new Promise((resolve) => setTimeout(resolve, 30));
let progressed = () => {};
const progress = () => new Promise<void>((resolve) => (progressed = resolve));
const follower = followSerialLog(
path,
(step) => {
seen.push(step);
progressed();
},
10,
);
const first = progress();
await appendFile(path, "booting\r\nidots-build: installing packages\r\nidots-bu");
await new Promise((resolve) => setTimeout(resolve, 50));
await first;
// The half line is held back until its end arrives.
expect(seen).toEqual(["installing packages"]);
await appendFile(path, "ild: installing Node 24\nIDOTS-BUILD-RESULT: ok");
await follower.stop();
+2 -2
View File
@@ -129,7 +129,7 @@ engine) and at a launch (the egress address, for the timezone, and the library's
## Run it
Only docker is needed, with the images `golang:1.26` and `ubuntu:24.04` (pulled when
Only docker is needed, with the images `golang:1.26.8` and `ubuntu:24.04` (pulled when
absent; the container downloads the pinned `uv` and the locked wheels).
```sh
@@ -157,7 +157,7 @@ removes the volume and the container on exit.
| file | what it is |
|---|---|
| `run.sh` | the entry: runs dot-agentd's `privileged` Go tests as root in `golang:1.26` (they run work as another user; a skipped one fails the run), builds `dot-agentd` there, starts `ubuntu:24.04` with the tree, checks the exit status and the summary line; `--suite browser` runs the browser smoke |
| `run.sh` | the entry: runs dot-agentd's `privileged` Go tests as root in `golang:1.26.8` (they run work as another user; a skipped one fails the run), builds `dot-agentd` there, starts `ubuntu:24.04` with the tree, checks the exit status and the summary line; `--suite browser` runs the browser smoke |
| `prepare-engine.sh` | in the container: the golden image's users, `uv`, the engine's environment, the staged engine source (and for the browser suite the apt packages and the browser); then it runs the suite's checks |
| `lib.sh` | what both suites share: the guest laid out as `install.sh` lays it out, `dot-agentd` and the engine started and restarted, the key and config push, the event stream and the helpers that read it |
| `smoke.sh` | the engine smoke's checks; prints `PASS:` or `FAIL:` per check and the summary line |
@@ -29,7 +29,8 @@ esac
export AGENTD_BIN
export DEBIAN_FRONTEND=noninteractive
apt-get update -qq >/dev/null
# nftables and iproute2: the check of the VM proxy's firewall runs install.sh's own generator and nft on its output.
# nftables and iproute2: the checks of the VM proxy run the runtime disk's vmproxy.py (it reads the default route)
# and nft on its output.
apt-get install -y -qq sudo procps jq curl ca-certificates python3 nftables iproute2 >/dev/null
builder=$TREE/guest/image-builder/builder
@@ -73,8 +74,11 @@ print('imported', len(names), 'modules from', nanobot.__file__)
"
if [ "$suite" = browser ]; then
# The packages of the golden image (pins.json), as provision.sh installs them.
apt-get install -y -qq --no-install-recommends $(jq -r '.apt_packages[]' "$TREE/guest/image-builder/pins.json") >/dev/null
# The packages of the golden image (pins.json), as provision.sh installs them: from the archive as of the pinned
# snapshot (reached over https, which the ca-certificates installed above make possible).
snapshot=$(jq -r '.apt_snapshot' "$TREE/guest/image-builder/pins.json")
apt-get --snapshot "$snapshot" --error-on=any update -qq >/dev/null
apt-get --snapshot "$snapshot" install -y -qq --no-install-recommends $(jq -r '.apt_packages[]' "$TREE/guest/image-builder/pins.json") >/dev/null
# The Dot's browser, with the script provision.sh runs on the same lock. dot reaches its home, not the tree.
bash "$builder/build-browser-env.sh" "$builder/mcp-requirements.lock" /home/dot/.local/share/invisible-dots/mcp
fi
+8 -8
View File
@@ -1,6 +1,6 @@
#!/usr/bin/env bash
# The engine smoke's entry point: builds dot-agentd from a source tree in
# golang:1.26, starts ubuntu:24.04 with the tree mounted, builds the engine's
# golang:1.26.8, starts ubuntu:24.04 with the tree mounted, builds the engine's
# environment there as provision.sh does (builder/build-engine-env.sh on the
# hashed lock), stages the engine's source as the runtime ISO does, and runs
# the checks of smoke.sh against a Dot.
@@ -78,21 +78,21 @@ fi
# --init: a process the tests kill is a zombie until its parent reaps it, and the tests ask whether it is gone.
if ! privileged=$(docker run --rm --init -v "$run_id":/work "${tree_mount[@]}" \
-e CGO_ENABLED=0 -e GOCACHE=/work/gocache -e GOTOOLCHAIN=local -e GOFLAGS=-buildvcs=false \
-w "$tree/guest/dot-agentd" golang:1.26 \
-w "$tree/guest/dot-agentd" golang:1.26.8 \
go test -count=1 -v -tags privileged ./... 2>&1) || printf '%s\n' "$privileged" | grep -q -e '--- SKIP'; then
printf '%s\n' "$privileged" | grep -v -e '^=== ' | tail -40 >&2
echo "run.sh: dot-agentd's privileged tests did not pass, or one was skipped" >&2
exit 1
fi
# dot-agentd for the guest, as CI's go job builds it. GOTOOLCHAIN=local: the
# image's Go is the one that builds, nothing is downloaded. -buildvcs=false:
# the mounted tree belongs to another user, and a build must not depend on git.
# dot-agentd for the guest, with the one build (guest/dot-agentd/README.md) and the Go
# go.mod's toolchain names. GOTOOLCHAIN=local: the image's Go is the one that
# builds, nothing is downloaded.
docker run --rm -v "$run_id":/work "${tree_mount[@]}" \
-e CGO_ENABLED=0 -e GOOS=linux -e GOARCH=amd64 \
-e GOCACHE=/work/gocache -e GOTOOLCHAIN=local -e GOFLAGS=-buildvcs=false \
-w "$tree/guest/dot-agentd" golang:1.26 \
go build -trimpath -ldflags="-s -w" -o /work/dot-agentd ./cmd/dot-agentd
-e GOCACHE=/work/gocache -e GOTOOLCHAIN=local \
-w "$tree/guest/dot-agentd" golang:1.26.8 \
go build -trimpath -buildvcs=false -ldflags="-s -w" -o /work/dot-agentd ./cmd/dot-agentd
# The checks of the tree under test, in a clean Ubuntu. PIN_REMOVALS reaches
# smoke.sh only when it is set here (see the top of smoke.sh).
+29 -8
View File
@@ -58,18 +58,39 @@ lay_out_guest
# The engine needs no privilege: no sudoers rule, no config directory.
check "the engine has no sudo rule (no /etc/sudoers.d/invisible-dots-engine, no sudo for dotengine)" "[ ! -e /etc/sudoers.d/invisible-dots-engine ] && ! su -s /bin/bash dotengine -c 'sudo -n true' >/dev/null 2>&1"
check "no sudoers file names dotengine (the old engine's rule is gone with the config installer)" "! grep -rqs dotengine /etc/sudoers /etc/sudoers.d"
# --- the VM proxy's firewall, as install.sh writes it, is a ruleset nft takes ---
# --- the VM proxy, as install.sh sets it up: the runtime disk's vmproxy.py on a config that has a proxy ---
# A rule nft refused stopped install.sh (set -e) before the tunnel started: the Dot went out directly with its proxy
# set. install.sh's own generator runs here on a config that has a proxy, read from a copy of its path.
# set. And a proxy that cannot be set up is reported for what it is: install.sh logs vmproxy.py's reason and lets
# nothing out.
VMPROXY="$TREE/guest/image-builder/runtime/vmproxy.py"
vmproxy() { # proxy value, [PATH]: vmproxy.py on a config with that proxy; what it wrote is in $VMPROXY_OUT
printf '{"dotId":"dot_smoke","token":"t","proxy":"%s"}' "$1" > /tmp/vmproxy-config.json
VMPROXY_OUT=$(mktemp -d)
PATH="${2:-$PATH}" python3 "$VMPROXY" /tmp/vmproxy-config.json "$VMPROXY_OUT" 2> /tmp/vmproxy-stderr
}
vmproxy_rules_ok() {
local out
out=$(mktemp -d)
printf '{"dotId":"dot_smoke","token":"t","proxy":"socks5://user:p%%[email protected]:1080"}' > /tmp/vmproxy-config.json
sed -n "/<<'PY'\$/,/^PY\$/p" "$TREE/guest/image-builder/runtime/install.sh" | sed '1d;$d' \
| sed 's#/etc/invisible-dots/config.json#/tmp/vmproxy-config.json#' > /tmp/vmproxy-gen.py
python3 /tmp/vmproxy-gen.py "$out" && [ -s "$out/allow.nft" ] && [ -s "$out/hev.yml" ] && nft -c -f "$out/allow.nft"
vmproxy 'socks5://user:p%[email protected]:1080' && [ -s "$VMPROXY_OUT/allow.nft" ] && [ -s "$VMPROXY_OUT/hev.yml" ] \
&& [ -s "$VMPROXY_OUT/up.sh" ] && grep -q 'password: "p@ss"' "$VMPROXY_OUT/hev.yml" && nft -c -f "$VMPROXY_OUT/allow.nft"
}
check "the VM proxy's firewall that install.sh writes for a proxy is a ruleset nft accepts" "vmproxy_rules_ok"
vmproxy_refuses() { # exit code, words of the reason, proxy value, [PATH]: that code, that reason, and nothing written
vmproxy "$3" "${4:-}"
local code=$?
if [ "$code" -eq "$1" ] && grep -q "$2" /tmp/vmproxy-stderr && [ -z "$(ls -A "$VMPROXY_OUT")" ]; then return 0; fi
echo "exit $code: $(cat /tmp/vmproxy-stderr)"
return 1
}
# An ip that shows a default route without a gateway, as a VPN's or a point-to-point link's would be.
mkdir -p /tmp/vmproxy-noroute
printf '#!/bin/sh\necho "default dev wg0 scope link"\n' > /tmp/vmproxy-noroute/ip
chmod 0755 /tmp/vmproxy-noroute/ip
NOROUTE_PATH="/tmp/vmproxy-noroute:$PATH"
check "a proxy that is not socks5://host:port is reported as that (exit 3) and nothing is written" \
"vmproxy_refuses 3 'is not socks5://' 'http://10.0.2.2:1080' && vmproxy_refuses 3 'is not socks5://' 'socks5://10.0.2.2:99999'"
check "a proxy whose host does not resolve is reported as that (exit 4), not as a bad value" \
"vmproxy_refuses 4 'does not resolve' 'socks5://proxy.invalid:1080'"
check "a VM with no default route through a gateway is reported as that (exit 5), not as a bad value" \
"vmproxy_refuses 5 'no default IPv4 route through a gateway' 'socks5://10.0.2.2:1080' \"\$NOROUTE_PATH\""
check "dotengine is in no group but its own and dot" "[ \"\$(id -nG dotengine | tr ' ' '\n' | sort | tr '\n' ' ')\" = 'dot dotengine ' ]"
write_host_token
check "the token file is dot-agentd's own (dotagentd, 0600): neither the engine nor dot, the user of the model's commands, can read it" "[ \"\$(stat -c '%U:%G %a' /etc/invisible-dots/config.json)\" = 'dotagentd:dotagentd 600' ] && ! su -s /bin/bash dotengine -c 'cat /etc/invisible-dots/config.json' >/dev/null 2>&1 && ! su -s /bin/bash dot -c 'cat /etc/invisible-dots/config.json' >/dev/null 2>&1"
+3 -3
View File
@@ -106,7 +106,7 @@ git clone https://github.com/feder-cr/invisible_dots && cd invisible_dots
npm ci
npm run build --workspace @invisible-dots/cli
npm run build --workspace @invisible-dots/web # skip with E2E_WEB=0
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go -C guest/dot-agentd build -trimpath -o bin/dot-agentd ./cmd/dot-agentd
(cd guest/dot-agentd && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false -ldflags="-s -w" -o bin/dot-agentd ./cmd/dot-agentd)
# The key goes in a file only you can read; the run never prints it.
install -m 600 /dev/null ~/openrouter.key && $EDITOR ~/openrouter.key
@@ -146,8 +146,8 @@ device's group, as on a real host; no `--privileged`.
```bash
# dot-agentd needs Go, which the host image deliberately lacks
docker run --rm -v "$PWD/guest/dot-agentd:/src" -w /src \
-e CGO_ENABLED=0 -e GOOS=linux -e GOARCH=amd64 -e GOFLAGS=-buildvcs=false \
golang:1.26 go build -trimpath -o bin/dot-agentd ./cmd/dot-agentd
-e CGO_ENABLED=0 -e GOOS=linux -e GOARCH=amd64 \
golang:1.26.8 go build -trimpath -buildvcs=false -ldflags="-s -w" -o bin/dot-agentd ./cmd/dot-agentd
# The container's user takes your uid and gid, so it can read ~/openrouter.key (mode 0600) as you do.
docker build -t idots-linux-host --build-arg HOST_UID="$(id -u)" --build-arg HOST_GID="$(id -g)" tests/e2e/linux-host
+55
View File
@@ -0,0 +1,55 @@
/**
* dot-agentd has one build (the image builder's agentdBuildCommand, AGENTD_BUILD_SHELL as a shell line) and one Go
* (go.mod's `toolchain`): the same source then gives the same bytes, so the same runtime disk, wherever it is built.
* These checks keep every place that builds it or tells a person how from drifting to another spelling or another Go.
*/
import { readFileSync } from "node:fs";
import { join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { AGENTD_BUILD_SHELL } from "@invisible-dots/image-builder";
import { describe, expect, it } from "vitest";
const repo = resolve(fileURLToPath(new URL(".", import.meta.url)), "../..");
const text = (path: string) => readFileSync(join(repo, path), "utf8").replace(/\r\n/g, "\n");
/** Every file that builds dot-agentd, or tells how. */
const BUILDERS = [
".github/workflows/tests.yml",
".github/workflows/golden-image.yml",
"guest/dot-agentd/README.md",
"guest/image-builder/README.md",
"guest/image-builder/test/smoke/run.sh",
"tests/e2e/README.md",
];
/** Those that run it with the environment on the same line, so quote the whole shell line. */
const WHOLE_LINE = [".github/workflows/tests.yml", ".github/workflows/golden-image.yml", "guest/dot-agentd/README.md", "guest/image-builder/README.md", "tests/e2e/README.md"];
/** The flags, as a line that passes the environment another way (docker's -e) still spells them. */
const FLAGS = AGENTD_BUILD_SHELL.slice(AGENTD_BUILD_SHELL.indexOf("go build"), AGENTD_BUILD_SHELL.indexOf(" -o ") + 4);
describe("the one build of dot-agentd", () => {
const toolchain = /^toolchain go(\d+\.\d+\.\d+)$/m.exec(text("guest/dot-agentd/go.mod"))?.[1];
it("go.mod names the Go that builds it", () => {
expect(toolchain).toMatch(/^\d+\.\d+\.\d+$/);
});
it.each(BUILDERS)("%s builds it with the same flags, and nothing else that builds it", (path) => {
const builds = text(path)
.split("\n")
.filter((line) => /\bgo (-C \S+ )?build\b/.test(line));
expect(builds.length, path).toBeGreaterThan(0);
for (const line of builds) expect(line, path).toContain(FLAGS);
});
it.each(WHOLE_LINE)("%s quotes the whole shell line", (path) => {
expect(text(path)).toContain(AGENTD_BUILD_SHELL);
});
it.each(BUILDERS)("%s uses the Go of go.mod's toolchain", (path) => {
const content = text(path);
for (const image of content.match(/golang:[\w.-]+/g) ?? []) expect(image, path).toBe(`golang:${toolchain}`);
// setup-go takes the toolchain line from go.mod (v6 reads `toolchain`, v5 read only the `go` line).
for (const use of content.match(/actions\/setup-go@\S+/g) ?? []) expect(use, path).toBe("actions/setup-go@v6");
for (const pin of content.match(/go-version(-file)?:.*/g) ?? []) expect(pin, path).toBe("go-version-file: guest/dot-agentd/go.mod");
});
});
+73
View File
@@ -0,0 +1,73 @@
/**
* The host-to-guest contract of architecture section 5 is written down once, in packages/shared (protocol.ts), and
* held twice more in the guest: by dot-agentd's Go sources (its routes and its defaults) and by the engine's Python
* (the sockets and the workspace it reaches when its unit names none). Neither copy imports the TypeScript, so these
* checks read them and compare: a route or a path changed on one side only fails here, not inside every Dot.
*/
import { readFileSync } from "node:fs";
import { join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { AGENTD_ROUTES, EXEC_OUTPUT_CAP_BYTES, GUEST_DISPLAY, GUEST_PATHS, GUEST_PORT, GUEST_PROOF_CONTEXT } from "@invisible-dots/shared";
import { describe, expect, it } from "vitest";
const repo = resolve(fileURLToPath(new URL(".", import.meta.url)), "../..");
const read = (path: string) => readFileSync(join(repo, path), "utf8").replace(/\r\n/g, "\n");
const agentd = (file: string) => read(join("guest/dot-agentd/internal/agentd", file));
/** The value of a Go constant: a string literal, a sum of literals and constants of the same sources, or `a << b`. */
function goConstant(sources: string, name: string): string | number {
const match = new RegExp(`^\\s*(?:const\\s+)?${name}\\s*=\\s*(.+?)\\s*(//.*)?$`, "m").exec(sources);
if (!match) throw new Error(`no Go constant ${name}`);
const expression = match[1]!;
const shift = /^(\d+)\s*<<\s*(\d+)$/.exec(expression);
if (shift) return Number(shift[1]) * 2 ** Number(shift[2]);
return expression
.split("+")
.map((term) => term.trim())
.map((term) => (term.startsWith('"') ? (JSON.parse(term) as string) : String(goConstant(sources, term))))
.join("");
}
/** The value of a top-level Python string constant. */
function pyConstant(source: string, name: string): string | undefined {
return new RegExp(`^${name} = "([^"]*)"$`, "m").exec(source)?.[1];
}
describe("dot-agentd's Go copy of the protocol", () => {
const go = ["server.go", "exec.go", "proof.go"].map(agentd).join("\n");
it("serves exactly the routes of AGENTD_ROUTES, plus the engine's process route on its socket", () => {
const served = new Set([...go.matchAll(/mux\.Handle(?:Func)?\("(?:[A-Z]+ )?(\/[^"]*)"/g)].map((m) => m[1]!));
served.delete("/");
// The agent proxy is registered by its prefix constant, with and without the trailing slash.
for (const handle of go.matchAll(/mux\.Handle\(agentProxyPrefix(\+"\/")?,/g)) served.add(`${goConstant(go, "agentProxyPrefix")}${handle[1] ? "/" : ""}`);
expect([...served].sort()).toEqual([...Object.values(AGENTD_ROUTES), `${AGENTD_ROUTES.agent}/`, "/v1/proc"].sort());
});
it("listens on GUEST_PORT, caps exec output at EXEC_OUTPUT_CAP_BYTES, and signs the proof with GUEST_PROOF_CONTEXT", () => {
expect(goConstant(go, "DefaultListenAddr")).toBe(`0.0.0.0:${GUEST_PORT}`);
expect(goConstant(go, "ExecOutputCap")).toBe(EXEC_OUTPUT_CAP_BYTES);
expect(goConstant(go, "proofContext")).toBe(GUEST_PROOF_CONTEXT);
});
it("defaults to the guest paths and the display of GUEST_PATHS and GUEST_DISPLAY", () => {
expect(goConstant(go, "DefaultConfigPath")).toBe(GUEST_PATHS.config);
expect(goConstant(go, "DefaultHome")).toBe(GUEST_PATHS.home);
expect(goConstant(go, "DefaultRunDir")).toBe(GUEST_PATHS.runDir);
expect(goConstant(go, "DefaultAgentdSocket")).toBe(GUEST_PATHS.agentdSocket);
expect(goConstant(go, "DefaultAgentSocket")).toBe(GUEST_PATHS.agentSocket);
expect(goConstant(go, "DefaultDisplay")).toBe(GUEST_DISPLAY);
});
});
describe("the engine's defaults", () => {
const computer = read("invisible_engine_dots/nanobot/dots/computer.py");
const main = read("invisible_engine_dots/nanobot/dots/main.py");
it("are the guest paths of GUEST_PATHS", () => {
expect(pyConstant(computer, "DEFAULT_AGENTD_SOCKET")).toBe(GUEST_PATHS.agentdSocket);
expect(pyConstant(computer, "DEFAULT_WORKSPACE")).toBe(GUEST_PATHS.workspace);
expect(pyConstant(computer, "DEFAULT_AGENTD_BIN")).toBe(`${GUEST_PATHS.runtime}/bin/dot-agentd`);
expect(pyConstant(main, "DEFAULT_AGENT_SOCKET")).toBe(GUEST_PATHS.agentSocket);
});
});
+1 -1
View File
@@ -79,7 +79,7 @@ const ALLOWED: Record<string, { owner: string; lines: string[] }> = {
"guest/image-builder/src/runtime.ts": {
owner:
"agentdBuildCommand(): the guest's platform, the same on both hosts (not a branch): dot-agentd is built for linux/amd64 wherever the host runs",
lines: ['env: { CGO_ENABLED: "0", GOOS: "linux", GOARCH: "amd64" },'],
lines: ['const AGENTD_BUILD_ENV = { CGO_ENABLED: "0", GOOS: "linux", GOARCH: "amd64" } as const;'],
},
"packages/shared/src/files.ts": {
owner: "permissionBitsEnforced() and restrictToOwner(): a private file is chmod 0600 on Linux and an owner-only ACL on Windows",
@@ -56,7 +56,7 @@ write_files:
mountpoint -q /opt/invisible-dots || mount /opt/invisible-dots
exec /opt/invisible-dots/install.sh
# The runtime ISO is the second CD-ROM drive (section 3.4). It is found by its
# volume label, written by packages/iso, so the device name does not matter.
# The runtime ISO is the third read-only virtio disk, after the seed (section 3.4). It is
# found by its volume label, written by packages/iso, so the device name does not matter.
mounts:
- [ {{runtimeDevice}}, /opt/invisible-dots, iso9660, "ro,nofail,x-systemd.device-timeout=30s", "0", "0" ]
+3 -1
View File
@@ -16,7 +16,9 @@ export default defineConfig({
// "@vitest-environment jsdom" line so that no other test pays for a DOM.
// invisible_engine_dots/ holds the nanobot fork: Python, with its own pytest suite
// (the engine job of the CI workflow); nothing of it runs here.
exclude: ["**/node_modules/**", "**/.next/**", "invisible_engine_dots/**"],
// A `*.linux.test.ts` drives a Linux program of the guest itself (dot-agentd, with its bash and its unix
// sockets): it is collected on Linux only, so on another host it is neither run nor counted as skipped.
exclude: ["**/node_modules/**", "**/.next/**", "invisible_engine_dots/**", ...(process.platform === "linux" ? [] : ["**/*.linux.test.ts"])],
environment: "node",
// Setup hooks boot an embedded PostgreSQL (PGlite, WebAssembly) and run
// the migrations, or start servers and child processes. On a loaded