mirror of
https://github.com/feder-cr/Jobs_Applier_AI_Agent.git
synced 2026-11-07 10:36:05 -10:00
Merge branch 'guest-fixes' into fixes-batch
This commit is contained in:
commit
ac26cec1bf
45 files changed
+950
-287
No files matched your search
@@ -37,7 +37,7 @@ jobs:
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: npm
|
||||
- uses: actions/setup-go@v5
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: guest/dot-agentd/go.mod
|
||||
- name: KVM for the builder VM, and QEMU
|
||||
@@ -50,7 +50,8 @@ jobs:
|
||||
- run: npm ci --no-audit --no-fund
|
||||
- run: npm run build --workspace @invisible-dots/cli
|
||||
# The runtime ISO, which `image build` makes first, carries dot-agentd.
|
||||
- run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go -C guest/dot-agentd build -trimpath -o bin/dot-agentd ./cmd/dot-agentd
|
||||
- working-directory: guest/dot-agentd
|
||||
run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false -ldflags="-s -w" -o bin/dot-agentd ./cmd/dot-agentd
|
||||
- name: the image (downloaded instead when its release exists already)
|
||||
id: build
|
||||
run: |
|
||||
|
||||
@@ -39,6 +39,14 @@ jobs:
|
||||
with:
|
||||
node-version: "24"
|
||||
cache: npm
|
||||
# Linux's vitest builds the real dot-agentd and drives it with the host's GuestClient
|
||||
# (apps/vm-manager/test/agentd.linux.test.ts, which vitest.config.ts collects on Linux only).
|
||||
- if: runner.os == 'Linux'
|
||||
uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: guest/dot-agentd/go.mod
|
||||
# The module has no dependencies, so there is no go.sum to key a cache on.
|
||||
cache: false
|
||||
- run: npm ci --no-audit --no-fund
|
||||
- run: npm run typecheck
|
||||
- name: the pre-push hook parses
|
||||
@@ -169,7 +177,7 @@ jobs:
|
||||
working-directory: guest/dot-agentd
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-go@v5
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version-file: guest/dot-agentd/go.mod
|
||||
# The module has no dependencies, so there is no go.sum to key a cache on.
|
||||
@@ -186,7 +194,7 @@ jobs:
|
||||
- name: refuse a run with too few tests or an unexpected skip
|
||||
if: always()
|
||||
run: node ../../.github/scripts/test-guard.mjs go-test.json --suite go --floors ../../.github/test-floors.json
|
||||
- run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o bin/dot-agentd ./cmd/dot-agentd
|
||||
- run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false -ldflags="-s -w" -o bin/dot-agentd ./cmd/dot-agentd
|
||||
- run: file bin/dot-agentd | tee /dev/stderr | grep -q "ELF 64-bit LSB executable, x86-64.*statically linked"
|
||||
|
||||
engine:
|
||||
@@ -217,7 +225,7 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
# Builds dot-agentd in golang:1.26, the engine's environment in ubuntu:24.04
|
||||
# Builds dot-agentd in golang:1.26.8, the engine's environment in ubuntu:24.04
|
||||
# the way provision.sh builds it, and runs the Dot. It exits non-zero on a
|
||||
# failed or a skipped check and prints "SMOKE: N passed, F failed, S skipped".
|
||||
- name: run the smoke on this checkout
|
||||
|
||||
@@ -79,6 +79,7 @@ export * from "./host.js";
|
||||
export * from "./logger.js";
|
||||
export * from "./ports.js";
|
||||
export * from "./powershell.js";
|
||||
export * from "./process-file.js";
|
||||
export * from "./qemu-args.js";
|
||||
export * from "./runner.js";
|
||||
export * from "./seed.js";
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
/**
|
||||
* The pid file of a Dot's VM (architecture section 3.2): what lets a control plane that restarted find the QEMU it
|
||||
* left running. The VmManager decides from it whether that QEMU still runs and may be killed (`processIsOurs`).
|
||||
*/
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import { replaceFile } from "@invisible-dots/shared";
|
||||
|
||||
/**
|
||||
* The QEMU process of a Dot, the guest port it forwards, and the host's uptime when it was spawned. A host uptime
|
||||
* lower than that one means the host restarted since, so the process is gone whatever now runs under that pid.
|
||||
*/
|
||||
export interface ProcessFile {
|
||||
pid: number;
|
||||
guest_port: number;
|
||||
host_uptime_s?: number;
|
||||
}
|
||||
|
||||
export function parseProcessFile(text: string): ProcessFile | undefined {
|
||||
try {
|
||||
const value = JSON.parse(text) as Partial<ProcessFile>;
|
||||
const pid = value.pid;
|
||||
const port = value.guest_port;
|
||||
if (Number.isSafeInteger(pid) && pid! > 0 && Number.isInteger(port) && port! >= 1 && port! <= 65535) {
|
||||
const record: ProcessFile = { pid: pid!, guest_port: port! };
|
||||
if (typeof value.host_uptime_s === "number" && Number.isFinite(value.host_uptime_s)) record.host_uptime_s = value.host_uptime_s;
|
||||
return record;
|
||||
}
|
||||
} catch {
|
||||
// Falls through: not JSON is as unreadable as JSON of the wrong shape.
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** The pid file at `path`: undefined when there is none, "unreadable" when it is not one this code wrote. */
|
||||
export async function readProcessFile(path: string): Promise<ProcessFile | "unreadable" | undefined> {
|
||||
let text: string;
|
||||
try {
|
||||
text = await readFile(path, "utf8");
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined;
|
||||
throw error;
|
||||
}
|
||||
return parseProcessFile(text) ?? "unreadable";
|
||||
}
|
||||
|
||||
/** Written whole or not at all (a temporary file renamed over it), so a reader never sees half a pid file. */
|
||||
export async function writeProcessFile(path: string, record: ProcessFile): Promise<void> {
|
||||
const temporary = `${path}.tmp`;
|
||||
await writeFile(temporary, `${JSON.stringify(record)}\n`, { mode: 0o600 });
|
||||
await replaceFile(temporary, path);
|
||||
}
|
||||
@@ -3,7 +3,7 @@
|
||||
* one function that builds it, on every host; only the accelerator differs,
|
||||
* and it arrives here as a value from host.ts.
|
||||
*/
|
||||
import { GUEST_PORT, vmName } from "@invisible-dots/shared";
|
||||
import { CONFIG_BOUNDS, GUEST_PORT, parseSizeMiB, vmName } from "@invisible-dots/shared";
|
||||
import { VmManagerError } from "./errors.js";
|
||||
import type { Accelerator } from "./host.js";
|
||||
|
||||
@@ -57,6 +57,21 @@ function integerInRange(label: string, value: number, min: number, max: number):
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* The cpus and memory every VM of this project may have, Dots and the image builder's VM alike: the range a Dot's
|
||||
* config allows (CONFIG_BOUNDS of packages/shared, which owns it), in the units QEMU takes.
|
||||
*/
|
||||
export const VM_RESOURCE_BOUNDS = {
|
||||
cpus: { min: CONFIG_BOUNDS.cpu.min, max: CONFIG_BOUNDS.cpu.max },
|
||||
memoryMiB: { min: parseSizeMiB(CONFIG_BOUNDS.memory.min), max: parseSizeMiB(CONFIG_BOUNDS.memory.max) },
|
||||
} as const;
|
||||
|
||||
/** Refuses cpus or memory outside VM_RESOURCE_BOUNDS, naming the value. */
|
||||
export function checkVmResources(spec: { cpus: number; memoryMiB: number }): void {
|
||||
integerInRange("cpus", spec.cpus, VM_RESOURCE_BOUNDS.cpus.min, VM_RESOURCE_BOUNDS.cpus.max);
|
||||
integerInRange("memoryMiB", spec.memoryMiB, VM_RESOURCE_BOUNDS.memoryMiB.min, VM_RESOURCE_BOUNDS.memoryMiB.max);
|
||||
}
|
||||
|
||||
/**
|
||||
* The CPU model of every VM of this project (section 3.4): the host's own
|
||||
* CPU, without the virtualization extensions. A Dot never runs a hypervisor,
|
||||
@@ -79,6 +94,7 @@ export const CPU_MODEL = "host,-vmx,-svm";
|
||||
* changes both.
|
||||
*/
|
||||
export function machineArgs(spec: { accelerator: Accelerator; cpus: number; memoryMiB: number }): string[] {
|
||||
checkVmResources(spec);
|
||||
return [
|
||||
"-machine",
|
||||
"q35",
|
||||
@@ -87,9 +103,9 @@ export function machineArgs(spec: { accelerator: Accelerator; cpus: number; memo
|
||||
"-cpu",
|
||||
CPU_MODEL,
|
||||
"-smp",
|
||||
String(integerInRange("cpus", spec.cpus, 1, 16)),
|
||||
String(spec.cpus),
|
||||
"-m",
|
||||
String(integerInRange("memoryMiB", spec.memoryMiB, 256, 64 * 1024)),
|
||||
String(spec.memoryMiB),
|
||||
];
|
||||
}
|
||||
|
||||
|
||||
@@ -42,7 +42,8 @@ import { GuestClient, waitForGuestHealth, type GuestClientOptions, type WaitForG
|
||||
import { accelerator as hostAccelerator, findQemu, type Accelerator, type QemuInstallation } from "./host.js";
|
||||
import { stderrLogger, type Logger } from "./logger.js";
|
||||
import { pickFreePort, portListens } from "./ports.js";
|
||||
import { CPU_MODEL, qemuArgs, qemuPathArg } from "./qemu-args.js";
|
||||
import { readProcessFile, writeProcessFile, type ProcessFile } from "./process-file.js";
|
||||
import { checkVmResources, CPU_MODEL, qemuArgs, qemuPathArg } from "./qemu-args.js";
|
||||
import {
|
||||
fileSize,
|
||||
NodeCommandRunner,
|
||||
@@ -182,12 +183,6 @@ function assertDotId(dotId: string): void {
|
||||
}
|
||||
}
|
||||
|
||||
function assertIntInRange(label: string, value: number, min: number, max: number): void {
|
||||
if (!Number.isInteger(value) || value < min || value > max) {
|
||||
throw new VmManagerError(`${label} must be an integer from ${min} to ${max}, got ${value}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function exists(path: string): Promise<boolean> {
|
||||
try {
|
||||
await access(path);
|
||||
@@ -218,18 +213,6 @@ const CPU_MODEL_FAILURES = [/CPU model 'host' requires/i, /unable to find CPU mo
|
||||
/** How much of the end of a log goes into an error: enough for QEMU's last words and the guest's last boot lines. */
|
||||
const LOG_TAIL_BYTES = 4 * 1024;
|
||||
|
||||
/**
|
||||
* The pid file (section 3.2): the QEMU process of a Dot, the guest port it
|
||||
* forwards, and the host's uptime when it was spawned. A host uptime lower
|
||||
* than that one means the host restarted since, so the process is gone
|
||||
* whatever now runs under that pid.
|
||||
*/
|
||||
export interface ProcessFile {
|
||||
pid: number;
|
||||
guest_port: number;
|
||||
host_uptime_s?: number;
|
||||
}
|
||||
|
||||
/** seed-instance.json, or undefined for one this code did not write: the next seed then takes the content's own id. */
|
||||
function parsePreviousSeed(text: string): PreviousSeed | undefined {
|
||||
try {
|
||||
@@ -241,22 +224,6 @@ function parsePreviousSeed(text: string): PreviousSeed | undefined {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function parseProcessFile(text: string): ProcessFile | undefined {
|
||||
try {
|
||||
const value = JSON.parse(text) as Partial<ProcessFile>;
|
||||
const pid = value.pid;
|
||||
const port = value.guest_port;
|
||||
if (Number.isSafeInteger(pid) && pid! > 0 && Number.isInteger(port) && port! >= 1 && port! <= 65535) {
|
||||
const record: ProcessFile = { pid: pid!, guest_port: port! };
|
||||
if (typeof value.host_uptime_s === "number" && Number.isFinite(value.host_uptime_s)) record.host_uptime_s = value.host_uptime_s;
|
||||
return record;
|
||||
}
|
||||
} catch {
|
||||
// Falls through: not JSON is as unreadable as JSON of the wrong shape.
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** A QEMU this process spawned: its exit is set the moment Node reports it. */
|
||||
interface OwnedProcess {
|
||||
pid: number;
|
||||
@@ -389,8 +356,7 @@ export class VmManager {
|
||||
|
||||
private validateSpec(spec: VmSpec): void {
|
||||
assertDotId(spec.dotId);
|
||||
assertIntInRange("cpus", spec.cpus, 1, 16);
|
||||
assertIntInRange("memoryMiB", spec.memoryMiB, 256, 64 * 1024);
|
||||
checkVmResources(spec);
|
||||
if (!Number.isSafeInteger(spec.diskBytes) || spec.diskBytes <= 0) {
|
||||
throw new VmManagerError(`diskBytes must be a positive integer, got ${spec.diskBytes}`);
|
||||
}
|
||||
@@ -400,6 +366,16 @@ export class VmManager {
|
||||
}
|
||||
}
|
||||
|
||||
/** Both images of the spec exist: a VM is never created or started on a path that leads nowhere. */
|
||||
private async assertImages(spec: VmSpec): Promise<void> {
|
||||
for (const [label, path] of [
|
||||
["golden image", spec.goldenImage],
|
||||
["runtime ISO", spec.runtimeImage],
|
||||
] as const) {
|
||||
if (!(await exists(path))) throw new VmManagerError(`${label} ${path} does not exist: build it with "invisible-dots image build" first`);
|
||||
}
|
||||
}
|
||||
|
||||
private async qemuImg(args: string[]) {
|
||||
const { img } = await this.qemu();
|
||||
return this.runner.run(img, args, { timeoutMs: this.options.qemuImgTimeoutMs ?? 300_000 });
|
||||
@@ -426,12 +402,7 @@ export class VmManager {
|
||||
async create(spec: VmSpec): Promise<CreateVmResult> {
|
||||
this.validateSpec(spec);
|
||||
return this.withLock(spec.dotId, async () => {
|
||||
for (const [label, path] of [
|
||||
["golden image", spec.goldenImage],
|
||||
["runtime ISO", spec.runtimeImage],
|
||||
] as const) {
|
||||
if (!(await exists(path))) throw new VmManagerError(`${label} ${path} does not exist: build it with "invisible-dots image build" first`);
|
||||
}
|
||||
await this.assertImages(spec);
|
||||
const current = await this.probe(spec.dotId);
|
||||
if (current.state !== "STOPPED") {
|
||||
throw new VmStateError(spec.dotId, `the VM is ${current.state}; stop it before creating its disk and seed again`);
|
||||
@@ -452,26 +423,6 @@ export class VmManager {
|
||||
});
|
||||
}
|
||||
|
||||
/** The pid file: undefined when there is none, "unreadable" when it is not one this code wrote. */
|
||||
private async readProcessFile(dotId: string): Promise<ProcessFile | "unreadable" | undefined> {
|
||||
let text: string;
|
||||
try {
|
||||
text = await readFile(this.paths.processFilePath(dotId), "utf8");
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined;
|
||||
throw error;
|
||||
}
|
||||
return parseProcessFile(text) ?? "unreadable";
|
||||
}
|
||||
|
||||
/** Written whole or not at all (a temporary file renamed over it), so a reader never sees half a pid file. */
|
||||
private async writeProcessFile(dotId: string, record: ProcessFile): Promise<void> {
|
||||
const path = this.paths.processFilePath(dotId);
|
||||
const temporary = `${path}.tmp`;
|
||||
await writeFile(temporary, `${JSON.stringify(record)}\n`, { mode: 0o600 });
|
||||
await replaceFile(temporary, path);
|
||||
}
|
||||
|
||||
/** Forget a QEMU that is gone: its pid file, and its ChildProcess if this process held it. */
|
||||
private async forget(dotId: string): Promise<void> {
|
||||
this.owned.delete(dotId);
|
||||
@@ -492,7 +443,7 @@ export class VmManager {
|
||||
}
|
||||
|
||||
private async qemuRuns(dotId: string): Promise<boolean> {
|
||||
const record = await this.readProcessFile(dotId);
|
||||
const record = await readProcessFile(this.paths.processFilePath(dotId));
|
||||
return record !== undefined && record !== "unreadable" && this.runs(dotId, record);
|
||||
}
|
||||
|
||||
@@ -517,7 +468,7 @@ export class VmManager {
|
||||
|
||||
/** Read a VM's state from its pid file and its process (section 3.4). */
|
||||
private async probe(dotId: string): Promise<Probe> {
|
||||
const record = await this.readProcessFile(dotId);
|
||||
const record = await readProcessFile(this.paths.processFilePath(dotId));
|
||||
if (record === undefined) return STOPPED;
|
||||
const file = this.paths.processFilePath(dotId);
|
||||
if (record === "unreadable") {
|
||||
@@ -590,12 +541,7 @@ export class VmManager {
|
||||
await this.forget(dotId);
|
||||
const diskPath = this.paths.diskPath(dotId);
|
||||
if (!(await exists(diskPath))) throw new VmStateError(dotId, `${diskPath} does not exist; create the VM first`);
|
||||
for (const [label, path] of [
|
||||
["golden image", spec.goldenImage],
|
||||
["runtime ISO", spec.runtimeImage],
|
||||
] as const) {
|
||||
if (!(await exists(path))) throw new VmManagerError(`${label} ${path} does not exist: build it with "invisible-dots image build" first`);
|
||||
}
|
||||
await this.assertImages(spec);
|
||||
await this.writeSeed(spec);
|
||||
const qemu = await this.qemu();
|
||||
const accel = this.accelerator;
|
||||
@@ -641,7 +587,7 @@ export class VmManager {
|
||||
const record: ProcessFile = { pid: child.pid, guest_port: guestPort, host_uptime_s: hostUptimeSeconds() };
|
||||
try {
|
||||
// Before anything can fail: from here on a restarted control plane finds this QEMU.
|
||||
await this.writeProcessFile(dotId, record);
|
||||
await writeProcessFile(this.paths.processFilePath(dotId), record);
|
||||
} catch (error) {
|
||||
await this.killOurs(dotId, record);
|
||||
await this.forget(dotId);
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
/**
|
||||
* The host's GuestClient against the real dot-agentd, not a stand-in: the daemon built with the image builder's one
|
||||
* build, run as its README's development mode describes (`--run-as=`: the model's work as the daemon's own user;
|
||||
* `--listen 127.0.0.1:0`), with a guest config holding the Dot's token, and a stand-in for the engine on the unix
|
||||
* socket the daemon proxies `/v1/agent` to. What the two sides each test alone with a fake of the other, this tests
|
||||
* across the seam: the proof, the token, the routes, the bodies, and an event stream passed on as it is written.
|
||||
*
|
||||
* Linux only (vitest.config.ts leaves `*.linux.test.ts` out elsewhere): the daemon is a Linux program, its exec route
|
||||
* runs /bin/bash and the engine's socket is a unix socket. It needs Go, which CI's node job sets up from go.mod.
|
||||
*/
|
||||
import { execFile, spawn, type ChildProcess } from "node:child_process";
|
||||
import { mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises";
|
||||
import { createServer, get, type Server, type ServerResponse } from "node:http";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { AGENTD_ROUTES, GUEST_UNPROVEN, type OutboundEvent } from "@invisible-dots/shared";
|
||||
import { agentdBuildCommand } from "@invisible-dots/image-builder";
|
||||
import { afterAll, beforeAll, describe, expect, it } from "vitest";
|
||||
import { GuestClient, GuestRequestError } from "../src/index.js";
|
||||
|
||||
const REPO = resolve(fileURLToPath(new URL(".", import.meta.url)), "../../..");
|
||||
const TOKEN = "integration-token-0123456789";
|
||||
|
||||
let dir: string;
|
||||
let home: string;
|
||||
let daemon: ChildProcess | undefined;
|
||||
let agent: Server | undefined;
|
||||
let port: number;
|
||||
/** What reached the stand-in engine through the daemon's proxy, as `METHOD /path?query`. */
|
||||
const agentSeen: string[] = [];
|
||||
/** The open event streams the stand-in holds, to write the next event into. */
|
||||
const streams: ServerResponse[] = [];
|
||||
|
||||
function event(seq: number): OutboundEvent {
|
||||
return { seq, id: `evt_${seq}`, type: "automation.next_run", ts: "2026-10-11T10:00:00.000Z", data: { next_run_at_ms: seq } };
|
||||
}
|
||||
|
||||
function sse(res: ServerResponse, seq: number): void {
|
||||
res.write(`id: ${seq}\ndata: ${JSON.stringify(event(seq))}\n\n`);
|
||||
}
|
||||
|
||||
/** The engine's side of the proxy: /health, and /events/stream, which sends one event and keeps the stream open. */
|
||||
function startAgent(socket: string): Promise<Server> {
|
||||
const server = createServer((req, res) => {
|
||||
const url = new URL(req.url!, "http://agent");
|
||||
agentSeen.push(`${req.method} ${url.pathname}${url.search}`);
|
||||
if (url.pathname === "/health") {
|
||||
res.writeHead(200, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify({ status: "ok", state: "idle", openrouter_configured: true, browser: { identities: 0, open: 0 }, checks: {} }));
|
||||
return;
|
||||
}
|
||||
if (url.pathname === "/events/stream") {
|
||||
res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" });
|
||||
sse(res, Number(url.searchParams.get("after") ?? 0) + 1);
|
||||
streams.push(res);
|
||||
return;
|
||||
}
|
||||
res.writeHead(404, { "content-type": "application/json" }).end(JSON.stringify({ error: "not_found" }));
|
||||
});
|
||||
return new Promise((done, fail) => {
|
||||
server.once("error", fail);
|
||||
server.listen(socket, () => done(server));
|
||||
});
|
||||
}
|
||||
|
||||
/** Starts the daemon and answers the TCP port it listens on, from its log line `listening on="tcp 127.0.0.1:<port>"`. */
|
||||
function startDaemon(binary: string, args: string[]): Promise<{ child: ChildProcess; port: number }> {
|
||||
const child = spawn(binary, args, { stdio: ["ignore", "ignore", "pipe"] });
|
||||
return new Promise((done, fail) => {
|
||||
let log = "";
|
||||
child.stderr!.setEncoding("utf8");
|
||||
child.stderr!.on("data", (chunk: string) => {
|
||||
log += chunk;
|
||||
const listening = /listening on="tcp 127\.0\.0\.1:(\d+)"/.exec(log);
|
||||
if (listening) done({ child, port: Number(listening[1]) });
|
||||
});
|
||||
child.once("error", fail);
|
||||
child.once("exit", (code, signal) => fail(new Error(`dot-agentd exited (${code ?? signal}) before it listened:\n${log}`)));
|
||||
});
|
||||
}
|
||||
|
||||
beforeAll(async () => {
|
||||
dir = await mkdtemp(join(tmpdir(), "idots-agentd-"));
|
||||
home = join(dir, "home");
|
||||
await mkdir(home);
|
||||
const binary = join(dir, "dot-agentd");
|
||||
const build = agentdBuildCommand(REPO, binary);
|
||||
await promisify(execFile)(build.command, build.args, { env: { ...process.env, ...build.env } });
|
||||
const config = join(dir, "config.json");
|
||||
await writeFile(config, JSON.stringify({ dotId: "dot_integration", token: TOKEN }), { mode: 0o600 });
|
||||
const agentSocket = join(dir, "agent.sock");
|
||||
agent = await startAgent(agentSocket);
|
||||
const started = await startDaemon(binary, [
|
||||
"--config", config,
|
||||
"--home", home,
|
||||
"--run-dir", join(dir, "run"),
|
||||
"--agent-socket", agentSocket,
|
||||
"--listen", "127.0.0.1:0",
|
||||
"--run-as=",
|
||||
]);
|
||||
daemon = started.child;
|
||||
port = started.port;
|
||||
// The build: a cold Go cache compiles the standard library for linux/amd64 first, and a host whose Go is older
|
||||
// than go.mod's toolchain downloads that one.
|
||||
}, 600_000);
|
||||
|
||||
afterAll(async () => {
|
||||
for (const res of streams) res.destroy();
|
||||
if (daemon && daemon.exitCode === null && daemon.signalCode === null) {
|
||||
const exited = new Promise((done) => daemon!.once("exit", done));
|
||||
daemon.kill("SIGTERM");
|
||||
await exited;
|
||||
}
|
||||
await new Promise<void>((done) => (agent ? agent.close(() => done()) : done()));
|
||||
if (dir) await rm(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe("GuestClient against dot-agentd", () => {
|
||||
it("is given the proof of the token before it sends the token", async () => {
|
||||
await expect(new GuestClient(port, TOKEN).prove()).resolves.toBeUndefined();
|
||||
const impostor = new GuestClient(port, "another-dot-token");
|
||||
await expect(impostor.health()).rejects.toMatchObject({ code: GUEST_UNPROVEN });
|
||||
});
|
||||
|
||||
it("is refused without the token: 401 with the Bearer challenge", async () => {
|
||||
const answer = await new Promise<{ status: number; challenge: string | undefined; body: string }>((done, fail) => {
|
||||
get({ host: "127.0.0.1", port, path: AGENTD_ROUTES.health }, (res) => {
|
||||
let body = "";
|
||||
res.setEncoding("utf8");
|
||||
res.on("data", (chunk: string) => (body += chunk));
|
||||
res.on("end", () => done({ status: res.statusCode ?? 0, challenge: res.headers["www-authenticate"], body }));
|
||||
}).once("error", fail);
|
||||
});
|
||||
expect(answer.status).toBe(401);
|
||||
expect(answer.challenge).toBe('Bearer realm="dot-agentd"');
|
||||
expect(JSON.parse(answer.body)).toMatchObject({ error: "unauthorized" });
|
||||
});
|
||||
|
||||
it("reads health with the engine's own answer passed through", async () => {
|
||||
const health = await new GuestClient(port, TOKEN).health();
|
||||
expect(health).toMatchObject({ agentd: "ok", agent: { status: "ok", state: "idle", openrouter_configured: true } });
|
||||
expect(agentSeen).toContain("GET /health");
|
||||
});
|
||||
|
||||
it("runs a command in home and reads its exit code and both streams", async () => {
|
||||
const client = new GuestClient(port, TOKEN);
|
||||
expect(await client.exec({ command: "printf out; printf err >&2; exit 3" })).toMatchObject({ exit_code: 3, stdout: "out", stderr: "err", timed_out: false });
|
||||
expect((await client.exec({ command: "pwd" })).stdout.trim()).toBe(await realpath(home));
|
||||
});
|
||||
|
||||
it("writes, reads and lists a file under home, and refuses one outside it", async () => {
|
||||
const client = new GuestClient(port, TOKEN);
|
||||
await client.writeFile("notes/hello.txt", "hello from the host");
|
||||
expect((await client.readFile("~/notes/hello.txt")).toString("utf8")).toBe("hello from the host");
|
||||
const listing = await client.listFiles("notes");
|
||||
expect(listing.entries).toEqual([expect.objectContaining({ name: "hello.txt", type: "file", size: 19 })]);
|
||||
const outside = await client.readFile("/etc/hostname").catch((error: unknown) => error);
|
||||
expect(outside).toBeInstanceOf(GuestRequestError);
|
||||
expect(outside).toMatchObject({ status: 403, code: "outside_home" });
|
||||
});
|
||||
|
||||
it("proxies the engine's routes, and its event stream as it is written", async () => {
|
||||
const client = new GuestClient(port, TOKEN);
|
||||
expect(await client.agentHealth()).toMatchObject({ status: "ok", state: "idle" });
|
||||
const controller = new AbortController();
|
||||
const got: number[] = [];
|
||||
for await (const evt of client.events({ after: 4, signal: controller.signal })) {
|
||||
got.push(evt.seq);
|
||||
// The first event arrived while the engine still holds the stream open: the proxy did not wait for its end.
|
||||
// The next one is written only now.
|
||||
if (got.length === 1) sse(streams.at(-1)!, 6);
|
||||
else controller.abort();
|
||||
}
|
||||
expect(got).toEqual([5, 6]);
|
||||
expect(agentSeen).toContain("GET /events/stream?after=4");
|
||||
});
|
||||
});
|
||||
+15
-9
@@ -282,21 +282,27 @@ QEMU on the same disk would corrupt it.
|
||||
`invisible-playwright-mcp`, transitive packages included, by
|
||||
`guest/image-builder/builder/mcp-requirements.lock`, every package at an
|
||||
exact version with the SHA-256 of its files. The builder installs it with
|
||||
`uv pip install --require-hashes`, so nothing is resolved from the index at
|
||||
build time; the lock is the one place the two top-level versions are
|
||||
`uv pip install --require-hashes --only-binary :all:`, so nothing is
|
||||
resolved from the index at build time and no package's build script runs;
|
||||
the lock is the one place the two top-level versions are
|
||||
written, and it is part of the inputs digest, so a changed transitive
|
||||
package is a new image. apt packages are not pinned by version; apt checks
|
||||
their signatures.
|
||||
package is a new image. The apt packages come from the Ubuntu archive as it
|
||||
was at the moment `apt_snapshot` of `pins.json` names (snapshot.ubuntu.com,
|
||||
`apt-get --snapshot`), so they and their dependencies are the same on any day
|
||||
of the build; apt checks their signatures.
|
||||
- The **runtime disk** (`runtime-<version>.iso`, attached read-only to every VM
|
||||
and mounted at `/opt/invisible-dots`) carries our code: the engine's source,
|
||||
the `dot-agentd` binary and the systemd units. A new
|
||||
version of our code is a new ISO and a VM restart, not a new golden image and
|
||||
not a rebuilt overlay.
|
||||
|
||||
Neither image is ever published by this project: the host builds both from
|
||||
public sources (`invisible-dots image build`, code in `guest/image-builder/`),
|
||||
with the same QEMU it runs Dots with and the same machine: the builder VM's
|
||||
command line is built from the vm-manager's `machineArgs()`, drive and device
|
||||
The host builds the runtime disk from this checkout, and the golden image from
|
||||
public sources (`invisible-dots image build`, code in `guest/image-builder/`)
|
||||
unless CI has published the golden image of exactly the same inputs: the
|
||||
release `golden-<inputs digest>` of `.github/workflows/golden-image.yml`,
|
||||
which `image build` downloads instead (`guest/image-builder/src/prebuilt.ts`;
|
||||
section 11.3). Either way the golden image is provisioned with the same QEMU a
|
||||
host runs Dots with and the same machine: the builder VM's command line is built from the vm-manager's `machineArgs()`, drive and device
|
||||
functions, the ones `qemuArgs()` uses, plus only what a builder needs, so a
|
||||
golden image is provisioned on the machine type, accelerator and CPU model a
|
||||
Dot boots it on. The seed and runtime ISO labels are each defined once, in
|
||||
@@ -2947,7 +2953,7 @@ system and the apt packages the golden image installs on it (`apt_packages` of
|
||||
`guest/image-builder/pins.json`: Xvfb, XFCE and their libraries, over the Linux kernel,
|
||||
bash and the rest of Ubuntu's base), most of it GPL or LGPL software under its own
|
||||
licenses and source offers, downloaded from Ubuntu's archive on the machine that builds
|
||||
the image and published by no one here. Whoever copies a golden image to another
|
||||
the image, which is CI when the image is the published release of the paragraph above. Whoever copies a golden image to another
|
||||
machine takes those licenses with it (the paragraph above).
|
||||
|
||||
### 11.4 setup --all
|
||||
|
||||
@@ -60,11 +60,14 @@ an environment only by its owner.
|
||||
## Build
|
||||
|
||||
```sh
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o bin/dot-agentd ./cmd/dot-agentd
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false -ldflags="-s -w" -o bin/dot-agentd ./cmd/dot-agentd
|
||||
```
|
||||
|
||||
The result is a static linux/amd64 binary with no dependencies outside the
|
||||
Go standard library. The unix-only pieces (socket umask, process groups,
|
||||
Go standard library, and the same bytes from the same source with the Go that
|
||||
`go.mod`'s `toolchain` names: no checkout path, commit or "modified" mark is
|
||||
stamped into it. This is the one build: `agentdBuildCommand()` of the image
|
||||
builder, CI and the smoke run it with these flags. The unix-only pieces (socket umask, process groups,
|
||||
`statfs`) sit behind build tags, so `go vet ./...` also works on other systems.
|
||||
|
||||
## Test
|
||||
|
||||
@@ -1,3 +1,7 @@
|
||||
module github.com/feder-cr/dots/guest/dot-agentd
|
||||
|
||||
go 1.25.0
|
||||
|
||||
// The Go that builds dot-agentd: CI's (setup-go reads this line), the smoke's (the golang image of this version),
|
||||
// and a host's whose own Go is older (go downloads this one). The same source and this Go give the same bytes.
|
||||
toolchain go1.26.8
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -93,24 +94,55 @@ func TestProcFollowsAResizeAndForwardsSignals(t *testing.T) {
|
||||
resize := make(chan ProcTTY, 1)
|
||||
signals := make(chan syscall.Signal, 1)
|
||||
resize <- ProcTTY{Cols: 50, Rows: 10}
|
||||
// The program says "resized" once its terminal has the new size, and the signal is sent only then: it must reach
|
||||
// a program that is running, and it ends the program, which would otherwise sleep for 30 s.
|
||||
out := &signalWriter{want: "resized", seen: make(chan struct{})}
|
||||
go func() {
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
<-out.seen
|
||||
signals <- syscall.SIGTERM
|
||||
}()
|
||||
code, out, _ := relay(t, sock, RelayOptions{
|
||||
Argv: []string{"/bin/sh", "-c", `sleep 0.3; stty size; sleep 30`},
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
code, err := RunRelay(ctx, RelayOptions{
|
||||
Socket: sock,
|
||||
Argv: []string{"/bin/sh", "-c", `while [ "$(stty size)" != "10 50" ]; do sleep 0.05; done; echo resized; sleep 30`},
|
||||
TTY: &ProcTTY{Cols: 80, Rows: 24},
|
||||
Resize: resize,
|
||||
Signals: signals,
|
||||
}, "")
|
||||
if !strings.Contains(out, "10 50") {
|
||||
t.Errorf("the resize did not reach the terminal: %q", out)
|
||||
}, nil, out, io.Discard)
|
||||
if err != nil {
|
||||
t.Fatalf("the relay failed (the resize never reached the terminal?): %v; output %q", err, out.String())
|
||||
}
|
||||
if code != 128+int(syscall.SIGTERM) {
|
||||
t.Errorf("code %d, want %d", code, 128+int(syscall.SIGTERM))
|
||||
t.Errorf("code %d, want %d; output %q", code, 128+int(syscall.SIGTERM), out.String())
|
||||
}
|
||||
}
|
||||
|
||||
// signalWriter keeps what is written to it and closes seen once it holds want.
|
||||
type signalWriter struct {
|
||||
mu sync.Mutex
|
||||
buf bytes.Buffer
|
||||
want string
|
||||
seen chan struct{}
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (w *signalWriter) Write(p []byte) (int, error) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
n, err := w.buf.Write(p)
|
||||
if strings.Contains(w.buf.String(), w.want) {
|
||||
w.once.Do(func() { close(w.seen) })
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (w *signalWriter) String() string {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
return w.buf.String()
|
||||
}
|
||||
|
||||
func TestProcDiesWithItsCaller(t *testing.T) {
|
||||
sock, _ := localSocket(t)
|
||||
procDiesWithItsCaller(t, sock, filepath.Join(t.TempDir(), "pid"))
|
||||
|
||||
@@ -2,8 +2,10 @@
|
||||
|
||||
`@invisible-dots/image-builder` builds the two guest images of architecture
|
||||
section 3.3 on the host, with the same code on Linux and Windows. It is what
|
||||
`invisible-dots image build` runs. Neither image is ever published: every host
|
||||
builds its own from public sources.
|
||||
`invisible-dots image build` runs. Every host builds its runtime ISO from its
|
||||
checkout; the golden image it downloads when CI published the one of exactly
|
||||
its inputs (the release `golden-<inputs digest>`, `src/prebuilt.ts`), and
|
||||
builds it from public sources otherwise.
|
||||
|
||||
Both images land in `INVISIBLE_DOTS_HOME/images`, each with a manifest next to
|
||||
it (`golden-<v>.json`, `runtime-<v>.json`) that records what went in and the
|
||||
@@ -33,13 +35,14 @@ image's SHA-256; `invisible-dots doctor` checks an image against it with
|
||||
same machine, CPU model and devices as a Dot (its command line is built
|
||||
from the vm-manager's `machineArgs()`). `provision.sh` installs the
|
||||
apt packages (Xvfb, a minimal XFCE, the browser's libraries, ImageMagick
|
||||
for dot-agentd's screenshots), uv, then removes the cloud image's services a
|
||||
Dot never uses (snapd, unattended-upgrades, apport, the LXD stubs, Ubuntu Pro,
|
||||
for dot-agentd's screenshots) from the Ubuntu archive as it was at
|
||||
`pins.json`'s `apt_snapshot` (`apt-get --snapshot`), uv, then removes the
|
||||
cloud image's services a Dot never uses (snapd, unattended-upgrades, apport, the LXD stubs, Ubuntu Pro,
|
||||
the release upgrader, the SSH server, pollinate, which would tell Canonical of
|
||||
every new Dot), then builds the Dot's browser with
|
||||
`build-browser-env.sh <lock> ~dot/.local/share/invisible-dots/mcp`, as user
|
||||
`dot`: a virtual environment filled with
|
||||
`uv pip install --require-hashes -r mcp-requirements.lock`, its
|
||||
`uv pip install --require-hashes --only-binary :all: -r mcp-requirements.lock`, its
|
||||
`invisible-playwright-mcp` linked into `~/.local/bin`, and
|
||||
`invisible-playwright fetch` run from that environment, so the cached browser
|
||||
engine is the one the MCP server expects (the library keeps its own GeoIP
|
||||
@@ -101,11 +104,13 @@ named in the error.
|
||||
| on the ISO | from |
|
||||
|---|---|
|
||||
| `install.sh` | `runtime/install.sh` |
|
||||
| `vmproxy.py` | `runtime/vmproxy.py`, which `install.sh` runs to set up the Dot's VM proxy |
|
||||
| `VERSION` | the version |
|
||||
| `bin/dot-agentd` | `guest/dot-agentd/bin/` (`CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/dot-agentd ./cmd/dot-agentd`), refused unless it is a linux/amd64 ELF |
|
||||
| `bin/dot-agentd` | `guest/dot-agentd/bin/` (`CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false -ldflags="-s -w" -o bin/dot-agentd ./cmd/dot-agentd`), refused unless it is a linux/amd64 ELF |
|
||||
| `bin/dot-desktop` | `runtime/dot-desktop.sh` |
|
||||
| `engine/nanobot/...` | `invisible_engine_dots/nanobot/`: every `*.py` and the `*.md` under `templates/` |
|
||||
| `engine/requirements.lock` | `builder/engine-requirements.lock`, byte for byte: the engine refuses to start when it differs from the venv's copy |
|
||||
| `bin/dot-install` | `runtime/dot-install.sh`, the one command `dot` may run as root |
|
||||
| `engine/nanobot/...` | `invisible_engine_dots/nanobot/`: every `*.py`, the `*.json` data and the `*.md` under `templates/` |
|
||||
| `engine/skills/<name>/SKILL.md` | `invisible_engine_dots/skills/`, the built-in skills |
|
||||
| `engine/LICENSE`, `engine/UPSTREAM.md` | `invisible_engine_dots/` |
|
||||
| `units/*.service` | `units/` |
|
||||
|
||||
|
||||
@@ -33,7 +33,9 @@ install -m 0644 "$lock" "$dot_lock"
|
||||
|
||||
as_dot mkdir -p "$bin_dir" "$(dirname "$env_dir")"
|
||||
as_dot uv venv --quiet --python /usr/bin/python3 "$env_dir"
|
||||
as_dot uv pip install --python "$env_dir/bin/python" --require-hashes -r "$dot_lock"
|
||||
# Wheels only, as for the engine: no build script of any package runs. --no-cache: the image does not need
|
||||
# uv's download cache in dot's home.
|
||||
as_dot uv pip install --no-cache --python "$env_dir/bin/python" --require-hashes --only-binary :all: -r "$dot_lock"
|
||||
[ -x "$env_dir/bin/invisible-playwright" ] || { echo "build-browser-env: no invisible-playwright in $env_dir/bin" >&2; exit 1; }
|
||||
[ -x "$env_dir/bin/invisible-playwright-mcp" ] || { echo "build-browser-env: no invisible-playwright-mcp in $env_dir/bin" >&2; exit 1; }
|
||||
as_dot ln -sfn "$env_dir/bin/invisible-playwright-mcp" "$bin_dir/invisible-playwright-mcp"
|
||||
|
||||
@@ -30,13 +30,18 @@ trap 'on_error "$LINENO" "$BASH_COMMAND"' ERR
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
export NEEDRESTART_MODE=a
|
||||
|
||||
step "installing packages: $APT_PACKAGES"
|
||||
step "installing packages from the Ubuntu archive as of $APT_SNAPSHOT: $APT_PACKAGES"
|
||||
# The cloud image may still be running its own apt jobs at first boot: wait for the lock.
|
||||
apt_wait=(-o DPkg::Lock::Timeout=600)
|
||||
apt-get "${apt_wait[@]}" update
|
||||
# The packages and every dependency come from the archive as it was at the pinned moment (snapshot.ubuntu.com,
|
||||
# which apt reaches for the archive this image is configured with), not as it is on the day of the build: the same
|
||||
# pins make the same image. --error-on=any: a list that cannot be fetched fails the build instead of leaving apt
|
||||
# with whatever lists the cloud image had.
|
||||
apt_snapshot=(--snapshot "$APT_SNAPSHOT")
|
||||
apt-get "${apt_wait[@]}" "${apt_snapshot[@]}" --error-on=any update
|
||||
# shellcheck disable=SC2086
|
||||
# unsafe-io: no fsync per package, the builder VM is thrown away if the build fails anyway.
|
||||
apt-get "${apt_wait[@]}" -o Dpkg::Options::=--force-unsafe-io install -y --no-install-recommends $APT_PACKAGES
|
||||
apt-get "${apt_wait[@]}" "${apt_snapshot[@]}" -o Dpkg::Options::=--force-unsafe-io install -y --no-install-recommends $APT_PACKAGES
|
||||
|
||||
step "removing what a Dot never uses"
|
||||
# Services of the cloud image that would run in every Dot for nothing: snaps, automatic upgrades (apt in the
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
"url": "https://github.com/heiher/hev-socks5-tunnel/releases/download/2.18.0/hev-socks5-tunnel-linux-x86_64",
|
||||
"sha256": "1903b23869cbd12b7b57045ca161c0f90314d9c4ef2835e7a9e4a3179bf7d925"
|
||||
},
|
||||
"apt_snapshot": "20261010T000000Z",
|
||||
"apt_packages": [
|
||||
"xvfb",
|
||||
"xfce4-session",
|
||||
|
||||
@@ -14,8 +14,13 @@ usage="usage: sudo dot-install <package>..."
|
||||
[ "$(id -u)" -eq 0 ] || { echo "dot-install: run it with sudo ($usage)" >&2; exit 2; }
|
||||
[ $# -ge 1 ] || { echo "$usage" >&2; exit 2; }
|
||||
for name in "$@"; do
|
||||
# A Debian package name: lower case letters, digits, "+", "." and "-", starting with a letter or a digit.
|
||||
[[ "$name" =~ ^[a-z0-9][a-z0-9+.-]+$ ]] || { echo "dot-install: \"$name\" is not a package name ($usage)" >&2; exit 2; }
|
||||
# A Debian package name (policy 5.6.1): lower case letters, digits, "+", "." and "-", at least two characters,
|
||||
# starting with a letter or a digit; that leaves out every option, version (=), release (/) and architecture (:).
|
||||
# And not ending in "-": `apt-get install` reads a trailing "-" as "remove this package" when no package has the
|
||||
# whole word as its name (apt-get(8)), so `foo-` would remove foo as root; no package of the archive ends in "-".
|
||||
# A trailing "+" stays: it is apt's "install" mark, so `foo+` installs foo, and a name that ends in it (g++,
|
||||
# ncbi-blast+) is matched whole first, so nothing but an install can come of it.
|
||||
[[ "$name" =~ ^[a-z0-9][a-z0-9+.-]*[a-z0-9+.]$ ]] || { echo "dot-install: \"$name\" is not a package name ($usage)" >&2; exit 2; }
|
||||
done
|
||||
|
||||
export DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a
|
||||
|
||||
@@ -57,43 +57,11 @@ install -d -o dotengine -g dotengine -m 0700 /home/dotengine/state
|
||||
proxy_dir=/run/dot-vmproxy
|
||||
if [ -n "$(python3 -c 'import json;print(json.load(open("/etc/invisible-dots/config.json")).get("proxy") or "")' 2>/dev/null)" ]; then
|
||||
install -d -o root -g root -m 0700 "$proxy_dir"
|
||||
# Writes hev.yml (0600) and env (the proxy's address and the uplink, for the routes); answers 1 for a value that
|
||||
# is not socks5://[user:password@]host:port.
|
||||
if ! python3 - "$proxy_dir" <<'PY'
|
||||
import json, os, socket, subprocess, sys
|
||||
from urllib.parse import unquote, urlsplit
|
||||
out = sys.argv[1]
|
||||
url = urlsplit(json.load(open("/etc/invisible-dots/config.json"))["proxy"].strip())
|
||||
if url.scheme != "socks5" or not url.hostname or not url.port:
|
||||
sys.exit(1)
|
||||
address = socket.getaddrinfo(url.hostname, url.port, socket.AF_INET, socket.SOCK_STREAM)[0][4][0]
|
||||
route = subprocess.run(["ip", "-4", "route", "show", "default"], capture_output=True, text=True).stdout.split()
|
||||
gateway, device = route[route.index("via") + 1], route[route.index("dev") + 1]
|
||||
q = lambda value: json.dumps(value)
|
||||
lines = [
|
||||
"tunnel:", " name: tun0", " mtu: 8500", " ipv4: 198.18.0.1", " post-up-script: /run/dot-vmproxy/up.sh",
|
||||
"socks5:", f" address: {q(address)}", f" port: {url.port}", " udp: 'udp'",
|
||||
]
|
||||
if url.username:
|
||||
lines += [f" username: {q(unquote(url.username))}", f" password: {q(unquote(url.password or ''))}"]
|
||||
lines += ["mapdns:", " address: 198.18.0.2", " port: 53", " network: 100.64.0.0", " netmask: 255.192.0.0", " cache-size: 10000"]
|
||||
def write(name, text, mode):
|
||||
fd = os.open(os.path.join(out, name), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode)
|
||||
with os.fdopen(fd, "wb") as f:
|
||||
f.write(text.encode())
|
||||
write("hev.yml", "\n".join(lines) + "\n", 0o600)
|
||||
write("up.sh", "#!/bin/sh\n"
|
||||
f"ip route replace {address}/32 via {gateway} dev {device}\n"
|
||||
"ip route replace default dev tun0 metric 1\n"
|
||||
"resolvectl dns tun0 198.18.0.2 && resolvectl domain tun0 '~.' && resolvectl default-route tun0 yes\n"
|
||||
f"resolvectl default-route {device} no || true\n", 0o700)
|
||||
write("allow.nft", "table inet dot_vmproxy {\n chain out {\n type filter hook output priority 0; policy drop;\n"
|
||||
" oifname { \"lo\", \"tun0\" } accept\n ct state established,related accept\n"
|
||||
f" ip daddr {address} tcp dport {url.port} accept\n ip daddr {address} meta l4proto udp accept\n"
|
||||
" ip daddr 10.0.2.0/24 accept\n udp dport 67 accept\n }\n}\n", 0o600)
|
||||
PY
|
||||
then
|
||||
log "error: the Dot's proxy is not socks5://[user:password@]host:port; nothing goes out until it is fixed"
|
||||
# vmproxy.py (on this disk) writes the tunnel's settings, its routes and the firewall; when it cannot (the value is
|
||||
# not socks5://[user:password@]host:port, its host does not resolve, the VM has no default route), it says why
|
||||
# and writes nothing, and the firewall below lets nothing out.
|
||||
if ! reason=$(python3 "$runtime/vmproxy.py" /etc/invisible-dots/config.json "$proxy_dir" 2>&1); then
|
||||
log "error: the Dot's proxy cannot be set up: $reason; nothing goes out until it is fixed"
|
||||
printf 'table inet dot_vmproxy {\n chain out {\n type filter hook output priority 0; policy drop;\n oifname "lo" accept\n ct state established,related accept\n ip daddr 10.0.2.0/24 accept\n }\n}\n' > "$proxy_dir/allow.nft"
|
||||
fi
|
||||
sysctl -q -w net.ipv6.conf.all.disable_ipv6=1 net.ipv6.conf.default.disable_ipv6=1
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
"""The Dot's VM proxy, as install.sh sets it up (it runs this as root on every boot when config.json has a proxy).
|
||||
|
||||
python3 vmproxy.py <config.json> <output directory>
|
||||
|
||||
Writes, into the output directory, hev.yml (0600: the tunnel's settings, password included), up.sh (0700: the routes
|
||||
and the DNS the tunnel sets once tun0 is up) and allow.nft (0600: the firewall that lets nothing leave but through the
|
||||
tunnel). Prints nothing on success. Otherwise it writes nothing, prints the reason on stderr and exits with:
|
||||
|
||||
3 the proxy is not socks5://[user:password@]host:port
|
||||
4 the proxy's host does not resolve to an IPv4 address
|
||||
5 the VM has no default IPv4 route through a gateway (the tunnel needs it to reach the proxy)
|
||||
|
||||
Whatever the reason, install.sh then puts a firewall in place that lets nothing out: the Dot never goes out directly
|
||||
when it has a proxy.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
from urllib.parse import unquote, urlsplit
|
||||
|
||||
INVALID, UNRESOLVED, NO_ROUTE = 3, 4, 5
|
||||
|
||||
|
||||
class Refused(Exception):
|
||||
def __init__(self, code, reason):
|
||||
super().__init__(reason)
|
||||
self.code = code
|
||||
|
||||
|
||||
def proxy_url(config_path):
|
||||
with open(config_path) as f:
|
||||
raw = json.load(f).get("proxy") or ""
|
||||
try:
|
||||
url = urlsplit(raw.strip())
|
||||
port = url.port
|
||||
except ValueError as error:
|
||||
raise Refused(INVALID, f"the proxy is not socks5://[user:password@]host:port ({error})")
|
||||
if url.scheme != "socks5" or not url.hostname or not port:
|
||||
raise Refused(INVALID, "the proxy is not socks5://[user:password@]host:port")
|
||||
return url
|
||||
|
||||
|
||||
def resolve(host, port):
|
||||
try:
|
||||
return socket.getaddrinfo(host, port, socket.AF_INET, socket.SOCK_STREAM)[0][4][0]
|
||||
except OSError as error:
|
||||
raise Refused(UNRESOLVED, f"the proxy's host {host} does not resolve to an IPv4 address ({error})")
|
||||
|
||||
|
||||
def default_route():
|
||||
"""The gateway and the device of the default IPv4 route."""
|
||||
shown = subprocess.run(["ip", "-4", "route", "show", "default"], capture_output=True, text=True, check=True).stdout
|
||||
words = shown.split()
|
||||
if "via" not in words or "dev" not in words:
|
||||
seen = shown.strip() or "none"
|
||||
raise Refused(NO_ROUTE, f"the VM has no default IPv4 route through a gateway (ip route shows: {seen})")
|
||||
return words[words.index("via") + 1], words[words.index("dev") + 1]
|
||||
|
||||
|
||||
def files(url, address, gateway, device):
|
||||
q = json.dumps
|
||||
lines = [
|
||||
"tunnel:", " name: tun0", " mtu: 8500", " ipv4: 198.18.0.1", " post-up-script: /run/dot-vmproxy/up.sh",
|
||||
"socks5:", f" address: {q(address)}", f" port: {url.port}", " udp: 'udp'",
|
||||
]
|
||||
if url.username:
|
||||
lines += [f" username: {q(unquote(url.username))}", f" password: {q(unquote(url.password or ''))}"]
|
||||
lines += ["mapdns:", " address: 198.18.0.2", " port: 53", " network: 100.64.0.0", " netmask: 255.192.0.0", " cache-size: 10000"]
|
||||
up = (
|
||||
"#!/bin/sh\n"
|
||||
f"ip route replace {address}/32 via {gateway} dev {device}\n"
|
||||
"ip route replace default dev tun0 metric 1\n"
|
||||
"resolvectl dns tun0 198.18.0.2 && resolvectl domain tun0 '~.' && resolvectl default-route tun0 yes\n"
|
||||
f"resolvectl default-route {device} no || true\n"
|
||||
)
|
||||
allow = (
|
||||
"table inet dot_vmproxy {\n chain out {\n type filter hook output priority 0; policy drop;\n"
|
||||
" oifname { \"lo\", \"tun0\" } accept\n ct state established,related accept\n"
|
||||
f" ip daddr {address} tcp dport {url.port} accept\n ip daddr {address} meta l4proto udp accept\n"
|
||||
" ip daddr 10.0.2.0/24 accept\n udp dport 67 accept\n }\n}\n"
|
||||
)
|
||||
return [("hev.yml", "\n".join(lines) + "\n", 0o600), ("up.sh", up, 0o700), ("allow.nft", allow, 0o600)]
|
||||
|
||||
|
||||
def write(out, name, text, mode):
|
||||
fd = os.open(os.path.join(out, name), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, mode)
|
||||
with os.fdopen(fd, "wb") as f:
|
||||
f.write(text.encode())
|
||||
|
||||
|
||||
def main(config_path, out):
|
||||
try:
|
||||
url = proxy_url(config_path)
|
||||
address = resolve(url.hostname, url.port)
|
||||
gateway, device = default_route()
|
||||
except Refused as refused:
|
||||
print(refused, file=sys.stderr)
|
||||
return refused.code
|
||||
for name, text, mode in files(url, address, gateway, device):
|
||||
write(out, name, text, mode)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) != 3:
|
||||
print("usage: vmproxy.py <config.json> <output directory>", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
sys.exit(main(sys.argv[1], sys.argv[2]))
|
||||
@@ -1,7 +1,7 @@
|
||||
/**
|
||||
* The files of this package that end up inside a guest: the builder's
|
||||
* cloud-init and provisioner, and the runtime ISO's install hook, desktop
|
||||
* script and systemd units. They are kept as plain files so they can be read,
|
||||
* cloud-init and provisioner, and the runtime ISO's install hook, VM proxy
|
||||
* setup, desktop script and systemd units. They are kept as plain files so they can be read,
|
||||
* diffed and checked with shell tools, and read from disk at build time.
|
||||
*/
|
||||
import { readFile } from "node:fs/promises";
|
||||
@@ -30,6 +30,8 @@ export const BUILDER_BROWSER_BUILD = "builder/build-browser-env.sh";
|
||||
export const RUNTIME_INSTALL = "runtime/install.sh";
|
||||
export const RUNTIME_DESKTOP = "runtime/dot-desktop.sh";
|
||||
export const RUNTIME_DOT_INSTALL = "runtime/dot-install.sh";
|
||||
/** Writes the VM proxy's tunnel settings, routes and firewall; install.sh runs it when the Dot has a proxy. */
|
||||
export const RUNTIME_VMPROXY = "runtime/vmproxy.py";
|
||||
/** The guest units, in the order install.sh enables them. */
|
||||
export const GUEST_UNITS = ["dot-desktop.service", "dot-agentd.service", "invisible-dots-agent.service"] as const;
|
||||
|
||||
@@ -38,7 +40,7 @@ export function unitAsset(name: (typeof GUEST_UNITS)[number]): string {
|
||||
}
|
||||
|
||||
/** Every guest file, for checks that apply to all of them. */
|
||||
export const GUEST_ASSETS: readonly string[] = [BUILDER_USER_DATA, BUILDER_PROVISION, BUILDER_PYTHON_LOCK, BUILDER_ENGINE_LOCK, BUILDER_ENGINE_BUILD, BUILDER_BROWSER_BUILD, RUNTIME_INSTALL, RUNTIME_DESKTOP, RUNTIME_DOT_INSTALL, ...GUEST_UNITS.map(unitAsset)];
|
||||
export const GUEST_ASSETS: readonly string[] = [BUILDER_USER_DATA, BUILDER_PROVISION, BUILDER_PYTHON_LOCK, BUILDER_ENGINE_LOCK, BUILDER_ENGINE_BUILD, BUILDER_BROWSER_BUILD, RUNTIME_INSTALL, RUNTIME_DESKTOP, RUNTIME_DOT_INSTALL, RUNTIME_VMPROXY, ...GUEST_UNITS.map(unitAsset)];
|
||||
|
||||
/**
|
||||
* Reads a guest file and refuses one a Windows checkout turned into CRLF:
|
||||
|
||||
@@ -8,11 +8,11 @@ import { createHash } from "node:crypto";
|
||||
import { chmod, copyFile, mkdir, readFile, rm, stat } from "node:fs/promises";
|
||||
import { basename, join } from "node:path";
|
||||
import { writeIso } from "@invisible-dots/iso";
|
||||
import { hostPaths, replaceFile, type HostPaths } from "@invisible-dots/shared";
|
||||
import { hostPaths, type HostPaths } from "@invisible-dots/shared";
|
||||
import { BUILDER_BROWSER_BUILD, BUILDER_ENGINE_BUILD, BUILDER_ENGINE_LOCK, BUILDER_PROVISION, BUILDER_PYTHON_LOCK, BUILDER_USER_DATA, defaultAssetRoot, readGuestAsset } from "./assets.js";
|
||||
import { fetchVerified, sha256File, type Fetch, type FetchVerifiedOptions } from "./download.js";
|
||||
import { acquireLock, type Lock } from "./lock.js";
|
||||
import { manifestPathFor, writeManifest, type GoldenManifest } from "./manifest.js";
|
||||
import { manifestPathFor, type GoldenManifest } from "./manifest.js";
|
||||
import { BASE_IMAGE, downloadFileName, GUEST_PINS, type BaseImagePin, type GuestPins, type PinnedDownload } from "./pins.js";
|
||||
import { waitForExit, type ProcessRunner } from "./process.js";
|
||||
import { PREBUILT_RELEASES, pullPrebuiltGolden } from "./prebuilt.js";
|
||||
@@ -21,7 +21,8 @@ import { builderQemuArgs, type Accelerator, type QemuPrograms } from "./qemu.js"
|
||||
import { SEED_VOLUME_ID } from "@invisible-dots/vm-manager";
|
||||
import { builderSeedEntries } from "./seed.js";
|
||||
import { buildVerdict, followSerialLog, installedComponents } from "./serial.js";
|
||||
import { checkVersion, findImageByDigest, inputsDigest, versionFor } from "./versions.js";
|
||||
import { inputsDigest } from "./versions.js";
|
||||
import { publishImage, resolveImageVersion } from "./publish.js";
|
||||
|
||||
export const GOLDEN_DEFAULTS = {
|
||||
/** Each Dot's overlay is larger; cloud-init grows the filesystem when a Dot boots. */
|
||||
@@ -112,28 +113,16 @@ export async function buildGoldenImage(options: GoldenBuildOptions): Promise<Gol
|
||||
// source is not: it travels on the runtime disk (section 3.3).
|
||||
const digest = inputsDigest([JSON.stringify(base), JSON.stringify(pins), userData, provision, pythonLock, diskSize, engineLock, engineBuild, browserBuild]);
|
||||
|
||||
await mkdir(paths.imagesDir, { recursive: true });
|
||||
let version: string;
|
||||
if (options.version === undefined) {
|
||||
const existing = await findImageByDigest(paths.imagesDir, "golden", digest);
|
||||
if (existing) {
|
||||
log(`${basename(existing.path)} already holds these inputs; nothing to do`);
|
||||
return { version: existing.version, image: existing.path, manifest: manifestPathFor(existing.path), created: false };
|
||||
}
|
||||
version = versionFor(now(), digest);
|
||||
} else {
|
||||
version = checkVersion(options.version);
|
||||
}
|
||||
|
||||
const image = paths.goldenImagePath(version);
|
||||
const manifest = manifestPathFor(image);
|
||||
if (await exists(image)) {
|
||||
if (await exists(manifest)) {
|
||||
log(`${basename(image)} already exists; nothing to do`);
|
||||
return { version, image, manifest, created: false };
|
||||
}
|
||||
throw new GoldenBuildError(`${image} exists without its manifest ${manifest}: it is not one this builder finished; remove it by hand`);
|
||||
}
|
||||
const { version, image, manifest, existing } = await resolveImageVersion({
|
||||
kind: "golden",
|
||||
imagesDir: paths.imagesDir,
|
||||
digest,
|
||||
version: options.version,
|
||||
now,
|
||||
imagePath: (v) => paths.goldenImagePath(v),
|
||||
log,
|
||||
});
|
||||
if (existing) return { version, image, manifest, created: false };
|
||||
|
||||
const lock = await acquireLock(join(paths.imagesDir, ".golden-build.lock"), "golden image build");
|
||||
try {
|
||||
@@ -264,18 +253,14 @@ async function buildLocked(options: GoldenBuildOptions & Resolved, target: Targe
|
||||
"invisible-playwright-mcp": target.python.mcpVersion,
|
||||
"invisible-playwright": target.python.playwrightVersion,
|
||||
"mcp-requirements.lock": createHash("sha256").update(target.pythonLock).digest("hex"),
|
||||
apt_snapshot: pins.apt_snapshot,
|
||||
apt_packages: [...pins.apt_packages],
|
||||
},
|
||||
engine: { lock_sha256: createHash("sha256").update(target.engineLock).digest("hex") },
|
||||
installed: installedComponents(consoleText),
|
||||
builder: { accelerator: options.accelerator },
|
||||
};
|
||||
// The manifest goes first: the control plane picks images by file name,
|
||||
// so the image must not appear before the record that describes it.
|
||||
await writeManifest(target.manifest, manifest);
|
||||
await replaceFile(partial, target.image);
|
||||
// Immutable once a VM uses it (section 3.3); QEMU opens a backing file read-only anyway.
|
||||
await chmod(target.image, 0o444);
|
||||
await publishImage(partial, target.image, target.manifest, manifest);
|
||||
await rm(workDir, { recursive: true, force: true });
|
||||
log(`done: ${target.image} (sha256 ${sha256})`);
|
||||
return { version: target.version, image: target.image, manifest: target.manifest, created: true };
|
||||
@@ -349,7 +334,3 @@ function duration(ms: number): string {
|
||||
function lastLines(text: string, count: number): string {
|
||||
return text.split(/\r?\n/).filter((line) => line.trim() !== "").slice(-count).join("\n");
|
||||
}
|
||||
|
||||
async function exists(path: string): Promise<boolean> {
|
||||
return (await stat(path).catch(() => undefined)) !== undefined;
|
||||
}
|
||||
@@ -26,6 +26,7 @@ export { normalizePackageName, parseHashedLock, parsePythonLock, type PythonLock
|
||||
export type { ProcessRunner, RunningProcess } from "./process.js";
|
||||
export { builderQemuArgs, type Accelerator, type QemuPrograms } from "./qemu.js";
|
||||
export {
|
||||
AGENTD_BUILD_SHELL,
|
||||
agentdBuildCommand,
|
||||
assertLinuxAmd64Elf,
|
||||
type BuildCommand,
|
||||
|
||||
@@ -38,6 +38,8 @@ export interface GoldenManifest {
|
||||
"invisible-playwright": string;
|
||||
/** SHA-256 of builder/mcp-requirements.lock: the whole Python environment, transitive packages included. */
|
||||
"mcp-requirements.lock": string;
|
||||
/** The moment of the Ubuntu archive apt_packages came from (pins.json). */
|
||||
apt_snapshot: string;
|
||||
apt_packages: string[];
|
||||
};
|
||||
/** The engine's Python environment (builder/engine-requirements.lock): the lock the runtime disk's copy must equal. */
|
||||
|
||||
@@ -38,6 +38,11 @@ export interface GuestPins {
|
||||
uv: PinnedDownload;
|
||||
/** hev-socks5-tunnel, the static binary the guest routes the whole VM through when the Dot has a VM proxy. */
|
||||
tunnel: PinnedDownload;
|
||||
/**
|
||||
* The moment of the Ubuntu archive the packages come from (snapshot.ubuntu.com, `YYYYMMDDTHHMMSSZ`): apt resolves
|
||||
* apt_packages and their dependencies as the archive was then, so the same pins give the same packages on any day.
|
||||
*/
|
||||
apt_snapshot: string;
|
||||
apt_packages: string[];
|
||||
}
|
||||
|
||||
@@ -49,6 +54,8 @@ const SHA256 = /^[0-9a-f]{64}$/;
|
||||
*/
|
||||
const PLAIN_WORD = /^[A-Za-z0-9][A-Za-z0-9.+_-]*$/;
|
||||
const FILE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
||||
/** A time of snapshot.ubuntu.com, as `apt-get --snapshot` takes it. */
|
||||
const APT_SNAPSHOT = /^\d{8}T\d{6}Z$/;
|
||||
|
||||
function field(object: Record<string, unknown>, key: string, where: string): string {
|
||||
const value = object[key];
|
||||
@@ -119,9 +126,12 @@ export function parseGuestPins(value: unknown): GuestPins {
|
||||
if ("python_packages" in o) throw new Error(`${where}: python_packages moved to builder/mcp-requirements.lock, which pins them with their hashes`);
|
||||
const apt = o.apt_packages;
|
||||
if (!Array.isArray(apt) || apt.length === 0) throw new Error(`${where}: apt_packages must be a non-empty array`);
|
||||
const snapshot = field(o, "apt_snapshot", where);
|
||||
if (!APT_SNAPSHOT.test(snapshot)) throw new Error(`${where}: apt_snapshot "${snapshot}" is not a snapshot time like 20261010T000000Z`);
|
||||
return {
|
||||
uv: parseDownload(o.uv, `${where} uv`),
|
||||
tunnel: parseDownload(o.tunnel, `${where} tunnel`),
|
||||
apt_snapshot: snapshot,
|
||||
apt_packages: apt.map((name, i) => plainWord(typeof name === "string" ? name : "", `${where} apt_packages[${i}]`)),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -14,11 +14,11 @@
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
import { createReadStream } from "node:fs";
|
||||
import { chmod, mkdir, open, rm } from "node:fs/promises";
|
||||
import { mkdir, open, rm } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { replaceFile } from "@invisible-dots/shared";
|
||||
import { DownloadError, fetchTextWithRetries, fetchVerified, type FetchVerifiedOptions } from "./download.js";
|
||||
import { writeManifest, type GoldenManifest } from "./manifest.js";
|
||||
import type { GoldenManifest } from "./manifest.js";
|
||||
import { publishImage } from "./publish.js";
|
||||
import { checkVersion } from "./versions.js";
|
||||
|
||||
/** Where the releases of the repository are downloaded from. */
|
||||
@@ -139,10 +139,8 @@ export async function pullPrebuiltGolden(options: PullOptions): Promise<PulledIm
|
||||
throw new Error(`the prebuilt golden image hashes to ${sha256}, its manifest says ${manifest.sha256}`);
|
||||
}
|
||||
const manifestPath = options.manifestPath(image);
|
||||
// The manifest first, as for a built image: the control plane picks images by file name.
|
||||
await writeManifest(manifestPath, manifest);
|
||||
await replaceFile(partial, image);
|
||||
await chmod(image, 0o444);
|
||||
// Published as a built image is: the manifest first, the image read-only.
|
||||
await publishImage(partial, image, manifestPath, manifest);
|
||||
await rm(workDir, { recursive: true, force: true });
|
||||
log(`done: ${image} (sha256 ${sha256})`);
|
||||
return { version, image, manifest: manifestPath };
|
||||
|
||||
@@ -1,22 +1,15 @@
|
||||
/**
|
||||
* The process boundary of the builder: the two calls it makes into the host.
|
||||
* It is the shape of the vm-manager's CommandRunner, declared here rather
|
||||
* than imported so this package's logic loads without the vm-manager; the
|
||||
* real runner is the vm-manager's (see host.ts), and tests pass a fake.
|
||||
* The process boundary of the builder: the two calls it makes into the host, in the vm-manager's terms (its
|
||||
* CommandRunner's `run` and the process `startProcess` gives). The real runner joins those two (the CLI's host.ts),
|
||||
* and tests pass a fake.
|
||||
*/
|
||||
import type { CommandResult, StartedProcess } from "@invisible-dots/vm-manager";
|
||||
|
||||
export interface RunningProcess {
|
||||
readonly pid: number | undefined;
|
||||
/** The last few KiB of stderr, for error messages when the process dies. */
|
||||
stderrTail(): string;
|
||||
/** Called once when the process exits or fails to start; `code` is null when killed by a signal. */
|
||||
onExit(listener: (code: number | null, signal: NodeJS.Signals | null, error?: Error) => void): void;
|
||||
kill(signal?: NodeJS.Signals): void;
|
||||
}
|
||||
export type RunningProcess = StartedProcess;
|
||||
|
||||
export interface ProcessRunner {
|
||||
/** Runs to completion; rejects unless the command exits 0. Argument arrays only, never a shell. */
|
||||
run(command: string, args: readonly string[], options?: { timeoutMs?: number }): Promise<{ stdout: string; stderr: string }>;
|
||||
run(command: string, args: readonly string[], options?: { timeoutMs?: number }): Promise<CommandResult>;
|
||||
/** `cwd` is the child's working directory: never inherited, see the vm-manager's startProcess. */
|
||||
spawn(command: string, args: readonly string[], options: { cwd: string }): RunningProcess;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
/**
|
||||
* What every way of getting an image shares (the golden image built or downloaded, the runtime ISO): which version
|
||||
* a build makes, or that an image of these inputs is there already, and how a finished image takes its name.
|
||||
*/
|
||||
import { chmod, mkdir, stat } from "node:fs/promises";
|
||||
import { basename } from "node:path";
|
||||
import { replaceFile, type ImageKind } from "@invisible-dots/shared";
|
||||
import { manifestPathFor, writeManifest, type ImageManifest } from "./manifest.js";
|
||||
import { checkVersion, findImageByDigest, versionFor } from "./versions.js";
|
||||
|
||||
export interface ImageTarget {
|
||||
version: string;
|
||||
image: string;
|
||||
manifest: string;
|
||||
/** True when the image is there already, complete with its manifest: there is nothing to build. */
|
||||
existing: boolean;
|
||||
}
|
||||
|
||||
export interface ResolveImageOptions {
|
||||
kind: ImageKind;
|
||||
imagesDir: string;
|
||||
/** The digest of the image's inputs. */
|
||||
digest: string;
|
||||
/** An explicit version, used as given; undefined is `<UTC time>-<digest>`, or the image that has this digest. */
|
||||
version: string | undefined;
|
||||
now: () => Date;
|
||||
/** The path of the image of a version. */
|
||||
imagePath: (version: string) => string;
|
||||
log: (line: string) => void;
|
||||
}
|
||||
|
||||
async function present(path: string): Promise<boolean> {
|
||||
return (await stat(path).catch(() => undefined)) !== undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* The image a build makes: with no explicit version, the image of the images directory that has these inputs if
|
||||
* there is one, otherwise a new version. An image file without its manifest is one no build finished (the manifest
|
||||
* is written first), and is refused rather than built over.
|
||||
*/
|
||||
export async function resolveImageVersion(options: ResolveImageOptions): Promise<ImageTarget> {
|
||||
await mkdir(options.imagesDir, { recursive: true });
|
||||
let version: string;
|
||||
if (options.version === undefined) {
|
||||
const found = await findImageByDigest(options.imagesDir, options.kind, options.digest);
|
||||
if (found) {
|
||||
options.log(`${basename(found.path)} already holds these inputs; nothing to do`);
|
||||
return { version: found.version, image: found.path, manifest: manifestPathFor(found.path), existing: true };
|
||||
}
|
||||
version = versionFor(options.now(), options.digest);
|
||||
} else {
|
||||
version = checkVersion(options.version);
|
||||
}
|
||||
const image = options.imagePath(version);
|
||||
const manifest = manifestPathFor(image);
|
||||
if (await present(image)) {
|
||||
if (await present(manifest)) {
|
||||
options.log(`${basename(image)} already exists; nothing to do`);
|
||||
return { version, image, manifest, existing: true };
|
||||
}
|
||||
throw new Error(`${image} exists without its manifest ${manifest}: it is not one this builder finished; remove it by hand`);
|
||||
}
|
||||
return { version, image, manifest, existing: false };
|
||||
}
|
||||
|
||||
/**
|
||||
* Puts a finished image (`partial`, on the same filesystem) under its name. The manifest goes first: the control
|
||||
* plane picks images by file name, so the image must never appear before the record that describes it. The image is
|
||||
* read-only from then on: immutable once a VM uses it (architecture section 3.3).
|
||||
*/
|
||||
export async function publishImage(partial: string, image: string, manifestPath: string, manifest: ImageManifest): Promise<void> {
|
||||
await writeManifest(manifestPath, manifest);
|
||||
await replaceFile(partial, image);
|
||||
await chmod(image, 0o444);
|
||||
}
|
||||
@@ -27,9 +27,8 @@ export interface BuilderVmSpec {
|
||||
serialLog: string;
|
||||
}
|
||||
|
||||
/** The builder VM's command line; its cpus and memory are held to a Dot's bounds by `machineArgs()`. */
|
||||
export function builderQemuArgs(spec: BuilderVmSpec): string[] {
|
||||
if (!Number.isInteger(spec.cpus) || spec.cpus < 1) throw new Error(`cpus must be a positive integer, got ${spec.cpus}`);
|
||||
if (!Number.isInteger(spec.memoryMib) || spec.memoryMib < 1024) throw new Error(`memory must be at least 1024 MiB, got ${spec.memoryMib}`);
|
||||
return [
|
||||
"-name",
|
||||
"invisible-dots-image-builder",
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
* /opt/invisible-dots:
|
||||
*
|
||||
* /install.sh the hook each Dot's seed runs on every boot
|
||||
* /vmproxy.py install.sh's setup of the Dot's VM proxy (tunnel, routes, firewall)
|
||||
* /VERSION this runtime's version
|
||||
* /bin/dot-agentd the computer daemon (linux/amd64)
|
||||
* /bin/dot-desktop ExecStart of dot-desktop.service
|
||||
@@ -22,11 +23,11 @@
|
||||
* permission bits having to survive a Windows host.
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
import { chmod, mkdir, open, readdir, rm, stat } from "node:fs/promises";
|
||||
import { open, readdir, rm, stat } from "node:fs/promises";
|
||||
import { basename, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { writeIso, type IsoEntry } from "@invisible-dots/iso";
|
||||
import { hostPaths, replaceFile, type HostPaths } from "@invisible-dots/shared";
|
||||
import { hostPaths, type HostPaths } from "@invisible-dots/shared";
|
||||
import { RUNTIME_ISO_LABEL } from "@invisible-dots/vm-manager";
|
||||
import {
|
||||
defaultAssetRoot,
|
||||
@@ -35,12 +36,14 @@ import {
|
||||
RUNTIME_DESKTOP,
|
||||
RUNTIME_DOT_INSTALL,
|
||||
RUNTIME_INSTALL,
|
||||
RUNTIME_VMPROXY,
|
||||
unitAsset,
|
||||
} from "./assets.js";
|
||||
import { sha256File } from "./download.js";
|
||||
import { acquireLock } from "./lock.js";
|
||||
import { manifestPathFor, writeManifest, type RuntimeFile, type RuntimeManifest } from "./manifest.js";
|
||||
import { checkVersion, findImageByDigest, inputsDigest, versionFor } from "./versions.js";
|
||||
import type { RuntimeFile, RuntimeManifest } from "./manifest.js";
|
||||
import { publishImage, resolveImageVersion } from "./publish.js";
|
||||
import { inputsDigest } from "./versions.js";
|
||||
|
||||
export interface RuntimeInputs {
|
||||
/** guest/dot-agentd/bin/dot-agentd, built for linux/amd64 */
|
||||
@@ -65,15 +68,28 @@ export interface BuildCommand {
|
||||
}
|
||||
|
||||
/**
|
||||
* The command that builds `defaultRuntimeInputs(repoRoot).agentdBinary`: a static linux/amd64 executable whatever
|
||||
* the host is (`assertLinuxAmd64Elf` refuses anything else), with the paths trimmed so the same source gives the same
|
||||
* bytes from any checkout. `go -C` enters the module itself, so the caller's working directory does not matter.
|
||||
* The one build of dot-agentd, the same wherever it runs (this function, CI, the smoke, the READMEs): a static
|
||||
* linux/amd64 executable whatever the host is (`assertLinuxAmd64Elf` refuses anything else), with the same bytes from
|
||||
* the same source and toolchain (go.mod's `toolchain`) in any checkout: `-trimpath` leaves out the checkout's path,
|
||||
* `-buildvcs=false` the commit and whether the tree had changes, which Go otherwise stamps into a binary built in a
|
||||
* git checkout, so that every commit, and every edit of any file, would make another runtime disk; `-s -w` leave out
|
||||
* the symbol table and the debug information.
|
||||
*/
|
||||
export function agentdBuildCommand(repoRoot: string): BuildCommand {
|
||||
const AGENTD_BUILD_ENV = { CGO_ENABLED: "0", GOOS: "linux", GOARCH: "amd64" } as const;
|
||||
const AGENTD_BUILD_FLAGS = ["-trimpath", "-buildvcs=false", "-ldflags=-s -w"] as const;
|
||||
|
||||
/** The same build as a shell line, run in guest/dot-agentd: what the documentation and the workflows quote. */
|
||||
export const AGENTD_BUILD_SHELL = 'CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false -ldflags="-s -w" -o bin/dot-agentd ./cmd/dot-agentd';
|
||||
|
||||
/**
|
||||
* The command that builds dot-agentd into `output`, by default `defaultRuntimeInputs(repoRoot).agentdBinary`, the
|
||||
* file the runtime disk takes. `go -C` enters the module itself, so the caller's working directory does not matter.
|
||||
*/
|
||||
export function agentdBuildCommand(repoRoot: string, output: string = defaultRuntimeInputs(repoRoot).agentdBinary): BuildCommand {
|
||||
return {
|
||||
command: "go",
|
||||
args: ["-C", join(repoRoot, "guest", "dot-agentd"), "build", "-trimpath", "-o", defaultRuntimeInputs(repoRoot).agentdBinary, "./cmd/dot-agentd"],
|
||||
env: { CGO_ENABLED: "0", GOOS: "linux", GOARCH: "amd64" },
|
||||
args: ["-C", join(repoRoot, "guest", "dot-agentd"), "build", ...AGENTD_BUILD_FLAGS, "-o", output, "./cmd/dot-agentd"],
|
||||
env: { ...AGENTD_BUILD_ENV },
|
||||
};
|
||||
}
|
||||
|
||||
@@ -112,8 +128,7 @@ export async function assertLinuxAmd64Elf(path: string): Promise<void> {
|
||||
const machine = header.readUInt16LE(18);
|
||||
if (!elf || !is64 || !little || machine !== 0x3e) {
|
||||
throw new Error(
|
||||
`${path} is not a linux/amd64 executable; build it with ` +
|
||||
`"go build" and GOOS=linux GOARCH=amd64 CGO_ENABLED=0 in guest/dot-agentd`,
|
||||
`${path} is not a linux/amd64 executable; build it in guest/dot-agentd with ${AGENTD_BUILD_SHELL}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -192,14 +207,14 @@ export async function runtimeFiles(inputs: RuntimeInputs, assetRoot: string = de
|
||||
const agentd = await stat(inputs.agentdBinary).catch(() => undefined);
|
||||
if (!agentd?.isFile()) {
|
||||
throw new Error(
|
||||
`dot-agentd binary ${inputs.agentdBinary} not found: build it first ` +
|
||||
`(in guest/dot-agentd: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o bin/dot-agentd ./cmd/dot-agentd)`,
|
||||
`dot-agentd binary ${inputs.agentdBinary} not found: build it first (in guest/dot-agentd: ${AGENTD_BUILD_SHELL})`,
|
||||
);
|
||||
}
|
||||
await assertLinuxAmd64Elf(inputs.agentdBinary);
|
||||
|
||||
const files = [
|
||||
stageBytes("install.sh", await readGuestAsset(assetRoot, RUNTIME_INSTALL)),
|
||||
stageBytes("vmproxy.py", await readGuestAsset(assetRoot, RUNTIME_VMPROXY)),
|
||||
stageBytes("bin/dot-desktop", await readGuestAsset(assetRoot, RUNTIME_DESKTOP)),
|
||||
stageBytes("bin/dot-install", await readGuestAsset(assetRoot, RUNTIME_DOT_INSTALL)),
|
||||
await stageFile("bin/dot-agentd", inputs.agentdBinary),
|
||||
@@ -216,28 +231,17 @@ export async function buildRuntimeIso(options: RuntimeBuildOptions = {}): Promis
|
||||
const files = await runtimeFiles(options.inputs ?? defaultRuntimeInputs(), options.assetRoot ?? defaultAssetRoot());
|
||||
const digest = inputsDigest(files.flatMap((file) => [file.path, file.sha256]));
|
||||
|
||||
await mkdir(paths.imagesDir, { recursive: true });
|
||||
let version: string;
|
||||
if (options.version === undefined) {
|
||||
const existing = await findImageByDigest(paths.imagesDir, "runtime", digest);
|
||||
if (existing) {
|
||||
log(`${basename(existing.path)} already holds this code; nothing to do`);
|
||||
return { version: existing.version, iso: existing.path, manifest: manifestPathFor(existing.path), created: false };
|
||||
}
|
||||
version = versionFor(now(), digest);
|
||||
} else {
|
||||
version = checkVersion(options.version);
|
||||
}
|
||||
|
||||
const iso = paths.runtimeIsoPath(version);
|
||||
const manifestPath = manifestPathFor(iso);
|
||||
if ((await stat(iso).catch(() => undefined)) !== undefined) {
|
||||
if ((await stat(manifestPath).catch(() => undefined)) !== undefined) {
|
||||
log(`${basename(iso)} already exists; nothing to do`);
|
||||
return { version, iso, manifest: manifestPath, created: false };
|
||||
}
|
||||
throw new Error(`${iso} exists without its manifest ${manifestPath}: it is not one this builder finished; remove it by hand`);
|
||||
}
|
||||
const target = await resolveImageVersion({
|
||||
kind: "runtime",
|
||||
imagesDir: paths.imagesDir,
|
||||
digest,
|
||||
version: options.version,
|
||||
now,
|
||||
imagePath: (v) => paths.runtimeIsoPath(v),
|
||||
log,
|
||||
});
|
||||
const { version, image: iso, manifest: manifestPath } = target;
|
||||
if (target.existing) return { version, iso, manifest: manifestPath, created: false };
|
||||
|
||||
const lock = await acquireLock(join(paths.imagesDir, ".runtime-build.lock"), "runtime ISO build");
|
||||
try {
|
||||
@@ -263,10 +267,7 @@ export async function buildRuntimeIso(options: RuntimeBuildOptions = {}): Promis
|
||||
content_digest: digest,
|
||||
files: all.map((file): RuntimeFile => ({ path: file.path, sha256: file.sha256, size_bytes: file.size })),
|
||||
};
|
||||
// Manifest first, then the image appears under the name the control plane looks for.
|
||||
await writeManifest(manifestPath, manifest);
|
||||
await replaceFile(partial, iso);
|
||||
await chmod(iso, 0o444);
|
||||
await publishImage(partial, iso, manifestPath, manifest);
|
||||
log(`done: ${iso} (sha256 ${manifest.sha256})`);
|
||||
return { version, iso, manifest: manifestPath, created: true };
|
||||
} catch (error) {
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
* The builder VM's seed disk. It is one ISO labelled `cidata`: cloud-init's
|
||||
* NoCloud datasource reads user-data and meta-data from it, and the
|
||||
* provisioner mounts it again to reach its own files, so the builder needs
|
||||
* one CD-ROM and no second payload disk.
|
||||
* this one read-only disk (a virtio disk, as every ISO of this project is
|
||||
* attached: the vm-manager's isoDriveArg) and no second payload disk.
|
||||
*/
|
||||
import type { IsoEntry } from "@invisible-dots/iso";
|
||||
import { downloadFileName, type GuestPins } from "./pins.js";
|
||||
@@ -53,6 +54,7 @@ export function pinsEnv(pins: GuestPins, python: PythonLock): string {
|
||||
["ENGINE_LOCK", SEED_ENGINE_LOCK],
|
||||
["ENGINE_BUILD", SEED_ENGINE_BUILD],
|
||||
["BROWSER_BUILD", SEED_BROWSER_BUILD],
|
||||
["APT_SNAPSHOT", pins.apt_snapshot],
|
||||
["APT_PACKAGES", pins.apt_packages.join(" ")],
|
||||
];
|
||||
return values
|
||||
|
||||
@@ -58,6 +58,7 @@ beforeEach(async () => {
|
||||
pins = {
|
||||
uv: { version: "0.12.22", url: http.url(`/uv/${UV_FILE}`), shasums_url: http.url(`/uv/${UV_FILE}.sha256`), shasums_entry: UV_FILE, sha256: sha256(UV) },
|
||||
tunnel: { version: "2.18.0", url: http.url(`/tunnel/${TUNNEL_FILE}`), sha256: sha256(TUNNEL) },
|
||||
apt_snapshot: "20261010T000000Z",
|
||||
apt_packages: ["xvfb", "imagemagick"],
|
||||
};
|
||||
logs = [];
|
||||
@@ -133,6 +134,7 @@ describe("buildGoldenImage", () => {
|
||||
uv: { version: "0.12.22", sha256: sha256(UV) },
|
||||
"invisible-playwright-mcp": "0.71.1",
|
||||
"invisible-playwright": "0.32.0",
|
||||
apt_snapshot: "20261010T000000Z",
|
||||
apt_packages: ["xvfb", "imagemagick"],
|
||||
},
|
||||
// The engine's lock, which the runtime disk's copy must equal.
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
// Checks of the files that run inside the guest. A mistake in them shows up
|
||||
// only inside a booted VM, minutes into a build or on every Dot's boot, so
|
||||
// the cheap ones are caught here.
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { execFileSync, spawnSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { delimiter, join } from "node:path";
|
||||
import { GUEST_PATHS } from "@invisible-dots/shared";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { BUILDER_PYTHON_LOCK, defaultAssetRoot, GUEST_ASSETS, GUEST_UNITS, unitAsset } from "../src/assets.js";
|
||||
@@ -115,6 +115,62 @@ describe("the sudo grants (architecture section 4.1)", () => {
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* dot-install as sudo runs it, with `id` answering root and `apt-get` a stand-in that records its arguments: what
|
||||
* reaches apt, or that nothing does.
|
||||
*/
|
||||
describe.skipIf(!bashUsable)("dot-install, the one command dot may run as root", () => {
|
||||
function dotInstall(...names: string[]): { status: number; stderr: string; apt: string[] } {
|
||||
const stubs = mkdtempSync(join(tmpdir(), "idots-dot-install-"));
|
||||
try {
|
||||
const calls = join(stubs, "apt-calls");
|
||||
writeFileSync(join(stubs, "id"), "#!/bin/sh\necho 0\n", { mode: 0o755 });
|
||||
writeFileSync(join(stubs, "apt-get"), `#!/bin/sh\necho "$*" >> '${calls.replace(/\\/g, "/")}'\n`, { mode: 0o755 });
|
||||
// Windows spells the variable Path, and an environment holding both spellings is ambiguous.
|
||||
const pathKey = Object.keys(process.env).find((key) => key.toUpperCase() === "PATH") ?? "PATH";
|
||||
const run = spawnSync("bash", [join(root, "runtime", "dot-install.sh"), ...names], {
|
||||
env: { ...process.env, [pathKey]: `${stubs}${delimiter}${process.env[pathKey] ?? ""}` },
|
||||
encoding: "utf8",
|
||||
timeout: 20_000,
|
||||
});
|
||||
const apt = existsSync(calls) ? readFileSync(calls, "utf8").trim().split("\n") : [];
|
||||
return { status: run.status ?? -1, stderr: run.stderr, apt };
|
||||
} finally {
|
||||
rmSync(stubs, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
const lock = "-o DPkg::Lock::Timeout=300";
|
||||
|
||||
it("installs package names, those that end in + included", () => {
|
||||
expect(dotInstall("nodejs", "libgtk-3-0t64", "python3.12", "g++", "ncbi-blast+")).toEqual({
|
||||
status: 0,
|
||||
stderr: "",
|
||||
apt: [`${lock} update -qq`, `${lock} install -y nodejs libgtk-3-0t64 python3.12 g++ ncbi-blast+`],
|
||||
});
|
||||
});
|
||||
|
||||
it("passes a trailing + on, which apt reads as install", () => {
|
||||
expect(dotInstall("foo+").apt).toEqual([`${lock} update -qq`, `${lock} install -y foo+`]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["a trailing -, which apt reads as remove", "coreutils-"],
|
||||
["an option", "-o"],
|
||||
["a long option", "--allow-remove-essential"],
|
||||
["a version", "foo=1.0"],
|
||||
["a release", "foo/noble"],
|
||||
["an architecture", "foo:i386"],
|
||||
["upper case", "Foo"],
|
||||
["one character", "a"],
|
||||
["an empty word", ""],
|
||||
])("refuses %s, and nothing reaches apt", (_what, name) => {
|
||||
const run = dotInstall("nodejs", name);
|
||||
expect(run.status).toBe(2);
|
||||
expect(run.stderr).toContain(`"${name}" is not a package name`);
|
||||
expect(run.apt).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the Dot's browser", () => {
|
||||
const provision = text("builder/provision.sh");
|
||||
const build = text("builder/build-browser-env.sh");
|
||||
@@ -123,7 +179,7 @@ describe("the Dot's browser", () => {
|
||||
expect(provision).toContain('bash "$payload/$BROWSER_BUILD" "$payload/$PYTHON_LOCK" "$mcp_env"');
|
||||
expect(provision).toContain("mcp_env=/home/dot/.local/share/invisible-dots/mcp\n");
|
||||
expect(build).toContain('as_dot uv venv --quiet --python /usr/bin/python3 "$env_dir"');
|
||||
expect(build).toContain('as_dot uv pip install --python "$env_dir/bin/python" --require-hashes -r "$dot_lock"');
|
||||
expect(build).toContain('as_dot uv pip install --no-cache --python "$env_dir/bin/python" --require-hashes --only-binary :all: -r "$dot_lock"');
|
||||
expect(build).not.toMatch(/uv tool install/);
|
||||
expect(build).toContain('as_dot ln -sfn "$env_dir/bin/invisible-playwright-mcp" "$bin_dir/invisible-playwright-mcp"');
|
||||
expect(build).toContain('as_dot "$env_dir/bin/invisible-playwright" fetch');
|
||||
|
||||
@@ -58,6 +58,13 @@ describe("pin validation", () => {
|
||||
expect(() => parseGuestPins({ ...good, apt_packages: ["xvfb", "x; rm -rf /"] })).toThrow(/apt_packages\[1\]/);
|
||||
});
|
||||
|
||||
it("takes the packages from a pinned snapshot of the archive, and refuses one that is not a snapshot time", () => {
|
||||
expect(GUEST_PINS.apt_snapshot).toMatch(/^\d{8}T\d{6}Z$/);
|
||||
expect(() => parseGuestPins({ ...good, apt_snapshot: "latest" })).toThrow(/apt_snapshot/);
|
||||
const { apt_snapshot: _snapshot, ...unpinned } = good;
|
||||
expect(() => parseGuestPins(unpinned)).toThrow(/apt_snapshot/);
|
||||
});
|
||||
|
||||
it("refuses python_packages, which would be a second copy of the lock's versions", () => {
|
||||
expect(() => parseGuestPins({ ...good, python_packages: { "invisible-playwright-mcp": "0.70.2" } })).toThrow(/mcp-requirements\.lock/);
|
||||
});
|
||||
|
||||
@@ -8,7 +8,7 @@ import { fileBytes, listFiles, parseIso } from "../../../packages/iso/test/iso-r
|
||||
import { defaultAssetRoot, GUEST_UNITS } from "../src/assets.js";
|
||||
import { readManifest, verifyImage, type RuntimeManifest } from "../src/manifest.js";
|
||||
import { RUNTIME_ISO_LABEL } from "@invisible-dots/vm-manager";
|
||||
import { agentdBuildCommand, assertLinuxAmd64Elf, buildRuntimeIso, defaultRuntimeInputs, runtimeFiles, type RuntimeInputs } from "../src/runtime.js";
|
||||
import { AGENTD_BUILD_SHELL, agentdBuildCommand, assertLinuxAmd64Elf, buildRuntimeIso, defaultRuntimeInputs, runtimeFiles, type RuntimeInputs } from "../src/runtime.js";
|
||||
import { sha256 } from "./http-fixture.js";
|
||||
|
||||
/** The first bytes of an ELF executable for `machine` (0x3e is x86-64, 0xb7 arm64). */
|
||||
@@ -73,11 +73,26 @@ describe("agentdBuildCommand", () => {
|
||||
join("repo", "root", "guest", "dot-agentd"),
|
||||
"build",
|
||||
"-trimpath",
|
||||
"-buildvcs=false",
|
||||
"-ldflags=-s -w",
|
||||
"-o",
|
||||
defaultRuntimeInputs(join("repo", "root")).agentdBinary,
|
||||
"./cmd/dot-agentd",
|
||||
]);
|
||||
});
|
||||
|
||||
it("builds into another file when asked, with the same flags", () => {
|
||||
const build = agentdBuildCommand(join("repo", "root"), join("elsewhere", "dot-agentd"));
|
||||
expect(build.args).toEqual([...agentdBuildCommand(join("repo", "root")).args.slice(0, -3), "-o", join("elsewhere", "dot-agentd"), "./cmd/dot-agentd"]);
|
||||
});
|
||||
|
||||
it("is the build the shell line quotes: its environment, then go build with its flags, run in guest/dot-agentd", () => {
|
||||
const build = agentdBuildCommand(join("repo", "root"), "bin/dot-agentd");
|
||||
const env = Object.entries(build.env).map(([name, value]) => `${name}=${value}`);
|
||||
// The shell line's one quoted word, -ldflags="-s -w", is the single argument -ldflags=-s -w.
|
||||
const args = build.args.slice(2).map((arg) => (arg.includes(" ") ? arg.replace(/=(.*)$/, '="$1"') : arg));
|
||||
expect(AGENTD_BUILD_SHELL).toBe([...env, "go", ...args].join(" "));
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildRuntimeIso", () => {
|
||||
@@ -108,6 +123,7 @@ describe("buildRuntimeIso", () => {
|
||||
"engine/nanobot/templates/agent/tool_contract.md",
|
||||
"engine/skills/web-forms/SKILL.md",
|
||||
"install.sh",
|
||||
"vmproxy.py",
|
||||
...GUEST_UNITS.map((unit) => `units/${unit}`),
|
||||
].sort(),
|
||||
);
|
||||
@@ -115,6 +131,7 @@ describe("buildRuntimeIso", () => {
|
||||
expect(read("VERSION").toString()).toBe(`${result.version}\n`);
|
||||
expect(read("bin/dot-agentd")).toEqual(elf(0x3e));
|
||||
expect(read("install.sh")).toEqual(await readFile(join(defaultAssetRoot(), "runtime", "install.sh")));
|
||||
expect(read("vmproxy.py")).toEqual(await readFile(join(defaultAssetRoot(), "runtime", "vmproxy.py")));
|
||||
expect(read("bin/dot-desktop")).toEqual(await readFile(join(defaultAssetRoot(), "runtime", "dot-desktop.sh")));
|
||||
expect(read("units/dot-agentd.service")).toEqual(await readFile(join(defaultAssetRoot(), "units", "dot-agentd.service")));
|
||||
// The engine's source as it is.
|
||||
|
||||
@@ -32,6 +32,7 @@ describe("pins.env", () => {
|
||||
expect(text).toContain("ENGINE_LOCK='engine-requirements.lock'\n");
|
||||
expect(text).toContain("ENGINE_BUILD='build-engine-env.sh'\n");
|
||||
expect(text).toContain("BROWSER_BUILD='build-browser-env.sh'\n");
|
||||
expect(text).toContain(`APT_SNAPSHOT='${GUEST_PINS.apt_snapshot}'\n`);
|
||||
expect(text).toContain(`APT_PACKAGES='${GUEST_PINS.apt_packages.join(" ")}'\n`);
|
||||
|
||||
const provision = (await readGuestAsset(defaultAssetRoot(), BUILDER_PROVISION)).toString("utf8");
|
||||
|
||||
@@ -43,13 +43,31 @@ describe("installedComponents", () => {
|
||||
});
|
||||
|
||||
describe("followSerialLog", () => {
|
||||
it("reports progress lines as they are appended, including a file that appears late", async () => {
|
||||
it("reports nothing and stops cleanly while the file does not exist", async () => {
|
||||
const seen: string[] = [];
|
||||
const follower = followSerialLog(join(dir, "serial.log"), (step) => seen.push(step), 10);
|
||||
await follower.stop();
|
||||
expect(seen).toEqual([]);
|
||||
expect(follower.tail(1)).toEqual([]);
|
||||
});
|
||||
|
||||
it("reports progress lines as they are appended, from a file that appears after it started", async () => {
|
||||
const path = join(dir, "serial.log");
|
||||
const seen: string[] = [];
|
||||
const follower = followSerialLog(path, (step) => seen.push(step), 10);
|
||||
await new Promise((resolve) => setTimeout(resolve, 30));
|
||||
let progressed = () => {};
|
||||
const progress = () => new Promise<void>((resolve) => (progressed = resolve));
|
||||
const follower = followSerialLog(
|
||||
path,
|
||||
(step) => {
|
||||
seen.push(step);
|
||||
progressed();
|
||||
},
|
||||
10,
|
||||
);
|
||||
const first = progress();
|
||||
await appendFile(path, "booting\r\nidots-build: installing packages\r\nidots-bu");
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
await first;
|
||||
// The half line is held back until its end arrives.
|
||||
expect(seen).toEqual(["installing packages"]);
|
||||
await appendFile(path, "ild: installing Node 24\nIDOTS-BUILD-RESULT: ok");
|
||||
await follower.stop();
|
||||
|
||||
@@ -129,7 +129,7 @@ engine) and at a launch (the egress address, for the timezone, and the library's
|
||||
|
||||
## Run it
|
||||
|
||||
Only docker is needed, with the images `golang:1.26` and `ubuntu:24.04` (pulled when
|
||||
Only docker is needed, with the images `golang:1.26.8` and `ubuntu:24.04` (pulled when
|
||||
absent; the container downloads the pinned `uv` and the locked wheels).
|
||||
|
||||
```sh
|
||||
@@ -157,7 +157,7 @@ removes the volume and the container on exit.
|
||||
|
||||
| file | what it is |
|
||||
|---|---|
|
||||
| `run.sh` | the entry: runs dot-agentd's `privileged` Go tests as root in `golang:1.26` (they run work as another user; a skipped one fails the run), builds `dot-agentd` there, starts `ubuntu:24.04` with the tree, checks the exit status and the summary line; `--suite browser` runs the browser smoke |
|
||||
| `run.sh` | the entry: runs dot-agentd's `privileged` Go tests as root in `golang:1.26.8` (they run work as another user; a skipped one fails the run), builds `dot-agentd` there, starts `ubuntu:24.04` with the tree, checks the exit status and the summary line; `--suite browser` runs the browser smoke |
|
||||
| `prepare-engine.sh` | in the container: the golden image's users, `uv`, the engine's environment, the staged engine source (and for the browser suite the apt packages and the browser); then it runs the suite's checks |
|
||||
| `lib.sh` | what both suites share: the guest laid out as `install.sh` lays it out, `dot-agentd` and the engine started and restarted, the key and config push, the event stream and the helpers that read it |
|
||||
| `smoke.sh` | the engine smoke's checks; prints `PASS:` or `FAIL:` per check and the summary line |
|
||||
|
||||
@@ -29,7 +29,8 @@ esac
|
||||
export AGENTD_BIN
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update -qq >/dev/null
|
||||
# nftables and iproute2: the check of the VM proxy's firewall runs install.sh's own generator and nft on its output.
|
||||
# nftables and iproute2: the checks of the VM proxy run the runtime disk's vmproxy.py (it reads the default route)
|
||||
# and nft on its output.
|
||||
apt-get install -y -qq sudo procps jq curl ca-certificates python3 nftables iproute2 >/dev/null
|
||||
|
||||
builder=$TREE/guest/image-builder/builder
|
||||
@@ -73,8 +74,11 @@ print('imported', len(names), 'modules from', nanobot.__file__)
|
||||
"
|
||||
|
||||
if [ "$suite" = browser ]; then
|
||||
# The packages of the golden image (pins.json), as provision.sh installs them.
|
||||
apt-get install -y -qq --no-install-recommends $(jq -r '.apt_packages[]' "$TREE/guest/image-builder/pins.json") >/dev/null
|
||||
# The packages of the golden image (pins.json), as provision.sh installs them: from the archive as of the pinned
|
||||
# snapshot (reached over https, which the ca-certificates installed above make possible).
|
||||
snapshot=$(jq -r '.apt_snapshot' "$TREE/guest/image-builder/pins.json")
|
||||
apt-get --snapshot "$snapshot" --error-on=any update -qq >/dev/null
|
||||
apt-get --snapshot "$snapshot" install -y -qq --no-install-recommends $(jq -r '.apt_packages[]' "$TREE/guest/image-builder/pins.json") >/dev/null
|
||||
# The Dot's browser, with the script provision.sh runs on the same lock. dot reaches its home, not the tree.
|
||||
bash "$builder/build-browser-env.sh" "$builder/mcp-requirements.lock" /home/dot/.local/share/invisible-dots/mcp
|
||||
fi
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
# The engine smoke's entry point: builds dot-agentd from a source tree in
|
||||
# golang:1.26, starts ubuntu:24.04 with the tree mounted, builds the engine's
|
||||
# golang:1.26.8, starts ubuntu:24.04 with the tree mounted, builds the engine's
|
||||
# environment there as provision.sh does (builder/build-engine-env.sh on the
|
||||
# hashed lock), stages the engine's source as the runtime ISO does, and runs
|
||||
# the checks of smoke.sh against a Dot.
|
||||
@@ -78,21 +78,21 @@ fi
|
||||
# --init: a process the tests kill is a zombie until its parent reaps it, and the tests ask whether it is gone.
|
||||
if ! privileged=$(docker run --rm --init -v "$run_id":/work "${tree_mount[@]}" \
|
||||
-e CGO_ENABLED=0 -e GOCACHE=/work/gocache -e GOTOOLCHAIN=local -e GOFLAGS=-buildvcs=false \
|
||||
-w "$tree/guest/dot-agentd" golang:1.26 \
|
||||
-w "$tree/guest/dot-agentd" golang:1.26.8 \
|
||||
go test -count=1 -v -tags privileged ./... 2>&1) || printf '%s\n' "$privileged" | grep -q -e '--- SKIP'; then
|
||||
printf '%s\n' "$privileged" | grep -v -e '^=== ' | tail -40 >&2
|
||||
echo "run.sh: dot-agentd's privileged tests did not pass, or one was skipped" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# dot-agentd for the guest, as CI's go job builds it. GOTOOLCHAIN=local: the
|
||||
# image's Go is the one that builds, nothing is downloaded. -buildvcs=false:
|
||||
# the mounted tree belongs to another user, and a build must not depend on git.
|
||||
# dot-agentd for the guest, with the one build (guest/dot-agentd/README.md) and the Go
|
||||
# go.mod's toolchain names. GOTOOLCHAIN=local: the image's Go is the one that
|
||||
# builds, nothing is downloaded.
|
||||
docker run --rm -v "$run_id":/work "${tree_mount[@]}" \
|
||||
-e CGO_ENABLED=0 -e GOOS=linux -e GOARCH=amd64 \
|
||||
-e GOCACHE=/work/gocache -e GOTOOLCHAIN=local -e GOFLAGS=-buildvcs=false \
|
||||
-w "$tree/guest/dot-agentd" golang:1.26 \
|
||||
go build -trimpath -ldflags="-s -w" -o /work/dot-agentd ./cmd/dot-agentd
|
||||
-e GOCACHE=/work/gocache -e GOTOOLCHAIN=local \
|
||||
-w "$tree/guest/dot-agentd" golang:1.26.8 \
|
||||
go build -trimpath -buildvcs=false -ldflags="-s -w" -o /work/dot-agentd ./cmd/dot-agentd
|
||||
|
||||
# The checks of the tree under test, in a clean Ubuntu. PIN_REMOVALS reaches
|
||||
# smoke.sh only when it is set here (see the top of smoke.sh).
|
||||
|
||||
@@ -58,18 +58,39 @@ lay_out_guest
|
||||
# The engine needs no privilege: no sudoers rule, no config directory.
|
||||
check "the engine has no sudo rule (no /etc/sudoers.d/invisible-dots-engine, no sudo for dotengine)" "[ ! -e /etc/sudoers.d/invisible-dots-engine ] && ! su -s /bin/bash dotengine -c 'sudo -n true' >/dev/null 2>&1"
|
||||
check "no sudoers file names dotengine (the old engine's rule is gone with the config installer)" "! grep -rqs dotengine /etc/sudoers /etc/sudoers.d"
|
||||
# --- the VM proxy's firewall, as install.sh writes it, is a ruleset nft takes ---
|
||||
# --- the VM proxy, as install.sh sets it up: the runtime disk's vmproxy.py on a config that has a proxy ---
|
||||
# A rule nft refused stopped install.sh (set -e) before the tunnel started: the Dot went out directly with its proxy
|
||||
# set. install.sh's own generator runs here on a config that has a proxy, read from a copy of its path.
|
||||
# set. And a proxy that cannot be set up is reported for what it is: install.sh logs vmproxy.py's reason and lets
|
||||
# nothing out.
|
||||
VMPROXY="$TREE/guest/image-builder/runtime/vmproxy.py"
|
||||
vmproxy() { # proxy value, [PATH]: vmproxy.py on a config with that proxy; what it wrote is in $VMPROXY_OUT
|
||||
printf '{"dotId":"dot_smoke","token":"t","proxy":"%s"}' "$1" > /tmp/vmproxy-config.json
|
||||
VMPROXY_OUT=$(mktemp -d)
|
||||
PATH="${2:-$PATH}" python3 "$VMPROXY" /tmp/vmproxy-config.json "$VMPROXY_OUT" 2> /tmp/vmproxy-stderr
|
||||
}
|
||||
vmproxy_rules_ok() {
|
||||
local out
|
||||
out=$(mktemp -d)
|
||||
printf '{"dotId":"dot_smoke","token":"t","proxy":"socks5://user:p%%[email protected]:1080"}' > /tmp/vmproxy-config.json
|
||||
sed -n "/<<'PY'\$/,/^PY\$/p" "$TREE/guest/image-builder/runtime/install.sh" | sed '1d;$d' \
|
||||
| sed 's#/etc/invisible-dots/config.json#/tmp/vmproxy-config.json#' > /tmp/vmproxy-gen.py
|
||||
python3 /tmp/vmproxy-gen.py "$out" && [ -s "$out/allow.nft" ] && [ -s "$out/hev.yml" ] && nft -c -f "$out/allow.nft"
|
||||
vmproxy 'socks5://user:p%[email protected]:1080' && [ -s "$VMPROXY_OUT/allow.nft" ] && [ -s "$VMPROXY_OUT/hev.yml" ] \
|
||||
&& [ -s "$VMPROXY_OUT/up.sh" ] && grep -q 'password: "p@ss"' "$VMPROXY_OUT/hev.yml" && nft -c -f "$VMPROXY_OUT/allow.nft"
|
||||
}
|
||||
check "the VM proxy's firewall that install.sh writes for a proxy is a ruleset nft accepts" "vmproxy_rules_ok"
|
||||
vmproxy_refuses() { # exit code, words of the reason, proxy value, [PATH]: that code, that reason, and nothing written
|
||||
vmproxy "$3" "${4:-}"
|
||||
local code=$?
|
||||
if [ "$code" -eq "$1" ] && grep -q "$2" /tmp/vmproxy-stderr && [ -z "$(ls -A "$VMPROXY_OUT")" ]; then return 0; fi
|
||||
echo "exit $code: $(cat /tmp/vmproxy-stderr)"
|
||||
return 1
|
||||
}
|
||||
# An ip that shows a default route without a gateway, as a VPN's or a point-to-point link's would be.
|
||||
mkdir -p /tmp/vmproxy-noroute
|
||||
printf '#!/bin/sh\necho "default dev wg0 scope link"\n' > /tmp/vmproxy-noroute/ip
|
||||
chmod 0755 /tmp/vmproxy-noroute/ip
|
||||
NOROUTE_PATH="/tmp/vmproxy-noroute:$PATH"
|
||||
check "a proxy that is not socks5://host:port is reported as that (exit 3) and nothing is written" \
|
||||
"vmproxy_refuses 3 'is not socks5://' 'http://10.0.2.2:1080' && vmproxy_refuses 3 'is not socks5://' 'socks5://10.0.2.2:99999'"
|
||||
check "a proxy whose host does not resolve is reported as that (exit 4), not as a bad value" \
|
||||
"vmproxy_refuses 4 'does not resolve' 'socks5://proxy.invalid:1080'"
|
||||
check "a VM with no default route through a gateway is reported as that (exit 5), not as a bad value" \
|
||||
"vmproxy_refuses 5 'no default IPv4 route through a gateway' 'socks5://10.0.2.2:1080' \"\$NOROUTE_PATH\""
|
||||
check "dotengine is in no group but its own and dot" "[ \"\$(id -nG dotengine | tr ' ' '\n' | sort | tr '\n' ' ')\" = 'dot dotengine ' ]"
|
||||
write_host_token
|
||||
check "the token file is dot-agentd's own (dotagentd, 0600): neither the engine nor dot, the user of the model's commands, can read it" "[ \"\$(stat -c '%U:%G %a' /etc/invisible-dots/config.json)\" = 'dotagentd:dotagentd 600' ] && ! su -s /bin/bash dotengine -c 'cat /etc/invisible-dots/config.json' >/dev/null 2>&1 && ! su -s /bin/bash dot -c 'cat /etc/invisible-dots/config.json' >/dev/null 2>&1"
|
||||
|
||||
+3
-3
@@ -106,7 +106,7 @@ git clone https://github.com/feder-cr/invisible_dots && cd invisible_dots
|
||||
npm ci
|
||||
npm run build --workspace @invisible-dots/cli
|
||||
npm run build --workspace @invisible-dots/web # skip with E2E_WEB=0
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go -C guest/dot-agentd build -trimpath -o bin/dot-agentd ./cmd/dot-agentd
|
||||
(cd guest/dot-agentd && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false -ldflags="-s -w" -o bin/dot-agentd ./cmd/dot-agentd)
|
||||
|
||||
# The key goes in a file only you can read; the run never prints it.
|
||||
install -m 600 /dev/null ~/openrouter.key && $EDITOR ~/openrouter.key
|
||||
@@ -146,8 +146,8 @@ device's group, as on a real host; no `--privileged`.
|
||||
```bash
|
||||
# dot-agentd needs Go, which the host image deliberately lacks
|
||||
docker run --rm -v "$PWD/guest/dot-agentd:/src" -w /src \
|
||||
-e CGO_ENABLED=0 -e GOOS=linux -e GOARCH=amd64 -e GOFLAGS=-buildvcs=false \
|
||||
golang:1.26 go build -trimpath -o bin/dot-agentd ./cmd/dot-agentd
|
||||
-e CGO_ENABLED=0 -e GOOS=linux -e GOARCH=amd64 \
|
||||
golang:1.26.8 go build -trimpath -buildvcs=false -ldflags="-s -w" -o bin/dot-agentd ./cmd/dot-agentd
|
||||
|
||||
# The container's user takes your uid and gid, so it can read ~/openrouter.key (mode 0600) as you do.
|
||||
docker build -t idots-linux-host --build-arg HOST_UID="$(id -u)" --build-arg HOST_GID="$(id -g)" tests/e2e/linux-host
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
/**
|
||||
* dot-agentd has one build (the image builder's agentdBuildCommand, AGENTD_BUILD_SHELL as a shell line) and one Go
|
||||
* (go.mod's `toolchain`): the same source then gives the same bytes, so the same runtime disk, wherever it is built.
|
||||
* These checks keep every place that builds it or tells a person how from drifting to another spelling or another Go.
|
||||
*/
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { AGENTD_BUILD_SHELL } from "@invisible-dots/image-builder";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
const repo = resolve(fileURLToPath(new URL(".", import.meta.url)), "../..");
|
||||
const text = (path: string) => readFileSync(join(repo, path), "utf8").replace(/\r\n/g, "\n");
|
||||
|
||||
/** Every file that builds dot-agentd, or tells how. */
|
||||
const BUILDERS = [
|
||||
".github/workflows/tests.yml",
|
||||
".github/workflows/golden-image.yml",
|
||||
"guest/dot-agentd/README.md",
|
||||
"guest/image-builder/README.md",
|
||||
"guest/image-builder/test/smoke/run.sh",
|
||||
"tests/e2e/README.md",
|
||||
];
|
||||
/** Those that run it with the environment on the same line, so quote the whole shell line. */
|
||||
const WHOLE_LINE = [".github/workflows/tests.yml", ".github/workflows/golden-image.yml", "guest/dot-agentd/README.md", "guest/image-builder/README.md", "tests/e2e/README.md"];
|
||||
/** The flags, as a line that passes the environment another way (docker's -e) still spells them. */
|
||||
const FLAGS = AGENTD_BUILD_SHELL.slice(AGENTD_BUILD_SHELL.indexOf("go build"), AGENTD_BUILD_SHELL.indexOf(" -o ") + 4);
|
||||
|
||||
describe("the one build of dot-agentd", () => {
|
||||
const toolchain = /^toolchain go(\d+\.\d+\.\d+)$/m.exec(text("guest/dot-agentd/go.mod"))?.[1];
|
||||
|
||||
it("go.mod names the Go that builds it", () => {
|
||||
expect(toolchain).toMatch(/^\d+\.\d+\.\d+$/);
|
||||
});
|
||||
|
||||
it.each(BUILDERS)("%s builds it with the same flags, and nothing else that builds it", (path) => {
|
||||
const builds = text(path)
|
||||
.split("\n")
|
||||
.filter((line) => /\bgo (-C \S+ )?build\b/.test(line));
|
||||
expect(builds.length, path).toBeGreaterThan(0);
|
||||
for (const line of builds) expect(line, path).toContain(FLAGS);
|
||||
});
|
||||
|
||||
it.each(WHOLE_LINE)("%s quotes the whole shell line", (path) => {
|
||||
expect(text(path)).toContain(AGENTD_BUILD_SHELL);
|
||||
});
|
||||
|
||||
it.each(BUILDERS)("%s uses the Go of go.mod's toolchain", (path) => {
|
||||
const content = text(path);
|
||||
for (const image of content.match(/golang:[\w.-]+/g) ?? []) expect(image, path).toBe(`golang:${toolchain}`);
|
||||
// setup-go takes the toolchain line from go.mod (v6 reads `toolchain`, v5 read only the `go` line).
|
||||
for (const use of content.match(/actions\/setup-go@\S+/g) ?? []) expect(use, path).toBe("actions/setup-go@v6");
|
||||
for (const pin of content.match(/go-version(-file)?:.*/g) ?? []) expect(pin, path).toBe("go-version-file: guest/dot-agentd/go.mod");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,73 @@
|
||||
/**
|
||||
* The host-to-guest contract of architecture section 5 is written down once, in packages/shared (protocol.ts), and
|
||||
* held twice more in the guest: by dot-agentd's Go sources (its routes and its defaults) and by the engine's Python
|
||||
* (the sockets and the workspace it reaches when its unit names none). Neither copy imports the TypeScript, so these
|
||||
* checks read them and compare: a route or a path changed on one side only fails here, not inside every Dot.
|
||||
*/
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { AGENTD_ROUTES, EXEC_OUTPUT_CAP_BYTES, GUEST_DISPLAY, GUEST_PATHS, GUEST_PORT, GUEST_PROOF_CONTEXT } from "@invisible-dots/shared";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
const repo = resolve(fileURLToPath(new URL(".", import.meta.url)), "../..");
|
||||
const read = (path: string) => readFileSync(join(repo, path), "utf8").replace(/\r\n/g, "\n");
|
||||
const agentd = (file: string) => read(join("guest/dot-agentd/internal/agentd", file));
|
||||
|
||||
/** The value of a Go constant: a string literal, a sum of literals and constants of the same sources, or `a << b`. */
|
||||
function goConstant(sources: string, name: string): string | number {
|
||||
const match = new RegExp(`^\\s*(?:const\\s+)?${name}\\s*=\\s*(.+?)\\s*(//.*)?$`, "m").exec(sources);
|
||||
if (!match) throw new Error(`no Go constant ${name}`);
|
||||
const expression = match[1]!;
|
||||
const shift = /^(\d+)\s*<<\s*(\d+)$/.exec(expression);
|
||||
if (shift) return Number(shift[1]) * 2 ** Number(shift[2]);
|
||||
return expression
|
||||
.split("+")
|
||||
.map((term) => term.trim())
|
||||
.map((term) => (term.startsWith('"') ? (JSON.parse(term) as string) : String(goConstant(sources, term))))
|
||||
.join("");
|
||||
}
|
||||
|
||||
/** The value of a top-level Python string constant. */
|
||||
function pyConstant(source: string, name: string): string | undefined {
|
||||
return new RegExp(`^${name} = "([^"]*)"$`, "m").exec(source)?.[1];
|
||||
}
|
||||
|
||||
describe("dot-agentd's Go copy of the protocol", () => {
|
||||
const go = ["server.go", "exec.go", "proof.go"].map(agentd).join("\n");
|
||||
|
||||
it("serves exactly the routes of AGENTD_ROUTES, plus the engine's process route on its socket", () => {
|
||||
const served = new Set([...go.matchAll(/mux\.Handle(?:Func)?\("(?:[A-Z]+ )?(\/[^"]*)"/g)].map((m) => m[1]!));
|
||||
served.delete("/");
|
||||
// The agent proxy is registered by its prefix constant, with and without the trailing slash.
|
||||
for (const handle of go.matchAll(/mux\.Handle\(agentProxyPrefix(\+"\/")?,/g)) served.add(`${goConstant(go, "agentProxyPrefix")}${handle[1] ? "/" : ""}`);
|
||||
expect([...served].sort()).toEqual([...Object.values(AGENTD_ROUTES), `${AGENTD_ROUTES.agent}/`, "/v1/proc"].sort());
|
||||
});
|
||||
|
||||
it("listens on GUEST_PORT, caps exec output at EXEC_OUTPUT_CAP_BYTES, and signs the proof with GUEST_PROOF_CONTEXT", () => {
|
||||
expect(goConstant(go, "DefaultListenAddr")).toBe(`0.0.0.0:${GUEST_PORT}`);
|
||||
expect(goConstant(go, "ExecOutputCap")).toBe(EXEC_OUTPUT_CAP_BYTES);
|
||||
expect(goConstant(go, "proofContext")).toBe(GUEST_PROOF_CONTEXT);
|
||||
});
|
||||
|
||||
it("defaults to the guest paths and the display of GUEST_PATHS and GUEST_DISPLAY", () => {
|
||||
expect(goConstant(go, "DefaultConfigPath")).toBe(GUEST_PATHS.config);
|
||||
expect(goConstant(go, "DefaultHome")).toBe(GUEST_PATHS.home);
|
||||
expect(goConstant(go, "DefaultRunDir")).toBe(GUEST_PATHS.runDir);
|
||||
expect(goConstant(go, "DefaultAgentdSocket")).toBe(GUEST_PATHS.agentdSocket);
|
||||
expect(goConstant(go, "DefaultAgentSocket")).toBe(GUEST_PATHS.agentSocket);
|
||||
expect(goConstant(go, "DefaultDisplay")).toBe(GUEST_DISPLAY);
|
||||
});
|
||||
});
|
||||
|
||||
describe("the engine's defaults", () => {
|
||||
const computer = read("invisible_engine_dots/nanobot/dots/computer.py");
|
||||
const main = read("invisible_engine_dots/nanobot/dots/main.py");
|
||||
|
||||
it("are the guest paths of GUEST_PATHS", () => {
|
||||
expect(pyConstant(computer, "DEFAULT_AGENTD_SOCKET")).toBe(GUEST_PATHS.agentdSocket);
|
||||
expect(pyConstant(computer, "DEFAULT_WORKSPACE")).toBe(GUEST_PATHS.workspace);
|
||||
expect(pyConstant(computer, "DEFAULT_AGENTD_BIN")).toBe(`${GUEST_PATHS.runtime}/bin/dot-agentd`);
|
||||
expect(pyConstant(main, "DEFAULT_AGENT_SOCKET")).toBe(GUEST_PATHS.agentSocket);
|
||||
});
|
||||
});
|
||||
@@ -79,7 +79,7 @@ const ALLOWED: Record<string, { owner: string; lines: string[] }> = {
|
||||
"guest/image-builder/src/runtime.ts": {
|
||||
owner:
|
||||
"agentdBuildCommand(): the guest's platform, the same on both hosts (not a branch): dot-agentd is built for linux/amd64 wherever the host runs",
|
||||
lines: ['env: { CGO_ENABLED: "0", GOOS: "linux", GOARCH: "amd64" },'],
|
||||
lines: ['const AGENTD_BUILD_ENV = { CGO_ENABLED: "0", GOOS: "linux", GOARCH: "amd64" } as const;'],
|
||||
},
|
||||
"packages/shared/src/files.ts": {
|
||||
owner: "permissionBitsEnforced() and restrictToOwner(): a private file is chmod 0600 on Linux and an owner-only ACL on Windows",
|
||||
|
||||
@@ -56,7 +56,7 @@ write_files:
|
||||
mountpoint -q /opt/invisible-dots || mount /opt/invisible-dots
|
||||
exec /opt/invisible-dots/install.sh
|
||||
|
||||
# The runtime ISO is the second CD-ROM drive (section 3.4). It is found by its
|
||||
# volume label, written by packages/iso, so the device name does not matter.
|
||||
# The runtime ISO is the third read-only virtio disk, after the seed (section 3.4). It is
|
||||
# found by its volume label, written by packages/iso, so the device name does not matter.
|
||||
mounts:
|
||||
- [ {{runtimeDevice}}, /opt/invisible-dots, iso9660, "ro,nofail,x-systemd.device-timeout=30s", "0", "0" ]
|
||||
+3
-1
@@ -16,7 +16,9 @@ export default defineConfig({
|
||||
// "@vitest-environment jsdom" line so that no other test pays for a DOM.
|
||||
// invisible_engine_dots/ holds the nanobot fork: Python, with its own pytest suite
|
||||
// (the engine job of the CI workflow); nothing of it runs here.
|
||||
exclude: ["**/node_modules/**", "**/.next/**", "invisible_engine_dots/**"],
|
||||
// A `*.linux.test.ts` drives a Linux program of the guest itself (dot-agentd, with its bash and its unix
|
||||
// sockets): it is collected on Linux only, so on another host it is neither run nor counted as skipped.
|
||||
exclude: ["**/node_modules/**", "**/.next/**", "invisible_engine_dots/**", ...(process.platform === "linux" ? [] : ["**/*.linux.test.ts"])],
|
||||
environment: "node",
|
||||
// Setup hooks boot an embedded PostgreSQL (PGlite, WebAssembly) and run
|
||||
// the migrations, or start servers and child processes. On a loaded
|
||||
|
||||
Reference in new issue
Block a user