diff --git a/.gitattributes b/.gitattributes index 4cee5c12..5820331b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,30 +1,2 @@ -# Everything text is stored and checked out with LF. Shell scripts, systemd -# units, cloud-init templates and the pre-push hook all end up on Linux, where a -# CRLF turns `#!/bin/sh` into `/bin/sh^M: bad interpreter` and a unit file into -# one systemd silently misreads. Git-for-Windows with core.autocrlf=true would -# otherwise convert them on checkout. * text=auto eol=lf - -.githooks/** text eol=lf -*.sh text eol=lf -*.service text eol=lf -*.timer text eol=lf -*.socket text eol=lf -*.target text eol=lf -*.env text eol=lf -# Guest files the image builder copies verbatim onto the runtime ISO or into -# the cloud-init seed, and the Go sources whose gofmt check rejects a CR. -*.tmpl text eol=lf -*.yaml text eol=lf -*.go text eol=lf -*.sql text eol=lf -guest/image-builder/** text eol=lf - -*.png binary -*.jpg binary -*.ico binary -*.iso binary -*.img binary -*.qcow2 binary -*.woff binary -*.woff2 binary +*.pdf binary diff --git a/.githooks/pre-push b/.githooks/pre-push deleted file mode 100755 index 640e2477..00000000 --- a/.githooks/pre-push +++ /dev/null @@ -1,151 +0,0 @@ -#!/bin/sh -# Pre-push gate. Two halves: -# -# 1. This repository's own suites, which the shared policy knows nothing -# about: the typecheck and vitest over the TypeScript workspace, and go -# test over dot-agentd, each counted against .github/test-floors.json. -# 2. The shared policy, `invisible_core.hooks` from the pinned invisible-core -# (.github/gates-requirements.txt): English, identity and the other checks -# declared in pyproject.toml under [tool.invisible.hooks]. That pyproject -# is configuration only; nothing here is a Python package. -# -# CI alone runs the rest of tests.yml: the engine's pytest (Linux only, see -# invisible_engine_dots), the PostgreSQL half of the pg suites, the web -# client's Playwright tests, the two smokes and the claims gate. -# -# Install once with: -# git config core.hooksPath .githooks -# Bypass a known-broken WIP push with `git push --no-verify`; that is the only -# escape hatch, and never for a branch that feeds a release. - -# No `set -e`: exit codes here have to be READ and explained, not turned into a -# silent abort. A gate that dies without a message is the worst shape of all. - -# Run from the repository root regardless of where `git push` was invoked. -cd "$(dirname "$0")/.." || exit 1 - -refuse() { - for line in "$@"; do - echo "[pre-push] $line" >&2 - done - exit 1 -} - -# git runs a hook with GIT_DIR, GIT_INDEX_FILE and friends pointing at this -# repository. A test that creates its own scratch repository would then -# address this one instead. Each suite runs in a subshell without them, and -# with stdin from /dev/null, because stdin is the list of refs being pushed -# and belongs to the policy in step 2. -without_git_env() { - ( - for name in $(env | sed -n 's/^\(GIT_[A-Za-z0-9_]*\)=.*/\1/p'); do - unset "$name" - done - "$@" - ) /dev/null 2>&1; then - refuse "REFUSED - npm is not on PATH, so the TypeScript suites cannot run." \ - "Install Node 24 and run 'npm ci' once, then push again." -fi -if [ ! -d node_modules ]; then - refuse "REFUSED - node_modules is missing. Run 'npm ci' once, then push again." -fi -# The same three gates as CI's node job: types first (vitest strips them, so -# a type error passes every test), then vitest with its report, then the -# guard that counts what ran against .github/test-floors.json, because a run -# that collected too few tests or skipped some still exits 0. -mkdir -p tmp -echo "[pre-push] npm run typecheck" -without_git_env npm run typecheck -status=$? -if [ "$status" -ne 0 ]; then - refuse "REFUSED - npm run typecheck exited with $status. Nothing is pushed past a type error." -fi -echo "[pre-push] vitest" -without_git_env npx vitest run --reporter=default --reporter=json --outputFile=tmp/vitest-report.json -status=$? -if [ "$status" -ne 0 ]; then - refuse "REFUSED - vitest exited with $status. Nothing is pushed past a red suite." -fi -without_git_env node .github/scripts/test-guard.mjs tmp/vitest-report.json --suite vitest -status=$? -if [ "$status" -ne 0 ]; then - refuse "REFUSED - the vitest run did not reach this host's floor in .github/test-floors.json (see above)." -fi - -# 1b. dot-agentd. Without Go its tests cannot run here, and a push that no -# one tested is refused rather than waved through on the strength of CI. -if ! command -v go >/dev/null 2>&1; then - refuse "REFUSED - go is not on PATH, so the dot-agentd tests cannot run." \ - "Install Go (1.25 or newer, see guest/dot-agentd/go.mod) and push again." -fi -# The same checks, in the same order, as the go job in tests.yml: a file gofmt -# would rewrite, or code go vet rejects, is refused here instead of in CI. -echo "[pre-push] gofmt and go vet (guest/dot-agentd)" -unformatted=$(cd guest/dot-agentd && gofmt -l .) -if [ -n "$unformatted" ]; then - refuse "REFUSED - gofmt would rewrite these dot-agentd files:" "$unformatted" \ - "Run gofmt -w on them and push again." -fi -(cd guest/dot-agentd && without_git_env go vet ./... && without_git_env env GOOS=windows go vet ./...) -status=$? -if [ "$status" -ne 0 ]; then - refuse "REFUSED - go vet exited with $status. Nothing is pushed past a vet error." -fi -echo "[pre-push] go test ./... (guest/dot-agentd)" -(cd guest/dot-agentd && without_git_env go test -json -count=1 ./... > ../../tmp/go-test.json) -status=$? -if [ "$status" -ne 0 ]; then - refuse "REFUSED - go test exited with $status (its events are in tmp/go-test.json). Nothing is pushed past a red suite." -fi -without_git_env node .github/scripts/test-guard.mjs tmp/go-test.json --suite go -status=$? -if [ "$status" -ne 0 ]; then - refuse "REFUSED - the go test run did not reach this host's floor in .github/test-floors.json (see above)." -fi - -# 2. The shared policy. The interpreter is, in order: INVISIBLE_GATE_PYTHON, -# the virtual environment under tmp/ that the command below creates, then a -# python on PATH. The venv layout differs between Windows (Scripts/) and the -# rest (bin/); both are looked for, so no platform check is needed. -python_bin="" -for candidate in "$INVISIBLE_GATE_PYTHON" tmp/gates-venv/bin/python tmp/gates-venv/Scripts/python.exe python3 python py; do - [ -n "$candidate" ] || continue - if [ -x "$candidate" ] || command -v "$candidate" >/dev/null 2>&1; then - python_bin="$candidate" - break - fi -done -if [ -z "$python_bin" ]; then - refuse "REFUSED - no python on PATH, so the shared gates cannot run." \ - "Nothing must be pushed past a hook that could not check it." \ - "Set INVISIBLE_GATE_PYTHON to an interpreter and push again." -fi - -pin=$(sed -n 's/^invisible-core==\([0-9.]*\).*/\1/p' .github/gates-requirements.txt) -if [ -z "$pin" ]; then - refuse "REFUSED - .github/gates-requirements.txt does not pin invisible-core exactly." -fi - -# exec, so stdin (the refs git is pushing) reaches the policy untouched. The -# installed invisible-core must be exactly the pinned one: CI runs that -# version, and a different policy locally would pass what CI refuses. -exec "$python_bin" -c 'import sys -pin = sys.argv[1] -setup = ("python -m venv tmp/gates-venv, then tmp/gates-venv/bin/python (or " - "tmp\\gates-venv\\Scripts\\python.exe) -m pip install -r .github/gates-requirements.txt") -try: - from importlib.metadata import PackageNotFoundError, version - installed = version("invisible-core") - from invisible_core.hooks import main -except (ImportError, PackageNotFoundError) as exc: - sys.exit("[pre-push] REFUSED - cannot import invisible_core.hooks (%s).\n" - "[pre-push] Create the gates environment once: %s" % (exc, setup)) -if installed != pin: - sys.exit("[pre-push] REFUSED - invisible-core %s is installed, the repository pins %s.\n" - "[pre-push] Reinstall: %s" % (installed, pin, setup)) -sys.exit(main())' "$pin" diff --git a/.github/ISSUE_TEMPLATE/01-setup.yml b/.github/ISSUE_TEMPLATE/01-setup.yml deleted file mode 100644 index 47ccd50c..00000000 --- a/.github/ISSUE_TEMPLATE/01-setup.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: Setup failure -description: "setup, doctor, the images or the server fail before you can talk to a Dot" -title: "[setup] " -labels: ["bug", "setup"] -body: - - type: markdown - attributes: - value: | - Use this when the host never gets to a working Dot. If a Dot runs but does something wrong, use the bug report. - The guide's [troubleshooting](https://github.com/feder-cr/invisible_dots/blob/main/docs/guide.md#troubleshooting) covers the common failures. - - - type: dropdown - id: os - attributes: - label: Host - options: - - Windows 11 (Windows Hypervisor Platform) - - Windows 10 - - Ubuntu 24.04 (KVM) - - Other Linux - - Other - validations: - required: true - - - type: input - id: commit - attributes: - label: Commit - description: Output of `git rev-parse --short HEAD`. - validations: - required: true - - - type: textarea - id: doctor - attributes: - label: invisible-dots doctor - description: The full output of `node apps/cli/dist/invisible-dots.mjs doctor`. - render: text - validations: - required: true - - - type: textarea - id: what - attributes: - label: What you ran and what happened - description: The command, and its output verbatim. Remove keys and personal paths. - render: text - validations: - required: true - - - type: checkboxes - id: confirm - attributes: - label: Before submitting - options: - - label: Searched existing issues. - required: true - - label: On the latest `main`. - required: true - - label: Removed keys, tokens and personal paths. - required: true diff --git a/.github/ISSUE_TEMPLATE/02-bug.yml b/.github/ISSUE_TEMPLATE/02-bug.yml deleted file mode 100644 index f31974ac..00000000 --- a/.github/ISSUE_TEMPLATE/02-bug.yml +++ /dev/null @@ -1,70 +0,0 @@ -name: Bug report -description: A Dot, the web UI, the command line, the API or a channel does something wrong -title: "[bug] " -labels: ["bug"] -body: - - type: textarea - id: what - attributes: - label: What happened - description: What you asked or did, what you expected, and what happened instead. - validations: - required: true - - - type: textarea - id: steps - attributes: - label: How to reproduce - description: The message or task you gave the Dot, or the steps in the UI or the command line. - validations: - required: true - - - type: dropdown - id: where - attributes: - label: Where - multiple: true - options: - - Web UI - - Command line - - HTTP API - - Telegram - - WhatsApp - - The Dot's computer or browser - validations: - required: true - - - type: input - id: commit - attributes: - label: Commit and host - placeholder: "a0bc3a64, Windows 11" - validations: - required: true - - - type: input - id: model - attributes: - label: Model - placeholder: z-ai/glm-5.3-flash - validations: - required: false - - - type: textarea - id: logs - attributes: - label: Logs - description: "`invisible-dots logs --tail 200 --no-follow`, and the server's output around the failure. Remove keys and personal data." - render: text - validations: - required: false - - - type: checkboxes - id: confirm - attributes: - label: Before submitting - options: - - label: Searched existing issues. - required: true - - label: Removed keys, tokens and personal data. - required: true diff --git a/.github/ISSUE_TEMPLATE/03-feature.yml b/.github/ISSUE_TEMPLATE/03-feature.yml deleted file mode 100644 index dd43ca90..00000000 --- a/.github/ISSUE_TEMPLATE/03-feature.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Feature request -description: Something a Dot, the web UI or the command line should do -title: "[feature] " -labels: ["enhancement"] -body: - - type: textarea - id: need - attributes: - label: What you want to do - description: The task or the situation, in your words. What a Dot should be able to do, not how to build it. - validations: - required: true - - - type: textarea - id: today - attributes: - label: What happens today - description: What you tried, and where it stops. - validations: - required: false - - - type: textarea - id: idea - attributes: - label: An idea of how - description: Optional. - validations: - required: false diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml deleted file mode 100644 index 98b845ae..00000000 --- a/.github/ISSUE_TEMPLATE/config.yml +++ /dev/null @@ -1,8 +0,0 @@ -blank_issues_enabled: false -contact_links: - - name: Security issue - url: https://github.com/feder-cr/invisible_dots/blob/main/SECURITY.md - about: Do not open a public issue; report it privately as SECURITY.md says. - - name: A site detects or blocks the browser - url: https://github.com/feder-cr/invisible_playwright/issues - about: The Dot's browser is invisible_playwright; its issues are the place for that. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index 5a479254..00000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,17 +0,0 @@ -## What changes - - - -## Related issues - - - -## How it was tested - - - -## Checklist - -- [ ] One logical change; tests added or updated for any change of behaviour. -- [ ] The pre-push gate passed (`git config core.hooksPath .githooks`). -- [ ] The README or `docs/` updated if what a person sees changed. diff --git a/.github/gates-requirements.txt b/.github/gates-requirements.txt deleted file mode 100644 index 57eb67fa..00000000 --- a/.github/gates-requirements.txt +++ /dev/null @@ -1 +0,0 @@ -invisible-core==34.31.0 diff --git a/.github/scripts/golden-release.mjs b/.github/scripts/golden-release.mjs deleted file mode 100644 index b93ea6da..00000000 --- a/.github/scripts/golden-release.mjs +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env node -// Puts the golden image `image build --compress` made into the shape of a release (guest/image-builder/src/prebuilt.ts): -// its manifest, the image cut in parts under the 2 GiB a release asset may hold, and release.json naming each part with -// its size and SHA-256. Prints the tag, golden-. -// -// node .github/scripts/golden-release.mjs -import { createHash } from "node:crypto"; -import { createReadStream } from "node:fs"; -import { copyFile, mkdir, open, readdir, readFile, writeFile } from "node:fs/promises"; -import { basename, join } from "node:path"; - -const PART_BYTES = 1900 * 1024 * 1024; - -const [imagesDir, outDir] = process.argv.slice(2); -if (!imagesDir || !outDir) { - console.error("usage: golden-release.mjs "); - process.exit(2); -} - -const manifests = (await readdir(imagesDir)).filter((name) => /^golden-.+\.json$/.test(name)); -if (manifests.length !== 1) { - console.error(`expected one golden manifest in ${imagesDir}, found ${manifests.length}`); - process.exit(1); -} -const manifestName = manifests[0]; -const manifest = JSON.parse(await readFile(join(imagesDir, manifestName), "utf8")); -const image = join(imagesDir, manifest.file); -await mkdir(outDir, { recursive: true }); -await copyFile(join(imagesDir, manifestName), join(outDir, manifestName)); - -const parts = []; -let out; -let hash; -let written = 0; -const whole = createHash("sha256"); -async function closePart() { - if (!out) return; - await out.close(); - parts[parts.length - 1].size = written; - parts[parts.length - 1].sha256 = hash.digest("hex"); - out = undefined; -} -for await (let chunk of createReadStream(image, { highWaterMark: 8 * 1024 * 1024 })) { - whole.update(chunk); - while (chunk.length > 0) { - if (!out) { - const name = `${basename(manifest.file)}.part${String(parts.length).padStart(2, "0")}`; - parts.push({ name, size: 0, sha256: "" }); - out = await open(join(outDir, name), "w"); - hash = createHash("sha256"); - written = 0; - } - const take = chunk.subarray(0, PART_BYTES - written); - await out.write(take); - hash.update(take); - written += take.length; - chunk = chunk.subarray(take.length); - if (written === PART_BYTES) await closePart(); - } -} -await closePart(); - -const sha256 = whole.digest("hex"); -if (sha256 !== manifest.sha256) { - console.error(`${image} hashes to ${sha256}, its manifest says ${manifest.sha256}`); - process.exit(1); -} -await writeFile(join(outDir, "release.json"), `${JSON.stringify({ inputs_digest: manifest.inputs_digest, manifest: manifestName, parts }, null, 2)}\n`); -console.log(`golden-${manifest.inputs_digest}`); diff --git a/.github/scripts/test-guard.mjs b/.github/scripts/test-guard.mjs deleted file mode 100644 index a4aecce0..00000000 --- a/.github/scripts/test-guard.mjs +++ /dev/null @@ -1,193 +0,0 @@ -// Reads a test run's report and refuses a run that only looks green. -// -// vitest, `go test` and pytest exit 0 when nothing failed, which is also what they do -// when a whole file stopped being collected, when a suite skipped itself -// because a tool or a database was missing, or when a build constraint left -// a file out. Each of those reads exactly like a pass. So the count of tests -// that really ran is checked against a floor, and every skipped test must be -// one this host is known to skip. -// -// node .github/scripts/test-guard.mjs --suite [--floors ] -// -// vitest's JSON report (--reporter=json --outputFile) for the -// vitest suites, the output of `go test -json` for "go", and a -// JUnit XML report (a file ending in .xml: pytest --junitxml for -// the Python engine, Playwright's junit reporter for the web -// client's browser tests). -// --suite the entry of the floors file to apply: vitest, postgres, whatsapp, -// go, pytest or playwright. -// --floors default .github/test-floors.json. The entry is chosen by the -// host this runs on (linux, win32), so CI and the pre-push hook -// read the same numbers from the same file. -// -// A floor entry: min_passed, allow_skipped (a regex over a skipped test's -// full name, "" for none), and require ({ regex: n }: at least n passed -// tests whose full name matches, e.g. the suites parametrized on pg). -// -// Raise the floors when tests are added; lowering one is a decision to write -// down in the commit that does it. -import { readFileSync } from "node:fs"; - -function fail(message) { - console.error(`test-guard: REFUSED - ${message}`); - process.exit(1); -} - -const args = process.argv.slice(2); -const reportPath = args.shift(); -const usage = "usage: test-guard.mjs --suite [--floors ]"; -if (!reportPath) fail(usage); -let suite; -let floorsPath = ".github/test-floors.json"; -while (args.length > 0) { - const flag = args.shift(); - const value = args.shift(); - if (value === undefined) fail(`${flag} needs a value`); - if (flag === "--suite") suite = value; - else if (flag === "--floors") floorsPath = value; - else fail(`unknown option ${flag}; ${usage}`); -} -if (!suite) fail(usage); - -let floors; -try { - floors = JSON.parse(readFileSync(floorsPath, "utf8")); -} catch (error) { - fail(`cannot read ${floorsPath}: ${error.message}`); -} -// The host this runs on picks the floor, because each host runs a different -// set (a test that needs a real Windows PowerShell, a Go file built only on -// unix). This script is a check of the test run, not product code. -const host = process.platform; -const floor = floors[suite]?.[host]; -if (!floor) fail(`${floorsPath} has no "${suite}" floor for ${host}`); -const minPassed = floor.min_passed; -if (!Number.isInteger(minPassed) || minPassed <= 0) fail(`${floorsPath} ${suite}.${host}.min_passed must be a positive integer`); -const allowSkipped = floor.allow_skipped ? new RegExp(floor.allow_skipped) : undefined; -const required = Object.entries(floor.require ?? {}).map(([pattern, min]) => ({ pattern: new RegExp(pattern), min })); - -let text; -try { - text = readFileSync(reportPath, "utf8"); -} catch (error) { - fail(`cannot read ${reportPath}: ${error.message}`); -} - -/** Every test of the run as { fullName, status, file }, status passed | failed | skipped. */ -function vitestTests() { - let report; - try { - report = JSON.parse(text); - } catch (error) { - fail(`${reportPath} is not vitest's JSON report (${error.message}); run vitest with --reporter=json --outputFile`); - } - const tests = report.testResults.flatMap((file) => - file.assertionResults.map((test) => ({ - fullName: test.fullName, - status: test.status === "passed" || test.status === "failed" ? test.status : "skipped", - file: file.name, - })), - ); - // A file that failed to load has no failed test of its own, only an error. - const broken = report.testResults - .filter((file) => file.status === "failed" && file.assertionResults.every((test) => test.status !== "failed")) - .map((file) => `${file.name} failed outside any test: ${(file.message ?? "").split("\n")[0]}`); - return { tests, broken, files: report.testResults.length }; -} - -/** The text an XML attribute value stands for. */ -function xmlText(value) { - return value - .replace(/&#x([0-9a-fA-F]+);/g, (_, hex) => String.fromCodePoint(parseInt(hex, 16))) - .replace(/&#([0-9]+);/g, (_, dec) => String.fromCodePoint(parseInt(dec, 10))) - .replace(/</g, "<") - .replace(/>/g, ">") - .replace(/"/g, '"') - .replace(/'/g, "'") - .replace(/&/g, "&"); -} - -/** The attributes of one start tag, as an object. */ -function xmlAttributes(tag) { - const attributes = {}; - for (const match of tag.matchAll(/([\w:.-]+)="([^"]*)"/g)) attributes[match[1]] = xmlText(match[2]); - return attributes; -} - -/** - * JUnit XML, the report of pytest --junitxml: one testcase element per test. - * A testcase holding a failure or an error element failed (a module that could - * not be collected is reported as one), one holding a skipped element was - * skipped, and any other passed. The name a skip is allowed by is - * " ". - */ -function junitTests() { - if (!/]/.test(text)) fail(`${reportPath} is not a JUnit XML report; run pytest with --junitxml or Playwright with the junit reporter`); - const tests = []; - const classes = new Set(); - for (const match of text.matchAll(/]*?)(\/?)>/g)) { - const attributes = xmlAttributes(match[1]); - let body = ""; - if (match[2] !== "/") { - const end = text.indexOf("", match.index); - body = end === -1 ? "" : text.slice(match.index + match[0].length, end); - } - const status = /<(failure|error)\b/.test(body) ? "failed" : / test.status === "passed"); -const failed = tests.filter((test) => test.status === "failed"); -const skipped = tests.filter((test) => test.status === "skipped"); - -console.log(`test-guard: ${suite} on ${host}: ${passed.length} passed, ${failed.length} failed, ${skipped.length} skipped, in ${files} files`); - -const problems = [...broken]; -if (failed.length > 0) problems.push(`${failed.length} failed tests`); -if (passed.length < minPassed) problems.push(`only ${passed.length} tests passed, the floor is ${minPassed}`); -for (const test of skipped) { - if (!allowSkipped || !allowSkipped.test(test.fullName)) problems.push(`skipped without being allowed to: ${test.fullName} (${test.file})`); -} -for (const { pattern, min } of required) { - const count = passed.filter((test) => pattern.test(test.fullName)).length; - console.log(`test-guard: ${count} passed tests match ${pattern}`); - if (count < min) problems.push(`${count} passed tests match ${pattern}, at least ${min} must`); -} - -if (problems.length > 0) { - for (const problem of problems) console.error(`test-guard: ${problem}`); - fail(`${problems.length} problem(s) with this run`); -} -console.log("test-guard: ok"); diff --git a/.github/test-floors.json b/.github/test-floors.json deleted file mode 100644 index ddac8746..00000000 --- a/.github/test-floors.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "//": "What each suite must reach on each host, read by .github/scripts/test-guard.mjs in CI (tests.yml) and in the pre-push hook. Raise a floor when tests are added; lowering one is a decision to write down in the commit that does it. Linux skips only the tests that drive a real Windows PowerShell; a Windows host compiles dot-agentd's unix-only test files out and skips the two Go tests that need a Linux guest. The playwright floor is the web client's browser tests (the web job of tests.yml; the same eighty-eight run on a Windows host, which is where they are developed). The pytest floor is the Python engine's (invisible_engine_dots, the engine job of tests.yml, Linux only, nothing may skip). The whatsapp floor is the suite that needs the real WhatsApp client, an opt-in install (npm run test:whatsapp, the whatsapp-client job of tests.yml, which installs the client and runs it with PostgreSQL, so the pg runs of the auth state count); a host without PostgreSQL reaches the win32 number.", - "vitest": { - "linux": { "min_passed": 2209, "allow_skipped": "^generated PowerShell on a real Windows PowerShell " }, - "win32": { "min_passed": 2206, "allow_skipped": "" } - }, - "postgres": { - "linux": { "min_passed": 744, "allow_skipped": "", "require": { "\\bpg\\b": 295 } } - }, - "whatsapp": { - "linux": { "min_passed": 25, "allow_skipped": "", "require": { "\\bpg\\b": 9 } }, - "win32": { "min_passed": 16, "allow_skipped": "" } - }, - "go": { - "linux": { "min_passed": 120, "allow_skipped": "" }, - "win32": { "min_passed": 70, "allow_skipped": "^agentd (TestSystemRoute|TestPowerOffStartsTheCommandAndAnswers202)$" } - }, - "pytest": { - "linux": { "min_passed": 2493, "allow_skipped": "" } - }, - "playwright": { - "linux": { "min_passed": 88, "allow_skipped": "" }, - "win32": { "min_passed": 88, "allow_skipped": "" } - } -} diff --git a/.github/workflows/golden-image.yml b/.github/workflows/golden-image.yml deleted file mode 100644 index 761056b5..00000000 --- a/.github/workflows/golden-image.yml +++ /dev/null @@ -1,76 +0,0 @@ -name: golden image - -# Builds the golden image once, here, and publishes it as the release golden-, which -# `invisible-dots image build` downloads instead of building (guest/image-builder/src/prebuilt.ts). The digest is the -# one every host computes from the pinned inputs and the newest release of the Dot's browser on the package index -# (browser-release.ts), so a host only ever takes the image of its own inputs. A digest that already has its release -# is not built again. -# -# The schedule is what makes a new release of the browser reach the hosts as a prebuilt image: no file of this -# repository changes when one is published, and the next run here finds a new digest and builds it. -on: - push: - branches: [main] - paths: - - guest/image-builder/** - - virtualization/images/** - schedule: - - cron: "17 */6 * * *" - workflow_dispatch: - -permissions: - contents: write - -concurrency: - group: golden-image - cancel-in-progress: false - -defaults: - run: - shell: bash - -jobs: - build: - name: build and publish the golden image - runs-on: ubuntu-latest - timeout-minutes: 180 - env: - INVISIBLE_DOTS_HOME: ${{ github.workspace }}/../idots-home - GH_TOKEN: ${{ github.token }} - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: "24" - cache: npm - - uses: actions/setup-go@v6 - with: - go-version-file: guest/dot-agentd/go.mod - - name: KVM for the builder VM, and QEMU - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - sudo apt-get update -qq - sudo apt-get install -y -qq --no-install-recommends qemu-system-x86 qemu-utils - - run: npm ci --no-audit --no-fund - - run: npm run build --workspace @invisible-dots/cli - # The runtime ISO, which `image build` makes first, carries dot-agentd. - - working-directory: guest/dot-agentd - run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false -ldflags="-s -w" -o bin/dot-agentd ./cmd/dot-agentd - - name: the image (downloaded instead when its release exists already) - id: build - run: | - node apps/cli/dist/invisible-dots.mjs image build --compress | tee build.log - tag=$(node .github/scripts/golden-release.mjs "$INVISIBLE_DOTS_HOME/images" "$RUNNER_TEMP/release") - echo "tag=$tag" >> "$GITHUB_OUTPUT" - - name: publish - run: | - tag='${{ steps.build.outputs.tag }}' - if gh release view "$tag" >/dev/null 2>&1; then - echo "$tag is published already" - exit 0 - fi - gh release create "$tag" "$RUNNER_TEMP"/release/* \ - --title "Golden image $tag" \ - --notes "The golden image of the inputs ${tag#golden-}, built by the golden image workflow from ${GITHUB_SHA}. invisible-dots image build downloads it when a checkout's inputs have this digest. It holds Ubuntu 24.04, the software guest/image-builder/pins.json and the engine's lock name, and the release of invisible-playwright-mcp its manifest records, each under its own license (THIRD_PARTY_NOTICES.md)." diff --git a/.github/workflows/jobs-database.yml b/.github/workflows/jobs-database.yml new file mode 100644 index 00000000..90c6e85e --- /dev/null +++ b/.github/workflows/jobs-database.yml @@ -0,0 +1,34 @@ +name: job database + +# Every day: read the public job databases, merge them and publish the result as a release, which `aihawk scan` reads. +# Users never scrape anything. +on: + schedule: + - cron: "23 4 * * *" + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: job-database + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 350 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: room on disk + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL + df -h / + - run: pip install duckdb + - run: python pipeline/build.py --out work + - run: python pipeline/publish.py --out work/out --repo "$GITHUB_REPOSITORY" + env: + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/publish-wiki.yml b/.github/workflows/publish-wiki.yml deleted file mode 100644 index 091ab5b5..00000000 --- a/.github/workflows/publish-wiki.yml +++ /dev/null @@ -1,82 +0,0 @@ -# Publishes docs/wiki/ to the GitHub wiki of this repository on every push to main that changes it. -# -# The wiki is its own git repository (.wiki.git): flat pages, no front matter, a _Sidebar.md. The render is -# invisible_playwright_mcp's own (scripts/build_wiki.py and scripts/sync_article_pixels.py, MIT, fetched at a fixed -# commit rather than copied): it strips the front matter, rewrites [x](page.md) links to the wiki's form, names -# index.md Home, builds the sidebar from parent/nav_order, and gives every page its view pixel (a release asset of -# the `article-views` release, created on first use). The wiki checkout is emptied of pages before the copy, so a -# page deleted from docs/wiki/ leaves the wiki too. -name: publish-wiki - -on: - push: - branches: [main] - paths: - - "docs/wiki/**" - - ".github/workflows/publish-wiki.yml" - workflow_dispatch: {} - -permissions: - contents: write - -concurrency: - group: publish-wiki - cancel-in-progress: true - -env: - # invisible_playwright_mcp's main on 2026-10-09: the converter this wiki is rendered with. - CONVERTER_REPO: feder-cr/invisible_playwright_mcp - CONVERTER_REF: b72e8b1937771149ed06b2ba73ccb9cb32a687f8 - -jobs: - publish-wiki: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Fetch the converter - run: | - set -euo pipefail - mkdir -p wiki_tools - for script in build_wiki.py sync_article_pixels.py; do - curl -fsSL -o "wiki_tools/$script" \ - "https://raw.githubusercontent.com/${CONVERTER_REPO}/${CONVERTER_REF}/scripts/$script" - done - - - name: Render docs/wiki/ into wiki pages - run: python wiki_tools/build_wiki.py docs/wiki wiki_build --pixels "${{ github.repository }}" - - # Upload before the push: a page published without its asset counts nothing until the next run. - - name: Give every new page its view pixel - env: - GITHUB_TOKEN: ${{ github.token }} - run: python wiki_tools/sync_article_pixels.py --repo "${{ github.repository }}" wiki_build - - - name: Check every page carries its own pixel - run: | - python wiki_tools/sync_article_pixels.py --selftest - python wiki_tools/sync_article_pixels.py --repo "${{ github.repository }}" --check wiki_build - - - name: Publish to the wiki - env: - WIKI_PUSH_TOKEN: ${{ secrets.WIKI_TOKEN != '' && secrets.WIKI_TOKEN || github.token }} - run: | - set -euo pipefail - repo="${{ github.repository }}" - git clone "https://x-access-token:${WIKI_PUSH_TOKEN}@github.com/${repo}.wiki.git" wiki - find wiki -maxdepth 1 -name '*.md' -delete - cp wiki_build/*.md wiki/ - cd wiki - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" - git add -A - if git diff --cached --quiet; then - echo "wiki already up to date" - else - git commit -m "docs: sync wiki from docs/wiki (${{ github.ref_name }})" - git push origin HEAD:master - fi diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3cb34411..37ddeb00 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,333 +4,43 @@ on: push: branches: [main] pull_request: - branches: [main] - workflow_dispatch: permissions: contents: read -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -defaults: - run: - shell: bash - -# `gate` is the ONLY context the branch protection requires: it depends on every -# job, so its name never drifts when the jobs change. It accepts nothing but -# success, because no job here is ever skipped on purpose and none has a paths -# filter: a skip would mean a job that did not run, which reads exactly like -# one that passed. jobs: - node: - name: typecheck, vitest and the bundle (${{ matrix.os }}) + unit: + name: lint and unit tests (${{ matrix.os }}) runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - # The floors each host must reach live in .github/test-floors.json, - # which the pre-push hook reads too. os: [ubuntu-latest, windows-latest] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: "24" - cache: npm - # Linux's vitest builds the real dot-agentd and drives it with the host's GuestClient - # (apps/vm-manager/test/agentd.linux.test.ts, which vitest.config.ts collects on Linux only). - - if: runner.os == 'Linux' - uses: actions/setup-go@v6 - with: - go-version-file: guest/dot-agentd/go.mod - # The module has no dependencies, so there is no go.sum to key a cache on. - cache: false - - run: npm ci --no-audit --no-fund - - run: npm run typecheck - - name: the pre-push hook parses - run: sh -n .githooks/pre-push - - name: vitest - run: npx vitest run --reporter=default --reporter=json --outputFile=vitest-report.json - # vitest's exit code cannot tell "everything passed" from "half of it - # was never collected" or "a suite skipped itself": count what ran. - - name: refuse a run with too few tests or an unexpected skip - if: always() - run: node .github/scripts/test-guard.mjs vitest-report.json --suite vitest - - name: bundle (the command, which carries the server) - run: npm run build --workspace @invisible-dots/cli - # The bundled command on a fresh home: doctor must run end to end and - # create nothing. Its exit code is not 0 here (no QEMU, no images), so - # only "it ran and printed every check" is asserted. - - name: the bundled doctor runs and creates nothing - env: - INVISIBLE_DOTS_HOME: ${{ runner.temp }}/idots-home - run: | - set +e - node apps/cli/dist/invisible-dots.mjs doctor > doctor.txt 2>&1 - code=$? - set -e - cat doctor.txt - echo "doctor exit code: $code" - grep -q "checks:" doctor.txt - test ! -e "$INVISIBLE_DOTS_HOME" - - postgres: - name: the pg adapter against PostgreSQL - runs-on: ubuntu-latest - services: - postgres: - image: postgres:18.4-alpine3.23 - env: - POSTGRES_PASSWORD: test - ports: - - 5432:5432 - options: >- - --health-cmd "pg_isready -U postgres" - --health-interval 2s - --health-timeout 5s - --health-retries 30 - env: - # With DATABASE_URL set, every suite parametrized with testAdapters() - # runs on pg as well as on PGlite. - DATABASE_URL: postgres://postgres:test@127.0.0.1:5432/postgres - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: "24" - cache: npm - - run: npm ci --no-audit --no-fund - - name: vitest on both adapters - run: npx vitest run packages/database packages/events packages/channels apps/api apps/scheduler --reporter=default --reporter=json --outputFile=vitest-report.json - - name: refuse a run in which the pg suites did not run - if: always() - run: node .github/scripts/test-guard.mjs vitest-report.json --suite postgres - - # The WhatsApp client (Baileys) is an opt-in install: it depends on libsignal, which is GPL-3.0, and the default - # npm install holds nothing GPL. The jobs above never install it. This one does, with the command a person runs - # (npm run whatsapp:install), to prove the adapter against the real library: the types (the real module has the - # shape the adapter writes down) and the tests that need it, on PGlite and on PostgreSQL. - whatsapp-client: - name: the WhatsApp client, opt-in (types and tests on the real library) - runs-on: ubuntu-latest - services: - postgres: - image: postgres:18.4-alpine3.23 - env: - POSTGRES_PASSWORD: test - ports: - - 5432:5432 - options: >- - --health-cmd "pg_isready -U postgres" - --health-interval 2s - --health-timeout 5s - --health-retries 30 - env: - DATABASE_URL: postgres://postgres:test@127.0.0.1:5432/postgres - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: "24" - cache: npm - - run: npm ci --no-audit --no-fund - - name: install the client, as a person who enables WhatsApp does - run: npm run whatsapp:install - - name: the real module has the shape the adapter writes down - run: npm run typecheck:whatsapp - - name: vitest with the real client - run: npm run test:whatsapp -- --reporter=default --reporter=json --outputFile=vitest-report.json - - name: refuse a run with too few tests or an unexpected skip - if: always() - run: node .github/scripts/test-guard.mjs vitest-report.json --suite whatsapp - - iso: - name: the Linux kernel reads our ISO images - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: "24" - cache: npm - - run: npm ci --no-audit --no-fund - # The unit tests read images back with our own parser; this is the - # reader a guest really uses. See packages/iso/scripts/kernel-check.ts. - - name: write, mount read-only, compare - run: | - npx tsx packages/iso/scripts/kernel-check.ts write "$RUNNER_TEMP/sample.iso" - blkid -p -o export "$RUNNER_TEMP/sample.iso" | tee blkid.txt - grep -qx "TYPE=iso9660" blkid.txt - grep -qx "LABEL=cidata" blkid.txt - mkdir -p "$RUNNER_TEMP/mnt" - sudo mount -o loop,ro "$RUNNER_TEMP/sample.iso" "$RUNNER_TEMP/mnt" - npx tsx packages/iso/scripts/kernel-check.ts verify "$RUNNER_TEMP/mnt" - sudo umount "$RUNNER_TEMP/mnt" - - go: - name: dot-agentd (gofmt, go vet, go test, static build) - runs-on: ubuntu-latest - defaults: - run: - working-directory: guest/dot-agentd - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v6 - with: - go-version-file: guest/dot-agentd/go.mod - # The module has no dependencies, so there is no go.sum to key a cache on. - cache: false - - run: test -z "$(gofmt -l .)" || { gofmt -l .; exit 1; } - - run: go vet ./... - - run: GOOS=windows go vet ./... - # go test exits 0 when a test skipped itself or a build constraint left - # a file out, so what ran is counted against the same floors file. - - uses: actions/setup-node@v4 - with: - node-version: "24" - - run: go test -json -count=1 ./... > go-test.json - - name: refuse a run with too few tests or an unexpected skip - if: always() - run: node ../../.github/scripts/test-guard.mjs go-test.json --suite go --floors ../../.github/test-floors.json - - run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -buildvcs=false -ldflags="-s -w" -o bin/dot-agentd ./cmd/dot-agentd - - run: file bin/dot-agentd | tee /dev/stderr | grep -q "ELF 64-bit LSB executable, x86-64.*statically linked" - - engine: - name: the engine (pytest) - runs-on: ubuntu-latest - defaults: - run: - working-directory: invisible_engine_dots steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: "3.12" - - run: python -m pip install -e ".[dev]" - - run: pytest -q --junitxml=pytest-report.xml - # pytest exits 0 when a module stopped being collected or a test skipped - # itself, so what ran is counted against the same floors file. - - uses: actions/setup-node@v4 - with: - node-version: "24" - - name: refuse a run with too few tests or an unexpected skip - if: always() - run: node ../.github/scripts/test-guard.mjs pytest-report.xml --suite pytest --floors ../.github/test-floors.json + - run: pip install -e ".[dev]" + - run: ruff check . + - run: pytest -m "not browser" - smoke: - name: the engine smoke (dot-agentd and the engine under their two users) - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@v4 - # Builds dot-agentd in golang:1.26.8, the engine's environment in ubuntu:24.04 - # the way provision.sh builds it, and runs the Dot. It exits non-zero on a - # failed or a skipped check and prints "SMOKE: N passed, F failed, S skipped". - - name: run the smoke on this checkout - run: bash guest/image-builder/test/smoke/run.sh - - browser-smoke: - name: the browser smoke (the real invisible-playwright-mcp and its Firefox, as the Dot runs them) - runs-on: ubuntu-latest - timeout-minutes: 45 - steps: - - uses: actions/checkout@v4 - # The golden image's browser built as provision.sh builds it (apt packages of pins.json, the newest - # invisible-playwright-mcp, the engine of the browser), a desktop on :0, and the Dot - # (dot-agentd and the engine under their two users) browsing pages of the container. It needs the - # network for those downloads and for the egress address of a launch. Same exit rule and summary - # line as the smoke job. - - name: run the browser smoke on this checkout - run: bash guest/image-builder/test/smoke/run.sh --suite browser - - web: - name: the web client builds and passes its browser tests - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: "24" - cache: npm - - run: npm ci --no-audit --no-fund - - name: next build - run: npm run build --workspace @invisible-dots/web - - name: install the browser the tests drive - run: npx playwright install --with-deps chromium - # Each worker starts the real control plane (over a fake VM layer) and the built web client; see - # apps/web/e2e/harness.ts. The JUnit report lands in apps/web/playwright-report.xml. - - name: browser tests - run: npm run test:e2e --workspace @invisible-dots/web - # Playwright exits 0 when a spec file stopped being collected or a test skipped itself, so what ran is - # counted against the same floors file. - - name: refuse a run with too few tests or an unexpected skip - if: always() - run: node .github/scripts/test-guard.mjs apps/web/playwright-report.xml --suite playwright - - name: keep the traces of a failed run - if: failure() - uses: actions/upload-artifact@v4 - with: - name: playwright-traces - path: apps/web/test-results - if-no-files-found: ignore - - english: - name: the repository is English + browser: + name: the real browser on a real form runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: "3.12" - # The gate ships in the pinned invisible-core; its settings are read - # from pyproject.toml ([tool.invisible.english]). - - name: install the pinned gates - run: | - python -m pip install --upgrade pip - pip install -r .github/gates-requirements.txt - - name: check the prose - run: | - python -m invisible_core.english --selftest - python -m invisible_core.english - - claims: - name: the README promises nothing it cannot support - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: install the pinned gates - run: | - python -m pip install --upgrade pip - pip install -r .github/gates-requirements.txt - - name: check the claims - run: | - python -m invisible_core.claims --selftest - python -m invisible_core.claims + - run: pip install -e ".[dev]" + - run: sudo apt-get update -qq && sudo apt-get install -y -qq xvfb libgtk-3-0t64 libasound2t64 libdbus-glib-1-2 libxt6t64 + - run: xvfb-run -a -s "-screen 0 1920x1080x24" pytest -m browser gate: - needs: [node, postgres, whatsapp-client, iso, go, engine, smoke, browser-smoke, web, english, claims] - if: always() + name: gate runs-on: ubuntu-latest + needs: [unit, browser] + if: always() steps: - - name: refuse unless every job succeeded - env: - RESULTS: ${{ toJSON(needs) }} - run: | - echo "$RESULTS" - node -e ' - const needs = JSON.parse(process.env.RESULTS); - const names = Object.keys(needs); - if (names.length !== 11) { console.error(`expected 11 jobs, got ${names.length}`); process.exit(1); } - let ok = true; - for (const name of names) { - console.log(`${name}: ${needs[name].result}`); - if (needs[name].result !== "success") ok = false; - } - if (!ok) { console.error("a job did not succeed"); process.exit(1); } - console.log("ok"); - ' + - run: test "${{ needs.unit.result }}" = success && test "${{ needs.browser.result }}" = success diff --git a/.gitignore b/.gitignore index 86ba6965..73f27f00 100644 --- a/.gitignore +++ b/.gitignore @@ -1,35 +1,14 @@ -node_modules/ -.next/ -dist/ -coverage/ -*.tsbuildinfo -*.log -.env -.env.* - -# Go build output (guest/dot-agentd/bin/dot-agentd) -bin/ - -# Images written by local checks and builds; the real ones live in -# INVISIBLE_DOTS_HOME, never in the tree. -*.iso -*.qcow2 -*.img - -# Scratch output of local checks, and the virtual environment the pre-push -# hook creates there for the English and claims gates -/tmp/ - -# vitest's JSON report, as CI and the pre-push hook write it -vitest-report.json - -# The benchmark's tasks: tests/bench/run.sh writes them with author.py and author_long.py before every run -/tests/bench/tasks/ -/tests/bench/tasks-long/ - -# Editor and Python leftovers of the gates __pycache__/ +*.pyc .venv/ +.pytest_cache/ +.ruff_cache/ +dist/ +build/ +*.egg-info/ -# The README animation's frames: docs/hero/build.mjs writes them on every run, from the pages and photos beside it -/docs/hero/frames/ +# A search's own files: your CV, your answers and what AIHawk records. +.aihawk/ +aihawk.yaml +cv.md +answers.yaml diff --git a/.npmrc b/.npmrc deleted file mode 100644 index 727cdb26..00000000 --- a/.npmrc +++ /dev/null @@ -1,2 +0,0 @@ -engine-strict=true -fund=false diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md deleted file mode 100644 index 127c7195..00000000 --- a/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,33 +0,0 @@ -# Code of Conduct - -This project follows the [Contributor Covenant, v2.1](https://www.contributor-covenant.org/version/2/1/code_of_conduct/). - -## Our Pledge - -We pledge to make participation in our community a harassment-free experience for everyone. - -## Standards - -Examples of behavior that contributes to a positive environment: - -- Using welcoming and inclusive language -- Being respectful of differing viewpoints and experiences -- Gracefully accepting constructive criticism -- Focusing on what is best for the community - -Examples of unacceptable behavior: - -- The use of sexualized language or imagery -- Trolling, insulting or derogatory comments, and personal or political attacks -- Public or private harassment -- Publishing others' private information without explicit permission - -## Enforcement - -Instances of unacceptable behavior may be reported by contacting the maintainer at **federico.elia.majo@gmail.com**. All complaints will be reviewed and investigated and will result in a response that is deemed necessary and appropriate to the circumstances. - -The maintainer is obligated to maintain confidentiality with regard to the reporter of an incident. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org), version 2.1, available at https://www.contributor-covenant.org/version/2/1/code_of_conduct.html. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 5d5b0a5a..00000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,41 +0,0 @@ -# Contributing to invisible_dots - -Issues and pull requests are welcome. - -## Quick links - -- **Setup or `doctor` fails?** Open a [setup report](https://github.com/feder-cr/invisible_dots/issues/new?template=01-setup.yml). -- **A Dot, the web UI or the command line misbehaves?** Open a [bug report](https://github.com/feder-cr/invisible_dots/issues/new?template=02-bug.yml). -- **Idea?** Open a [feature request](https://github.com/feder-cr/invisible_dots/issues/new?template=03-feature.yml). -- **Security issue?** Do **not** open a public issue: see [SECURITY.md](SECURITY.md). -- **A site detects or blocks the browser?** That belongs to - [invisible_playwright](https://github.com/feder-cr/invisible_playwright/issues), the Dot's browser. - -## Development - -Everything is in [the guide's development section](docs/guide.md#development-and-tests). In short: - -```sh -npm ci -npm run typecheck -npm test -git config core.hooksPath .githooks # the pre-push gate: typecheck, vitest, go test, prose checks -``` - -The engine (`invisible_engine_dots/`, `pytest`, Linux only) and dot-agentd (`guest/dot-agentd/`, `go test ./...`) -have their own suites; [docs/architecture.md](docs/architecture.md) explains every component. - -## Pull requests - -1. Branch from `main`; one logical change per PR. -2. Add or update the tests for any change of behaviour. A test that stops running fails CI: the counts are held - to `.github/test-floors.json`. -3. Write the commit message as the repository does: `area: what is now true`, for example - `scheduler: a task the claim skipped is claimed when the row is free`. -4. Update the README or `docs/` when what a person sees changes. The README promises nothing it cannot - support, and CI checks it. -5. Every check must be green before merge. - -## License - -By contributing, you agree that your contributions are licensed under the MIT License ([LICENSE](LICENSE)). diff --git a/NOTICE b/NOTICE new file mode 100644 index 00000000..f1b0baff --- /dev/null +++ b/NOTICE @@ -0,0 +1,11 @@ +AIHawk includes or adapts work from these projects. + +src/aihawk/documents.py + The shape of the job report (dimensions weighed by judgment, a score from 1 to 5, posting legitimacy + judged apart from the score) is adapted from career-ops (https://github.com/santifer/career-ops), + MIT License, Copyright (c) 2026 Santiago Fernandez de Valderrama. No file was copied. + +The published job database + Is merged from the public sources named in pipeline/build.py, each with its home and licence as far as + it states one; each release's manifest.json lists them and which were read that day, and every row + names the sources that carried it. Each posting belongs to its employer and links back to the original. diff --git a/README.md b/README.md index 3a25a462..eff158fa 100644 --- a/README.md +++ b/README.md @@ -1,104 +1,50 @@ -
- - - - - - - invisible_dots, an open-source, self-hosted alternative to OpenAI Dots, Meta Muse, Grok Bot, Manus Cue and Claude Cowork. An animation of what a Dot does: asked once in its chat to save a shop's invoices every month and send the total to Telegram, it works on a computer of its own (a desktop, a terminal, files and a Firefox that keeps the shop's login), writes the steps down as a skill, asks before it adds the monthly automation, and then sends the total every month; several Dots, one computer each. - -
+# AIHawk -Open-source, self-hosted alternative to OpenAI Dots, Grok Bot. Built to be undetectable by anti-bot systems. +Find the jobs that fit your CV and apply to them. -## Quickstart +AIHawk searches a job database rebuilt every day from every public job database we know of, judges each job against your CV, writes a CV for the job, and fills in the application form in a visible browser. It sends nothing until you say yes. -You need Node 24+, Go 1.25+, Git and hardware virtualization (x86-64), plus an OpenRouter key. +```sh +pip install git+https://github.com/feder-cr/Jobs_Applier_AI_Agent_AIHawk +export ORCAROUTER_API_KEY=sk-orca-... -**Windows** (PowerShell): - -```powershell -winget install -e --id OpenJS.NodeJS.LTS; winget install -e --id GoLang.Go; winget install -e --id Git.Git -git clone https://github.com/feder-cr/invisible_dots; cd invisible_dots -npm ci; npm run build --workspace @invisible-dots/cli -node apps/cli/dist/invisible-dots.mjs setup --all -node apps/cli/dist/invisible-dots.mjs server +mkdir my-search && cd my-search +aihawk init # writes aihawk.yaml, cv.md and answers.yaml: fill them in +aihawk scan # searches the database; uses no model, scrapes nothing +aihawk score # judges the new jobs against your CV and goals +aihawk list # the best first +aihawk apply --top 5 # fills the forms and asks before each send ``` -**Linux** (Ubuntu 24.04): +## How it works -```bash -curl -fsSL https://deb.nodesource.com/setup_24.x | sudo -E bash - && sudo apt-get install -y nodejs git && sudo snap install go --classic -git clone https://github.com/feder-cr/invisible_dots && cd invisible_dots -npm ci && npm run build --workspace @invisible-dots/cli -node apps/cli/dist/invisible-dots.mjs setup --all -node apps/cli/dist/invisible-dots.mjs server -``` - -`setup --all` installs QEMU and turns on the accelerator (KVM, or the Windows Hypervisor Platform), then builds or -downloads the Dot's images; run it again after a restart and it carries on. Then open **http://127.0.0.2:3000**, -paste your OpenRouter key and create your first Dot. Every step and its failure modes: -[the guide's quickstart](docs/guide.md#quickstart). - -## What to ask a Dot - -Anything that needs a computer and a person's judgement, for as long as it takes: - -> Open a browser identity called research, go to https://example.com and tell me the page's title and its first -> sentence. Leave the browser open. - -> Every morning, check one-way fares from Milan to Lisbon for the next two weeks, write them to -> ~/workspace/fares.csv and tell me the cheapest day. - -> Log in to the shop with the shopping identity, download this month's invoices to ~/documents, and remember where -> the invoices page is for next time. - -## What a Dot has - -| | | +| Step | What happens | |---|---| -| **[A computer of its own](docs/guide.md#what-a-dot-can-do)** | A hardware-accelerated VM with a Linux desktop and a persistent disk. It sleeps when idle and wakes for the next message, task or automation. | -| **[A browser that is not blocked](docs/guide.md#the-browser)** | [invisible_playwright_mcp](https://github.com/feder-cr/invisible_playwright_mcp): Firefox patched in C++, the fingerprint set inside the engine. Each identity keeps its own cookies and logins. | -| **[Memory and skills](docs/guide.md#what-a-dot-can-do)** | It writes its own notes and how-tos in its home folder, as Claude Code does, and reads them on the next task. | -| **[Your MCP servers](docs/guide.md#mcp-servers)** | Any MCP server you declare, a program on its computer or a URL, as in Claude Code. Its keys stay out of the config. | -| **[Permissions you set](docs/guide.md#approvals)** | Every tool belongs to a permission: allow, ask or deny. An ask waits in your Inbox, survives a restart and runs the call once. | -| **[Tasks and automations](docs/guide.md#the-web-ui)** | A queue with priorities and start times, and its own recurring jobs, for which its computer is started on time. | -| **[Many ways to reach it](docs/guide.md#talk-to-it-from-your-phone)** | Web UI, command line, HTTP API with live events, Telegram, and WhatsApp as an opt-in. | -| **[Nothing lost on a crash](docs/guide.md#how-it-works)** | A restart or a `kill -9` keeps every message and task you saw accepted, and a tool call cut short is never run twice. | +| **Find** | Queries the daily database in place, keeping the titles and locations you asked for, without downloading the whole of it. | +| **Judge** | Scores each job from 1 to 5 by reasoning about your CV against the posting, and reports the gaps. | +| **Tailor** | Reorders and rephrases your CV for the job. Every fact must already be in your CV. | +| **Apply** | Fills the form through [invisible-playwright-mcp](https://github.com/feder-cr/invisible_playwright_mcp) with what your CV and answers file say. What they do not say comes back to you as a question, never as a guess. | -## Documentation +The model never sends an application. It fills the form and names the submit button; AIHawk clicks it after your yes. A click by the model on anything that would send the form is refused. -- [Guide](docs/guide.md): install, the web UI, the command line, the browser, channels, configuration, updating -- [Architecture](docs/architecture.md): every component and why it is there -- [Troubleshooting](docs/guide.md#troubleshooting) and [development and tests](docs/guide.md#development-and-tests) +`aihawk apply --auto-submit` sends without asking. An automated application in your name can be wrong, and sites can close accounts for it: read the forms first, and use it only when you accept that. -## Related projects +## Models -**The pieces of this one.** A Dot's browser is -[invisible_playwright_mcp](https://github.com/feder-cr/invisible_playwright_mcp), on -[invisible_playwright](https://github.com/feder-cr/invisible_playwright) and -[invisible_core](https://github.com/feder-cr/invisible_core); the engine is a fork of -[nanobot](https://github.com/HKUDS/nanobot). +AIHawk talks to one gateway, [OrcaRouter](https://orcarouter.ai), with `orcarouter/auto` as the default model. Change `model` in `aihawk.yaml`, or point `base_url` at any endpoint compatible with OpenAI's API that supports tool calling and structured output. + +## Limits + +- AIHawk fills forms on Greenhouse, Lever and Ashby. Other postings are found and listed, to apply to by hand. +- Forms behind a login or a captcha come back as blocked; AIHawk does not try to get around them. +- The tailored PDF uses a Latin font, so CVs in other scripts need your own PDF. + +## Thanks + +The job database is built from the public databases listed in [NOTICE](NOTICE); each posting belongs to its employer and links back to it. The scoring rubric's shape is adapted from [career-ops](https://github.com/santifer/career-ops) (MIT). + +The Dots project that used to live here moved to [feder-cr/invisible_dots](https://github.com/feder-cr/invisible_dots). ## License -MIT, see [LICENSE](LICENSE). Third-party components and their licenses: [THIRD_PARTY_NOTICES.md](THIRD_PARTY_NOTICES.md). - -## Disclaimer - -This project is provided as-is, with no warranties. Use it at your own risk and in compliance with the laws of your -jurisdiction. A Dot acts with your accounts and from your connection: respect the terms of the sites it visits and -their robots.txt. - ---- - -

- tests - license: MIT - status: alpha - hosts: Linux and Windows -

- -

- Built by Federico Elia -

+[MIT](LICENSE) diff --git a/SECURITY.md b/SECURITY.md deleted file mode 100644 index a752d756..00000000 --- a/SECURITY.md +++ /dev/null @@ -1,40 +0,0 @@ -# Security Policy - -## Supported versions - -Only `main` receives fixes: nothing is packaged yet, and you build from this repository. - -## Reporting a vulnerability - -Do not report security issues in public issues, discussions or pull requests. - -Send a report to `federico.elia.majo@gmail.com` with the subject prefix `[security][invisible_dots]`, with: - -- what the issue is and what it affects -- the steps to reproduce -- the commit (`git rev-parse --short HEAD`), the host OS and `invisible-dots doctor --json` -- a fix, if you have one - -What a Dot is and is not protected against is written down in -[the guide's security model](docs/guide.md#security-model-and-known-limits): a way around one of those guarantees -is in scope. - -## Scope - -In scope: - -- The control plane: API, scheduler, VM manager, database, web client and command line -- The guest: dot-agentd, the engine and the separation between them and the Dot's user -- The images and how they are built and downloaded - -Out of scope: - -- The Dot's browser: report to [invisible_playwright_mcp](https://github.com/feder-cr/invisible_playwright_mcp) or - [invisible_playwright](https://github.com/feder-cr/invisible_playwright) -- QEMU, Firefox and other upstream projects: report to them directly - -Not security issues: - -- A site detecting or blocking the browser: open a regular issue in - [invisible_playwright](https://github.com/feder-cr/invisible_playwright/issues) -- What a model chooses to do with permissions you set to allow diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md deleted file mode 100644 index 4b6306da..00000000 --- a/THIRD_PARTY_NOTICES.md +++ /dev/null @@ -1,634 +0,0 @@ -# Third-party notices - -The MIT license in `LICENSE` covers this repository, except: - -- everything under `invisible_engine_dots/`, a hard fork of nanobot, which - is under `invisible_engine_dots/LICENSE` and the nested notice listed in - its section below. - -That fork, the text this repository's history holds through the earlier -TypeScript engine, and the web client's files that derive from other projects -(each starts with a `Derived from` comment naming its source, and the project -has a section below) come with the notices below. The default `npm install` -holds nothing under the GPL -(the golden image's guest operating system is Ubuntu, which carries GPL software -under its own licenses, downloaded from Ubuntu's archive by whoever builds the -image: docs/architecture.md section 11.3); the -one GPL-3.0 dependency, `libsignal` of the optional WhatsApp client, is left out -of the default install and is described in the section "The opt-in WhatsApp -client and `libsignal` (GPL-3.0)". - -## Open Multi-Agent - -The earlier TypeScript engine (`guest-runtime/engine/`) was derived in part -from it. That engine is deleted; its files and its `UPSTREAM.md` (source, -version and commit) are in this repository's history, before the commit -"repo: delete the earlier TypeScript engine and agent", and this notice stays -for as long as that history does. - -```text -MIT License - -Copyright (c) Shenzhen YuanASI Technology Co., Ltd. and open-multi-agent contributors - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -``` - -## context-chef - -`@context-chef/core` 4.2.1. Open Multi-Agent's `agent/runner.ts`, as imported -into this repository's history, contained `groupIntoTurns` and -`stripMediaBlocksForSummary`, modelled on this library (the deleted engine's -`UPSTREAM.md` records the details). - -```text -MIT License - -Copyright (c) 2025 MyPrototypeWhat - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -``` - -## Context compaction (OpenHands, OpenAI Codex, opencode) - -How the engine compacts a request that reaches its model's context window -(`invisible_engine_dots/nanobot/agent/context_governance.py`, `memory.py` and -`templates/agent/consolidator_archive.md`) takes from three projects: - -- the headings of the summary (USER_CONTEXT, TASK_TRACKING, COMPLETED, PENDING, - CURRENT_STATE, CODE_STATE, TESTS, CHANGES, DEPS, VERSION_CONTROL_STATUS) from - the summarizing prompt of OpenHands' software-agent-sdk - (`openhands-sdk/openhands/sdk/context/condenser/prompts/summarizing_system.j2`); -- the handoff framing of the summary and of the text that introduces it, the - person's latest messages kept as they wrote them up to 20000 tokens, and - dropping the oldest messages when the summary's own request does not fit, - from OpenAI Codex (`codex-rs/prompts/templates/compact/`, - `codex-rs/core/src/compact.rs`), under the Apache License 2.0 - (https://www.apache.org/licenses/LICENSE-2.0), with this notice: - -```text -OpenAI Codex -Copyright 2025 OpenAI -``` - -- the room kept for the answer (20000 tokens) and the thresholds for clearing old - tool results (the newest 40000 tokens kept, 20000 freed at least) from - opencode (`packages/opencode/src/session/overflow.ts` and `compaction.ts`). - -```text -MIT License - -Copyright (c) 2026 OpenHands contributors - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -``` - -```text -MIT License - -Copyright (c) 2025 opencode - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -``` - -## nanobot - -`invisible_engine_dots/` is imported from nanobot -(https://github.com/HKUDS/nanobot, commit -`f75470e72f0993dcf92accc81282adaa48b16f56`) and modified for invisible_dots. -It is a hard fork: upstream changes are not tracked or merged. Source, commit, -what the import left out and what was removed since: -`invisible_engine_dots/UPSTREAM.md`. Its -`LICENSE`, byte for byte: - -```text -MIT License - -Copyright (c) 2025-present Xubin Ren and the nanobot contributors - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -``` - -The fork carries one further notice, which stays next to the code it covers: - -- `invisible_engine_dots/THIRD_PARTY_NOTICES.md`: says that no third-party - component is vendored in the fork. - -Its Python dependencies are not part of this repository: the golden image's -build installs them from PyPI as the wheels their publishers released, pinned -by hash in `guest/image-builder/builder/engine-requirements.lock`, each under -its own license. The runtime disk carries the fork's own source with its -`LICENSE` and `UPSTREAM.md` (`/opt/invisible-dots/engine/` in the guest). - -Two files of the web client derive from nanobot's web UI instead (the -`webui/` of the upstream repository at commit 9dc0aba, under the same license -as above): `apps/web/src/components/chat/activity-step.tsx` and -`apps/web/src/components/channels/qr-connect.tsx`. The first comment of each -says what was changed. - -## hev-socks5-tunnel (in the golden image) - -The golden image carries the static binary `hev-socks5-tunnel` 2.18.0 -(https://github.com/heiher/hev-socks5-tunnel), pinned by URL and SHA-256 in -`guest/image-builder/pins.json` and downloaded when the image is built. It -routes a Dot's VM through its VM proxy. It is MIT-licensed: - -> Copyright (c) 2022 hev -> -> Permission is hereby granted, free of charge, to any person obtaining a copy of -> this software and associated documentation files (the "Software"), to deal in -> the Software without restriction, including without limitation the rights to -> use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -> of the Software, and to permit persons to whom the Software is furnished to do -> so, subject to the following conditions: -> -> The above copyright notice and this permission notice shall be included in all -> copies or substantial portions of the Software. -> -> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -> SOFTWARE. - -## shadcn/ui - -The primitives in `apps/web/src/components/ui/` and `apps/web/src/lib/utils.ts` -were generated from shadcn/ui (https://github.com/shadcn-ui/ui, commit -0e3abd65) and changed; each file's first comment says what changed. - -```text -MIT License - -Copyright (c) 2023 shadcn - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -``` - -## OpenDots (CopilotKit) - -`apps/web/src/lib/time.ts`, `apps/web/src/lib/use-visible-interval.ts`, -`packages/shared/src/labels.ts` and -`apps/web/src/components/tasks/task-status.tsx` derive from OpenDots by CopilotKit -(https://github.com/CopilotKit/OpenDots, commit 88f2a08); each file's first -comment says what was changed. - -```text -The MIT License - -Copyright (c) Atai Barkai - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -``` - -## AI Elements (Vercel) - -`apps/web/src/components/chat/conversation.tsx` and -`apps/web/src/components/chat/tool-status.tsx` derive from AI Elements by Vercel -(https://github.com/vercel/ai-elements, commit 6a9d5b1), licensed under the Apache -License, Version 2.0, which is given once, below. Each file's first comment states -what was changed, as the license asks. - -```text -Copyright 2023 Vercel, Inc. - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - Copyright 2022 Joe Bell - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -``` - -## open-cowork - -`apps/web/src/components/computer/screen-view.tsx` derives from open-cowork by -Coasty (https://github.com/coasty-ai/open-cowork, commit fbbc671); its first comment -says what was changed. No asset of that project is used. - -```text -MIT License - -Copyright (c) 2026 Coasty / open-cowork contributors - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -``` - -## assistant-ui - -`apps/web/src/components/elements/approval-card.tsx` and -`apps/web/src/lib/range.ts` derive from assistant-ui -(https://github.com/assistant-ui/assistant-ui, commit 0bdf050); each file's first -comment says what was changed. - -```text -MIT License - -Copyright (c) 2026 AgentbaseAI Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -``` - -## OpenDots (Shashankss1205) - -`apps/web/src/components/approvals/diff-preview.tsx` and -`apps/web/src/lib/diff.ts` derive from OpenDots by Shashank Shekhar Singh -(https://github.com/Shashankss1205/OpenDots, commit bb8db95), which includes source -previously distributed under the name Spots (Copyright (c) 2026 Spots contributors, -also MIT); each file's first comment says what was changed. No other project of the -same name is meant: the CopilotKit one has its own section above. - -```text -MIT License - -Copyright (c) 2026 Shashank Shekhar Singh - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -``` - -## The opt-in WhatsApp client and `libsignal` (GPL-3.0) - -The default `npm install` holds nothing under the GPL. The WhatsApp adapter -(`packages/channels/src/whatsapp-baileys/`) uses Baileys (MIT), a client of the -WhatsApp Web protocol, which depends on the npm package `libsignal`, under the -GNU General Public License, version 3, and so does not fit this repository's MIT -license. They are therefore not part of the default install: - -- No workspace declares Baileys, as a dependency of any kind, and the root - `package-lock.json` holds none of it or of what only it needs, so `npm ci` - installs no GPL package (`tests/repo/default-install-licenses.test.ts` scans - the lock file for licenses). -- The client lives apart, in `optional/whatsapp/`, with a `package.json` that - pins Baileys to one exact release and a `package-lock.json` with the - integrity hash of every package. A person who turns WhatsApp on installs it - with one command from the repository root: `npm run whatsapp:install` (a - clean install of that lock file, without install scripts). Installing it is - that person's choice, and what they then hold, Baileys and `libsignal`, - is under those licenses. -- The adapter loads the client from that folder, by path, only when a - connection opens and only if it is there. Otherwise it says how to enable it: - run `npm run whatsapp:install`, then start the server with - `INVISIBLE_DOTS_WHATSAPP=1`. -- The command bundle that `apps/cli/scripts/build.mjs` makes does not hold it - (the bundle has no import of it), so no build of this repository is a GPL - work. Whoever distributes a build together with an installed - `optional/whatsapp/node_modules`, which is GPL-3.0 code, has to take that into - account. - -## LGPL image libraries that Next.js may install - -The web client's framework, Next.js, lists `sharp` (an image library, -Apache-2.0) as an optional dependency, and `sharp` lists prebuilt binaries for -each platform, `@img/sharp-*` and `@img/sharp-libvips-*`, which carry libvips -under the LGPL-3.0 (`LGPL-3.0-or-later`, or `Apache-2.0 AND LGPL-3.0-or-later` -for the Windows builds). `npm ci` installs the ones that fit the host, as -optional packages. They are LGPL, not GPL, and unmodified; whoever distributes -an installation together with its `node_modules` has to meet the LGPL for them. -`tests/repo/default-install-licenses.test.ts` allows no LGPL package in the lock -file other than these. diff --git a/apps/api/package.json b/apps/api/package.json deleted file mode 100644 index 0e174c03..00000000 --- a/apps/api/package.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "name": "@invisible-dots/api", - "version": "0.0.0", - "private": true, - "description": "Control-plane HTTP API and SSE stream, and startServer: the control plane composed as one process (invisible-dots server).", - "license": "MIT", - "type": "module", - "exports": { - ".": "./src/index.ts" - }, - "scripts": { - "typecheck": "tsc -p tsconfig.json" - }, - "dependencies": { - "@invisible-dots/shared": "0.0.0", - "fastify": "^5.12.5", - "@invisible-dots/channels": "0.0.0", - "@invisible-dots/database": "0.0.0", - "@invisible-dots/events": "0.0.0", - "@invisible-dots/image-builder": "0.0.0", - "@invisible-dots/scheduler": "0.0.0", - "@invisible-dots/sdk": "0.0.0", - "@invisible-dots/vm-manager": "0.0.0" - } -} diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts deleted file mode 100644 index a615dde2..00000000 --- a/apps/api/src/config.ts +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Process settings of the control plane: the listen address, the API token - * and the master key (architecture sections 3.2 and 9.6). Both secrets live - * in INVISIBLE_DOTS_HOME/config and are created at the first start, so a new - * host needs no installer step before `invisible-dots server`. - */ -import { randomBytes } from "node:crypto"; -import { MASTER_KEY_BYTES } from "@invisible-dots/database"; -import { DEFAULT_LISTEN, ENV, readApiToken, readSecretFile, writeSecretFile, type HostPaths } from "@invisible-dots/shared"; - -/** A generated token: 32 random bytes as hex, so it survives copy and paste and any shell quoting. */ -const GENERATED_TOKEN_BYTES = 32; - -export interface ListenAddress { - host: string; - port: number; -} - -/** What differs between the settings parseListen reads. */ -export interface ListenSetting { - /** The environment variable the value came from, named in the errors. */ - variable: string; - /** The address shown as the example in the error: the setting's own default. */ - example: string; - /** Whether port 0 (the system picks one) is refused, for a server whose address the person has to be told. */ - fixedPort: boolean; -} - -export const API_LISTEN: ListenSetting = { variable: ENV.LISTEN, example: DEFAULT_LISTEN, fixedPort: false }; - -/** Parse `host:port`, `[v6]:port` or a bare port (which binds 127.0.0.1). */ -export function parseListen(value: string = DEFAULT_LISTEN, setting: ListenSetting = API_LISTEN): ListenAddress { - const text = value.trim(); - let host = "127.0.0.1"; - let portText = text; - const v6 = /^\[([^\]]+)\]:(\d+)$/.exec(text); - if (v6) { - host = v6[1]!; - portText = v6[2]!; - } else if (text.includes(":")) { - const at = text.lastIndexOf(":"); - host = text.slice(0, at); - portText = text.slice(at + 1); - } - const port = Number(portText); - if (!host || !/^\d+$/.test(portText) || port < 0 || port > 65535) { - throw new Error(`${setting.variable} must look like "${setting.example}", got "${value}"`); - } - if (setting.fixedPort && port === 0) throw new Error(`${setting.variable} needs a fixed port, got "${value}"`); - return { host, port }; -} - -export interface LoadedSecret { - value: T; - /** Where it came from: a file path or an environment variable name. */ - origin: string; - /** True when this call generated it and wrote the file. */ - created: boolean; -} - -/** - * The API token as readApiToken reads it (INVISIBLE_DOTS_TOKEN, else the - * first line of `config/api.token`); the file is generated (0600) when it - * does not exist yet. - */ -export async function loadOrCreateApiToken( - paths: HostPaths, - env: Record = process.env, -): Promise> { - const existing = await readApiToken(env, paths); - if (existing) return { ...existing, created: false }; - const path = paths.apiTokenPath; - const token = randomBytes(GENERATED_TOKEN_BYTES).toString("hex"); - await writeSecretFile(path, `${token}\n`); - return { value: token, origin: path, created: true }; -} - -/** - * The master key that encrypts secrets in the database: `config/master.key`, - * 32 raw bytes (64 hex characters are accepted too, so a key restored by - * hand works). When the file does not exist a new key is generated but NOT - * written: `created` tells the caller to check first that the database - * holds nothing encrypted under a key that was lost, and only then to call - * `saveMasterKey`. Writing it earlier would turn a missing key into a wrong - * one that the next start accepts without a word. - */ -export async function loadOrGenerateMasterKey(paths: HostPaths): Promise> { - const path = paths.masterKeyPath; - const raw = await readSecretFile(path); - if (raw === undefined) return { value: randomBytes(MASTER_KEY_BYTES), origin: path, created: true }; - if (raw.length === MASTER_KEY_BYTES) return { value: raw, origin: path, created: false }; - const hex = raw.toString("utf8").trim(); - if (new RegExp(`^[0-9a-fA-F]{${MASTER_KEY_BYTES * 2}}$`).test(hex)) return { value: Buffer.from(hex, "hex"), origin: path, created: false }; - throw new Error(`${path} must hold ${MASTER_KEY_BYTES} bytes (or ${MASTER_KEY_BYTES * 2} hexadecimal characters), found ${raw.length} bytes`); -} - -/** Write a key from `loadOrGenerateMasterKey` (0600, atomically). */ -export async function saveMasterKey(paths: HostPaths, key: Uint8Array): Promise { - await writeSecretFile(paths.masterKeyPath, key); -} diff --git a/apps/api/src/database.ts b/apps/api/src/database.ts deleted file mode 100644 index a4a6e245..00000000 --- a/apps/api/src/database.ts +++ /dev/null @@ -1,59 +0,0 @@ -/** - * Opening the control plane's database at start: the adapter the - * environment names (PGlite in INVISIBLE_DOTS_HOME/db, or DATABASE_URL), - * migrated, and checked against a master key that was lost. - */ -import { Database, databaseTarget, describeDatabaseTarget } from "@invisible-dots/database"; -import { errorMessage, type Logger } from "@invisible-dots/shared"; - -export interface OpenDatabaseOptions { - env: Record; - masterKey: Uint8Array; - logger: Logger; -} - -/** Open and migrate; a failure names the database and is never left half open. */ -export async function openControlPlaneDatabase(options: OpenDatabaseOptions): Promise { - const target = databaseTarget(options.env); - const where = describeDatabaseTarget(target); - // Both adapters already name the database they could not open (pg without - // its password), so the error is passed on as it is. - const db = await Database.open({ target, masterKey: options.masterKey }); - try { - // Before anything reads or writes: a second server on the same database - // (another INVISIBLE_DOTS_HOME with the same DATABASE_URL) would take - // over the first one's Dots. - if (!(await db.holdInstanceLock())) { - throw new Error(`another invisible-dots server already uses the ${where}; stop it first (one server per database)`); - } - } catch (error) { - await db.close().catch(() => {}); - throw error; - } - try { - const migrated = await db.migrate({ log: (line) => options.logger.info(line) }); - options.logger.info("database ready", { - database: where, - applied: migrated.applied.length, - total: migrated.applied.length + migrated.alreadyApplied.length, - }); - } catch (error) { - await db.close().catch(() => {}); - throw new Error(`cannot migrate the ${where}: ${errorMessage(error)}`, { cause: error }); - } - return db; -} - -/** - * Refuse a newly generated master key for a database that already holds - * values encrypted under another one: the key file was lost or deleted, - * and going on would leave every Dot token and secret unreadable. - */ -export async function assertNothingEncrypted(db: Database, masterKeyPath: string): Promise { - if (await db.holdsEncryptedValues()) { - throw new Error( - `${masterKeyPath} does not exist, but the database holds Dot tokens or secrets encrypted under a master key: ` + - `restore the original master.key (a new one cannot read them), then start again`, - ); - } -} diff --git a/apps/api/src/doctor.ts b/apps/api/src/doctor.ts deleted file mode 100644 index 16babee5..00000000 --- a/apps/api/src/doctor.ts +++ /dev/null @@ -1,55 +0,0 @@ -/** - * The doctor rows only the control plane's side can read: the images a Dot - * would get now. The rest of the report is the vm-manager's `runDoctor` - * (architecture section 11.1); `startServer` serves it as `GET /api/doctor` - * and `invisible-dots doctor` runs it from the command line, both over this - * one function. - */ -import { stat } from "node:fs/promises"; -import { basename } from "node:path"; -import { manifestPathFor, verifyImage, type ImageCheck } from "@invisible-dots/image-builder"; -import type { DoctorCheck, HostPaths } from "@invisible-dots/shared"; -import { latestImage } from "./vm-driver.js"; - -/** The size and modification time of a file, or "absent": what a verdict on it depends on. */ -async function fingerprint(path: string): Promise { - const info = await stat(path).catch(() => undefined); - return info ? `${info.size}:${info.mtimeMs}` : "absent"; -} - -/** - * The doctor's image rows for one host. `verifyImage` re-hashes a whole - * multi-GB image, and the onboarding checklist asks again and again, so a - * verdict is kept for as long as the image and its manifest keep the same - * size and modification time: the hash reruns only when a file changes. The - * command line runs it once per command and loses nothing. - */ -export function createImageChecks(paths: HostPaths, verify: (image: string) => Promise = verifyImage): () => Promise { - const verdicts = new Map(); - - async function verified(image: string): Promise { - const current = `${await fingerprint(image)}|${await fingerprint(manifestPathFor(image))}`; - const known = verdicts.get(image); - if (known?.fingerprint === current) return known.check; - const check = await verify(image); - verdicts.set(image, { fingerprint: current, check }); - return check; - } - - /** The image a Dot would get now (the newest one, as the control plane picks it), checked against its manifest. */ - async function imageCheck(id: "golden-image" | "runtime-image"): Promise { - const golden = id === "golden-image"; - const label = golden ? "golden image" : "runtime ISO"; - let image: string; - try { - image = await latestImage(paths.imagesDir, golden ? "golden" : "runtime"); - } catch { - return { id, label, status: "missing", detail: `none in ${paths.imagesDir}`, fix: "invisible-dots image build" }; - } - const verdict = await verified(image); - if (verdict.ok) return { id, label, status: "ok", detail: `${basename(image)} matches its manifest` }; - return { id, label, status: "failed", detail: verdict.reason, fix: verdict.fix }; - } - - return async () => [await imageCheck("golden-image"), await imageCheck("runtime-image")]; -} diff --git a/apps/api/src/file-types.ts b/apps/api/src/file-types.ts deleted file mode 100644 index 142ece9b..00000000 --- a/apps/api/src/file-types.ts +++ /dev/null @@ -1,22 +0,0 @@ -/** - * How a file of a Dot's computer is served by `GET /api/dots/:id/files`. The bytes are the Dot's own, written - * by a model that may have read hostile text, and the web server answers from the same origin as the page, so a - * file is never given a type a browser would run: images are images, text (source and markup included) is - * `text/plain`, and everything else is a download. - */ -import { fileType } from "@invisible-dots/shared"; - -export interface FileServing { - contentType: string; - /** The `Content-Disposition` header: `inline` for what a page may show, `attachment` for the rest. */ - disposition: string; -} - -/** The type and disposition for the file at `path` (a guest path). */ -export function serveFile(path: string): FileServing { - const name = path.slice(path.lastIndexOf("/") + 1); - const { kind, contentType } = fileType(name); - // The plain filename is for old clients (printable ASCII, no quote, backslash or percent); filename* carries the real one. - const plain = name.replace(/[^\x20-\x7e]|["\\%]/g, "_"); - return { contentType, disposition: `${kind === "other" ? "attachment" : "inline"}; filename="${plain}"; filename*=UTF-8''${encodeURIComponent(name)}` }; -} diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts deleted file mode 100644 index 85924b8f..00000000 --- a/apps/api/src/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export * from "./config.js"; -export * from "./database.js"; -export * from "./doctor.js"; -export * from "./lock.js"; -export * from "./server.js"; -export * from "./start.js"; -export * from "./vm-driver.js"; diff --git a/apps/api/src/lock.ts b/apps/api/src/lock.ts deleted file mode 100644 index 0cdf4e5c..00000000 --- a/apps/api/src/lock.ts +++ /dev/null @@ -1,22 +0,0 @@ -/** - * One control plane per INVISIBLE_DOTS_HOME. Two servers on the same home - * would both open the embedded PGlite data directory, which has no locking - * of its own and is corrupted by a second writer, and both would start and - * stop the same QEMU processes. A lock file holding the owner's pid stops - * the second one before it touches anything; the lock itself is the shared - * pid lock of packages/shared, which also serves the image builds. - */ -import { acquirePidLock, ensurePrivateDir, type HostPaths, type PidLock, type ProcessPresence } from "@invisible-dots/shared"; - -export type ServerLock = PidLock; - -/** Take the server lock of `paths.home`, or fail naming the pid that holds it and the file to remove. */ -export async function acquireServerLock(paths: HostPaths, presence?: (pid: number) => ProcessPresence): Promise { - // The whole data directory is this user's alone, on Windows too (files.ts). - await ensurePrivateDir(paths.home); - return acquirePidLock(paths.serverLockPath, { - what: "invisible-dots server", - stopHint: `stop it first (it serves ${paths.home})`, - ...(presence ? { presence } : {}), - }); -} diff --git a/apps/api/src/server.ts b/apps/api/src/server.ts deleted file mode 100644 index 4bb00fcd..00000000 --- a/apps/api/src/server.ts +++ /dev/null @@ -1,516 +0,0 @@ -/** - * The control-plane HTTP API of architecture section 9.6. Every route needs - * the bearer token; every error is `{ error, message }`. The handlers are - * thin: the work happens in the Scheduler. - */ -import { createHash, timingSafeEqual } from "node:crypto"; -import { TELEGRAM_TOKEN_SECRET, type ChannelHub } from "@invisible-dots/channels"; -import { StreamOverflowError } from "@invisible-dots/events"; -import { type Scheduler } from "@invisible-dots/scheduler"; -import { STREAM_ERROR_EVENT } from "@invisible-dots/sdk"; -import type { - ApprovalsAnswer, - ChannelPairingAnswer, - ChannelRecord, - ChannelsAnswer, - EveryChannelAnswer, - DoctorAnswer, - DotsAnswer, - EventsAnswer, - FilesListAnswer, - HealthResponse, - HostFacts, - IdentitiesAnswer, - MessagesAnswer, - TasksAnswer, - UsageAnswer, -} from "@invisible-dots/sdk/types"; -import { APPROVAL_LIST_LIMIT, APPROVAL_STATUSES, type ApprovalStatus, CHANNEL_KINDS, type ChannelKind, ControlPlaneError, CONVERSATION_LIST_LIMIT, type DoctorCheck, errorMessage, LIST_ORDERS, type ListOrder, type Logger, MAX_EVENT_PAGE, type McpSecretsAnswer, silentLogger, type SkillListAnswer, TASK_LIST_LIMIT, type ToolListAnswer } from "@invisible-dots/shared"; -import { doctorAnswer } from "@invisible-dots/vm-manager"; -import { serveFile } from "./file-types.js"; -import Fastify, { type FastifyInstance, type FastifyReply, type FastifyRequest } from "fastify"; -import apiPackage from "../package.json" with { type: "json" }; - -/** The version `GET /api/health` reports: this package's, as the CLI's `--version` reports its own. */ -export const API_VERSION: string = apiPackage.version; - -/** A comment line on idle SSE connections, so proxies and clients see the connection is alive. */ -export const SSE_HEARTBEAT_MS = 15_000; - -export interface ServerOptions { - scheduler: Scheduler; - channels: ChannelHub; - /** Runs the host checks of architecture section 11.1 on the machine this server runs on; `startServer` binds the real ones. */ - doctor(): Promise; - /** What `GET /api/health` says about where the state lives; `startServer` reads it from the paths and the database it opened. */ - host: HostFacts; - token: string; - logger?: Logger; - heartbeatMs?: number; -} - -type Params = { id: string; identityId: string; kind: string; peer: string }; -type Body = Record | undefined; - -function digest(value: string): Buffer { - return createHash("sha256").update(value, "utf8").digest(); -} - -function bad(message: string): ControlPlaneError { - return new ControlPlaneError(400, "invalid_request", message); -} - -/** A non-negative integer query parameter, or undefined when absent. */ -function intParam(value: unknown, name: string, max = Number.MAX_SAFE_INTEGER): number | undefined { - if (value === undefined || value === "") return undefined; - if (typeof value !== "string" || !/^\d+$/.test(value) || Number(value) > max) { - throw bad(`${name} must be a non-negative integer${max < Number.MAX_SAFE_INTEGER ? ` up to ${max}` : ""}`); - } - return Number(value); -} - -/** The `order` of a list route, which only an `order=desc` list can be paged on (`before`) from. */ -function orderParam(order: unknown, before: unknown): ListOrder | undefined { - if (order !== undefined && !(LIST_ORDERS as readonly unknown[]).includes(order)) throw bad(`order must be one of ${LIST_ORDERS.join(", ")}`); - if (before !== undefined && order !== "desc") throw bad("before pages a list in order=desc"); - return order as ListOrder | undefined; -} - -/** An ISO 8601 date-time query parameter, or undefined when absent. */ -function sinceParam(value: unknown): Date | undefined { - if (value === undefined || value === "") return undefined; - const parsed = typeof value === "string" && /^\d{4}-\d{2}-\d{2}T/.test(value) ? new Date(value) : undefined; - if (parsed === undefined || Number.isNaN(parsed.getTime())) { - throw bad("since must be an ISO 8601 timestamp such as 2026-10-05T00:00:00Z"); - } - return parsed; -} - -function channelKind(value: string): ChannelKind { - if (!(CHANNEL_KINDS as readonly string[]).includes(value)) throw bad(`no "${value}" channel: the channels are ${CHANNEL_KINDS.join(", ")}`); - return value as ChannelKind; -} - -function bodyOf(request: FastifyRequest): Record { - const body = request.body as Body; - if (body === undefined || body === null) return {}; - if (typeof body !== "object" || Array.isArray(body)) throw bad("the body must be a JSON object"); - return body; -} - -export function buildServer(options: ServerOptions): FastifyInstance { - const { scheduler, channels } = options; - const log = options.logger ?? silentLogger; - const expected = digest(options.token); - const heartbeatMs = options.heartbeatMs ?? SSE_HEARTBEAT_MS; - const streams = new Set(); - - const app = Fastify({ logger: false, bodyLimit: 1024 * 1024, return503OnClosing: true }); - - app.addHook("onRequest", async (request, reply) => { - const header = request.headers.authorization ?? ""; - const match = /^Bearer\s+(.+)$/i.exec(header); - // Comparing digests keeps the comparison constant-time whatever the length of the guess. - if (!match || !timingSafeEqual(digest(match[1]!.trim()), expected)) { - return reply - .code(401) - .header("www-authenticate", 'Bearer realm="invisible-dots"') - .send({ error: "unauthorized", message: "a valid Authorization: Bearer header is required" }); - } - }); - - app.setNotFoundHandler((request, reply) => { - void reply.code(404).send({ error: "not_found", message: `no route ${request.method} ${request.url.split("?")[0]}` }); - }); - - app.setErrorHandler((error: unknown, request, reply) => { - if (error instanceof ControlPlaneError) { - const body: Record = { error: error.code, message: error.message }; - if (error.details !== undefined) body.details = error.details; - // A busy browser is the answer a polling page expects, not a failure. - if (error.status >= 500 && error.code !== "busy") log.error("request failed", { method: request.method, url: request.url, error: error.message }); - return reply.code(error.status).send(body); - } - const e = error as { statusCode?: number; code?: string; message?: string }; - if (typeof e.statusCode === "number" && e.statusCode >= 400 && e.statusCode < 500) { - const code = e.code === "FST_ERR_CTP_INVALID_MEDIA_TYPE" ? "unsupported_media_type" : "invalid_request"; - return reply.code(e.statusCode).send({ error: code, message: e.message ?? "invalid request" }); - } - log.error("unhandled error", { method: request.method, url: request.url, error: errorMessage(error) }); - return reply.code(500).send({ error: "internal", message: errorMessage(error) }); - }); - - /** - * Answer with a server-sent event stream, the one way every stream of this API starts: headers, a heartbeat comment - * so proxies and clients see the connection is alive, and an abort when the client goes or the server closes. `body` - * writes the frames and returns when there are no more. - */ - const eventStream = async (reply: FastifyReply, controller: AbortController, body: (raw: FastifyReply["raw"]) => Promise): Promise => { - reply.hijack(); - const raw = reply.raw; - raw.writeHead(200, { - "content-type": "text/event-stream; charset=utf-8", - "cache-control": "no-store", - connection: "keep-alive", - "x-accel-buffering": "no", - }); - raw.write(": connected\n\n"); - streams.add(controller); - const heartbeat = setInterval(() => raw.write(": ping\n\n"), heartbeatMs); - heartbeat.unref(); - raw.on("close", () => controller.abort()); - try { - await body(raw); - } finally { - clearInterval(heartbeat); - streams.delete(controller); - raw.end(); - } - }; - - // Before the server stops listening and waits for its connections: a stream only ends when it is told to, so a hook that - // ran after that wait (`onClose`) would wait for the person to close the tab. - app.addHook("preClose", async () => { - for (const controller of streams) controller.abort(); - }); - - app.get("/api/health", async (): Promise => { - const { database, openrouter_configured } = await scheduler.health(); - return { status: "ok", database, version: API_VERSION, openrouter_configured, ...options.host }; - }); - - // The host report `invisible-dots doctor` prints: what is missing for a Dot to run, and the command that fixes it. It runs QEMU's accelerator probe, so it takes a moment: - // one run at a time, and a request that arrives while it runs shares its answer (the onboarding checklist re-checks and polls). - let doctorRun: Promise | undefined; - app.get("/api/doctor", (): Promise => { - doctorRun ??= options.doctor().then(doctorAnswer).finally(() => { - doctorRun = undefined; - }); - return doctorRun; - }); - - // Dots - - app.post("/api/dots", async (request, reply) => { - const dot = await scheduler.createDot(bodyOf(request).config); - return reply.code(201).send(dot); - }); - - app.get("/api/dots", async (): Promise => ({ dots: await scheduler.listDots() })); - - app.get<{ Params: Params }>("/api/dots/:id", async (request) => scheduler.requireDot(request.params.id)); - - app.patch<{ Params: Params }>("/api/dots/:id", async (request) => { - const { config, expected_config_version } = bodyOf(request); - return scheduler.updateDot(request.params.id, config, expected_config_version); - }); - - app.delete<{ Params: Params }>("/api/dots/:id", async (request, reply) => { - return reply.code(202).send(await scheduler.deleteDot(request.params.id)); - }); - - // Messages and tasks - - app.post<{ Params: Params }>("/api/dots/:id/messages", async (request, reply) => { - return reply.code(202).send(await scheduler.sendMessage(request.params.id, bodyOf(request).text)); - }); - - // The conversation pages like the event log: oldest first by default, `order=desc` the newest, `before` (the - // event id of the oldest message of the previous page) the ones older than that. - app.get<{ Params: Params; Querystring: { limit?: string; order?: unknown; before?: string } }>( - "/api/dots/:id/messages", - async (request): Promise => { - const limit = intParam(request.query.limit, "limit", CONVERSATION_LIST_LIMIT); - if (limit === 0) throw bad(`limit must be between 1 and ${CONVERSATION_LIST_LIMIT}`); - const before = intParam(request.query.before, "before"); - const order = orderParam(request.query.order, before); - return { messages: await scheduler.conversation(request.params.id, { limit, order, before }) }; - }, - ); - - app.post<{ Params: Params }>("/api/dots/:id/tasks", async (request, reply) => { - const body = bodyOf(request); - const task = await scheduler.createTask(request.params.id, { - description: body.description as string, - priority: body.priority as number | undefined, - scheduled_at: body.scheduled_at as string | undefined, - }); - return reply.code(201).send(task); - }); - - // The newest tasks first; `before` (the id of the last task of the previous page) goes on, older. - app.get<{ Params: Params; Querystring: { limit?: unknown; before?: unknown } }>("/api/dots/:id/tasks", async (request): Promise => { - const limit = intParam(request.query.limit, "limit", TASK_LIST_LIMIT); - if (limit === 0) throw bad(`limit must be between 1 and ${TASK_LIST_LIMIT}`); - const { before } = request.query; - if (before !== undefined && (typeof before !== "string" || before === "")) throw bad("before must be the id of a task"); - return { tasks: await scheduler.listTasks(request.params.id, { limit, before }) }; - }); - - // The tasks of every Dot that ended in a state after a time, the latest end first (the web's failed tasks of the day). - app.get<{ Querystring: { status?: unknown; finished_after?: unknown } }>("/api/tasks", async (request): Promise => ({ - tasks: await scheduler.listEndedTasks(request.query.status, request.query.finished_after), - })); - - app.get<{ Params: Params }>("/api/tasks/:id", async (request) => scheduler.getTask(request.params.id)); - - app.post<{ Params: Params }>("/api/tasks/:id/cancel", async (request) => scheduler.cancelTask(request.params.id)); - - // Computer - - app.get<{ Params: Params }>("/api/dots/:id/computer", async (request) => scheduler.computer(request.params.id)); - - const lifecycleRoute = (action: "start" | "stop" | "reboot") => - app.post<{ Params: Params }>(`/api/dots/:id/computer/${action}`, async (request, reply) => { - const id = request.params.id; - const answer = - action === "start" - ? await scheduler.startComputer(id) - : action === "stop" - ? await scheduler.stopComputer(id) - : await scheduler.rebootComputer(id); - return reply.code(202).send(answer); - }); - lifecycleRoute("start"); - lifecycleRoute("stop"); - lifecycleRoute("reboot"); - - app.get<{ Params: Params }>("/api/dots/:id/computer/screenshot", async (request, reply) => { - const png = await scheduler.screenshot(request.params.id); - return reply.code(200).header("content-type", "image/png").header("cache-control", "no-store").send(Buffer.from(png)); - }); - - // Browser identities - - app.get<{ Params: Params }>( - "/api/dots/:id/browser-identities", - async (request): Promise => ({ identities: await scheduler.listIdentities(request.params.id) }), - ); - - app.post<{ Params: Params }>("/api/dots/:id/browser-identities", async (request, reply) => { - const body = bodyOf(request); - const identity = await scheduler.createIdentity(request.params.id, { - name: body.name as string, - proxy: body.proxy as string | undefined, - }); - return reply.code(201).send(identity); - }); - - app.get<{ Params: Params }>("/api/dots/:id/browser-identities/:identityId", async (request) => - scheduler.getIdentity(request.params.id, request.params.identityId), - ); - - app.delete<{ Params: Params }>("/api/dots/:id/browser-identities/:identityId", async (request, reply) => { - await scheduler.deleteIdentity(request.params.id, request.params.identityId); - return reply.code(204).send(); - }); - - app.get<{ Params: Params }>("/api/dots/:id/browser-identities/:identityId/frame", async (request, reply) => { - const jpeg = await scheduler.identityFrame(request.params.id, request.params.identityId); - return reply.code(200).header("content-type", "image/jpeg").header("cache-control", "no-store").send(Buffer.from(jpeg)); - }); - - app.post<{ Params: Params }>("/api/dots/:id/browser-identities/:identityId/close", async (request, reply) => { - await scheduler.closeIdentity(request.params.id, request.params.identityId); - return reply.code(204).send(); - }); - - // The tools (its table) and the skills are the engine's, so they need the computer running - - app.get<{ Params: Params }>( - "/api/dots/:id/tools", - async (request): Promise => scheduler.listTools(request.params.id), - ); - - app.get<{ Params: Params }>( - "/api/dots/:id/skills", - async (request): Promise => ({ skills: await scheduler.listSkills(request.params.id) }), - ); - - // Channels (the hub never returns a credential, and no route here echoes one) - - app.get<{ Params: Params }>( - "/api/dots/:id/channels", - async (request): Promise => ({ channels: await channels.list(request.params.id), available: channels.kinds }), - ); - - // Every channel of every Dot, each with its Dot's id (the web's shell watches the ones that need the person). - app.get("/api/channels", async (): Promise => ({ channels: await channels.listEvery() })); - - // Link the Dot to a Telegram bot, or give the linked one a new token (a revoked token is the only way back from needs_relink). - app.put<{ Params: Params }>("/api/dots/:id/channels/telegram", async (request, reply): Promise => { - const token = bodyOf(request).token; - if (typeof token !== "string" || token.trim() === "") throw bad("token must be the bot token from @BotFather"); - const credentials = { [TELEGRAM_TOKEN_SECRET]: token.trim() }; - const linked = (await channels.list(request.params.id)).some((channel) => channel.kind === "telegram"); - if (linked) return channels.setCredentials(request.params.id, "telegram", credentials); - return reply.code(201).send(await channels.add(request.params.id, "telegram", { credentials })); - }); - - // Link WhatsApp: the channel starts and shows a code; the person scans it (the stream below) with the phone that holds the number. - app.post<{ Params: Params }>("/api/dots/:id/channels/whatsapp/link", async (request, reply): Promise => { - return reply.code(202).send(await channels.link(request.params.id, "whatsapp")); - }); - - // The codes to scan and how the link ends, as server-sent events of `ChannelLinkFrame`. The code is a way into the account for as long as it is shown: it is not stored and the reply is never cached. - app.get<{ Params: Params }>("/api/dots/:id/channels/whatsapp/qr", async (request, reply) => { - const controller = new AbortController(); - const frames = await channels.watchLink(request.params.id, "whatsapp", controller.signal); - await eventStream(reply, controller, async (raw) => { - for await (const frame of frames) raw.write(`data: ${JSON.stringify(frame)}\n\n`); - }); - }); - - app.patch<{ Params: Params }>("/api/dots/:id/channels/:kind", async (request): Promise => { - const kind = channelKind(request.params.kind); - const { settings, enabled } = bodyOf(request); - if (settings === undefined && enabled === undefined) throw bad("give settings, enabled, or both"); - if (enabled !== undefined && typeof enabled !== "boolean") throw bad("enabled must be true or false"); - let record: ChannelRecord | undefined; - if (settings !== undefined) record = await channels.setSettings(request.params.id, kind, settings); - if (enabled !== undefined) record = await channels.setEnabled(request.params.id, kind, enabled); - return record!; - }); - - app.delete<{ Params: Params }>("/api/dots/:id/channels/:kind", async (request, reply) => { - await channels.remove(request.params.id, channelKind(request.params.kind)); - return reply.code(204).send(); - }); - - app.post<{ Params: Params }>("/api/dots/:id/channels/:kind/pairing", async (request, reply): Promise => { - return reply.code(201).send(await channels.pair(request.params.id, channelKind(request.params.kind))); - }); - - app.delete<{ Params: Params }>("/api/dots/:id/channels/:kind/peers/:peer", async (request, reply) => { - await channels.removePeer(request.params.id, channelKind(request.params.kind), request.params.peer); - return reply.code(204).send(); - }); - - // Approvals - - app.get<{ Querystring: { status?: unknown; limit?: unknown; order?: unknown; before?: unknown; dot_id?: unknown } }>("/api/approvals", async (request): Promise => { - const { status, order, before, dot_id: dotId } = request.query; - if (dotId !== undefined && (typeof dotId !== "string" || dotId === "")) throw bad("dot_id must be the id or the name of one Dot"); - if (status !== undefined && typeof status !== "string") throw bad("status must be one list, comma-separated"); - const statuses = status?.split(",").map((one) => one.trim()); - if (statuses?.some((one) => !(APPROVAL_STATUSES as readonly string[]).includes(one))) { - throw bad(`status must be one or more of ${APPROVAL_STATUSES.join(", ")}`); - } - const limit = intParam(request.query.limit, "limit", APPROVAL_LIST_LIMIT); - if (limit === 0) throw bad(`limit must be between 1 and ${APPROVAL_LIST_LIMIT}`); - if (before !== undefined && (typeof before !== "string" || before === "")) throw bad("before must be the id of an approval"); - return { - approvals: await scheduler.listApprovals(statuses as ApprovalStatus[] | undefined, { limit, order: orderParam(order, before), before, dot: dotId }), - }; - }); - - for (const decision of ["approve", "reject"] as const) { - app.post<{ Params: Params }>(`/api/approvals/:id/${decision}`, async (request) => { - const { note, always } = bodyOf(request); - return scheduler.resolveApproval(request.params.id, decision, { note, always }); - }); - } - - // Events and usage - - app.get<{ Params: Params; Querystring: { since?: string } }>( - "/api/dots/:id/usage", - async (request): Promise => scheduler.usage(request.params.id, sinceParam(request.query.since)), - ); - - app.get<{ Params: Params; Querystring: { after?: string; before?: string; limit?: string; types?: unknown; tools?: unknown; task_id?: unknown; order?: unknown } }>( - "/api/dots/:id/events", - async (request): Promise => { - const after = intParam(request.query.after, "after"); - const before = intParam(request.query.before, "before"); - const limit = intParam(request.query.limit, "limit", MAX_EVENT_PAGE); - const { types, tools, task_id: taskId, order } = request.query; - if (types !== undefined && typeof types !== "string") throw bad("types must be one comma-separated list"); - if (tools !== undefined && typeof tools !== "string") throw bad("tools must be one comma-separated list"); - if (taskId !== undefined && typeof taskId !== "string") throw bad("task_id must be a single value"); - const listOrder = orderParam(order, request.query.before); - const list = (value: string | undefined) => value?.split(",").map((one) => one.trim()).filter(Boolean); - return { - events: await scheduler.listEvents(request.params.id, { - after, - before, - limit, - types: list(types), - tools: list(tools), - taskId, - order: listOrder, - }), - }; - }, - ); - - // Files of the Dot's computer, under /home/dot, read-only; they need the computer running (409 computer_stopped). - - app.get<{ Params: Params; Querystring: { path?: unknown } }>( - "/api/dots/:id/files/list", - async (request): Promise => scheduler.listFiles(request.params.id, request.query.path as string | undefined), - ); - - app.get<{ Params: Params; Querystring: { path?: unknown } }>("/api/dots/:id/files", async (request, reply) => { - const { path, content } = await scheduler.readFile(request.params.id, request.query.path); - const { contentType, disposition } = serveFile(path); - return reply - .code(200) - .header("content-type", contentType) - .header("content-disposition", disposition) - .header("x-content-type-options", "nosniff") - .header("content-security-policy", "default-src 'none'; sandbox") - .header("cache-control", "no-store") - .send(Buffer.from(content)); - }); - - app.get<{ Querystring: { dot_id?: string; after?: string } }>("/api/stream", async (request, reply) => { - const dotId = request.query.dot_id ? await scheduler.historyDotId(request.query.dot_id) : undefined; - // EventSource sends the last id it saw when it reconnects; it wins over the query. - const lastEventId = request.headers["last-event-id"]; - const after = intParam(typeof lastEventId === "string" ? lastEventId : request.query.after, "after"); - - const controller = new AbortController(); - await eventStream(reply, controller, async (raw) => { - try { - for await (const event of scheduler.events.stream(dotId === undefined ? {} : { dotId }, { after, signal: controller.signal })) { - raw.write(`id: ${event.id}\ndata: ${JSON.stringify(event)}\n\n`); - } - } catch (error) { - // The client reconnects with its last id; tell it why the server hung up. - const code = error instanceof StreamOverflowError ? "stream_overflow" : "stream_error"; - raw.write(`event: ${STREAM_ERROR_EVENT}\ndata: ${JSON.stringify({ error: code, message: errorMessage(error) })}\n\n`); - log.warn("event stream closed with an error", { error: errorMessage(error) }); - } - }); - }); - - // Secrets - - app.put("/api/secrets/openrouter", async (request) => { - const body = bodyOf(request); - return scheduler.setOpenRouterKey(body.value, body.dot_id); - }); - - // A Dot's VM proxy: write-only, the answers say only whether there is one. - app.get<{ Params: { id: string } }>("/api/dots/:id/proxy", async (request) => scheduler.vmProxy(request.params.id)); - app.put<{ Params: { id: string } }>("/api/dots/:id/proxy", async (request) => scheduler.setVmProxy(request.params.id, bodyOf(request).value)); - app.delete<{ Params: { id: string } }>("/api/dots/:id/proxy", async (request) => scheduler.setVmProxy(request.params.id, null)); - - // The secrets of a Dot's MCP servers: write-only, the answers say only which are set. - type McpSecretParams = { id: string; server: string; name: string }; - app.get<{ Params: Params }>("/api/dots/:id/mcp-secrets", async (request): Promise => scheduler.mcpSecrets(request.params.id)); - app.put<{ Params: McpSecretParams }>( - "/api/dots/:id/mcp-secrets/:server/:name", - async (request): Promise => // A PUT sets: a null value is no value (a clear is the DELETE). - scheduler.setMcpSecret(request.params.id, request.params.server, request.params.name, bodyOf(request).value ?? undefined), - ); - app.delete<{ Params: McpSecretParams }>( - "/api/dots/:id/mcp-secrets/:server/:name", - async (request): Promise => scheduler.setMcpSecret(request.params.id, request.params.server, request.params.name, null), - ); - - return app; -} - -export type { FastifyInstance, FastifyReply }; diff --git a/apps/api/src/start.ts b/apps/api/src/start.ts deleted file mode 100644 index cdef0a43..00000000 --- a/apps/api/src/start.ts +++ /dev/null @@ -1,217 +0,0 @@ -/** - * The control plane as one process (architecture section 2): the database, - * the vm-manager, the scheduler and the HTTP API, composed in the order a - * first start needs. `invisible-dots server` calls `startServer` and waits - * with `untilStopSignal`; tests call `startServer` and close the handle - * themselves. - */ -import { fileURLToPath } from "node:url"; -import { - ChannelHub, - TelegramChannelType, - WhatsAppChannelType, - whatsappClientDir, - whatsappClientProblem, - type ChannelType, -} from "@invisible-dots/channels"; -import type { Database } from "@invisible-dots/database"; -import type { HostFacts } from "@invisible-dots/sdk/types"; -import { Scheduler, type ComputerDriver, type SchedulerOptions } from "@invisible-dots/scheduler"; -import { ensureDir, ENV, errorMessage, hostPaths, type HostPaths, type Logger, prefixedStderrLogger } from "@invisible-dots/shared"; -import { hostDoctorDeps, openRouterCheck, runDoctor, type DoctorDeps } from "@invisible-dots/vm-manager"; -import { loadOrCreateApiToken, loadOrGenerateMasterKey, parseListen, saveMasterKey, type ListenAddress } from "./config.js"; -import { assertNothingEncrypted, openControlPlaneDatabase } from "./database.js"; -import { createImageChecks } from "./doctor.js"; -import { acquireServerLock } from "./lock.js"; -import { API_VERSION, buildServer, type FastifyInstance } from "./server.js"; -import { createVmDriver } from "./vm-driver.js"; - -export interface StartServerOptions { - /** Where every setting is read from; default `process.env`. */ - env?: Record; - /** Overrides INVISIBLE_DOTS_LISTEN; port 0 picks a free port (tests). */ - listen?: string; - logger?: Logger; - /** The VM layer; default the real one over QEMU. Tests pass a fake. */ - driver?: ComputerDriver; - /** Replaces single dependencies of the host report (QEMU discovery, the runner, the disk, the images); the OpenRouter row always asks the Scheduler. Tests pass fakes. */ - doctor?: Partial; - /** Passed through to the Scheduler (timers, lifecycle and dispatcher tuning). */ - scheduler?: Omit; - /** The kinds of messaging channel this server can run; default Telegram, and WhatsApp when INVISIBLE_DOTS_WHATSAPP=1. Tests pass fakes. */ - channelTypes?: readonly ChannelType[]; - /** Where the opt-in WhatsApp client is installed; default `optional/whatsapp` of the repository. Tests point it at a folder of their own. */ - whatsappClientDir?: string; -} - -/** The logger of the `invisible-dots` process: lines on stderr prefixed with its name, debug lines when INVISIBLE_DOTS_DEBUG=1. */ -export function serverLogger(env: Record = process.env): Logger { - return prefixedStderrLogger("invisible-dots", env.INVISIBLE_DOTS_DEBUG === "1"); -} - -export interface RunningServer { - /** The base URL clients use, e.g. http://127.0.0.1:8787. */ - url: string; - address: ListenAddress; - token: string; - paths: HostPaths; - /** "pglite" or "pg", as the database adapter reports it. */ - database: string; - /** Secrets this start created because they did not exist yet. */ - created: { apiToken: boolean; masterKey: boolean }; - app: FastifyInstance; - scheduler: Scheduler; - /** The messaging channels of the Dots: pairing, and the messages between a chat and its Dot. */ - channels: ChannelHub; - db: Database; - /** Stop the API, the channels, the scheduler and the database. VMs keep running. Idempotent. */ - close(): Promise; -} - -/** - * Where the opt-in WhatsApp client is installed, under the repository root. apps/api/src/start.ts and the command - * bundle apps/cli/dist/invisible-dots.mjs sit at the same depth, so the root is the same for both. - */ -const DEFAULT_CLIENT_DIR = whatsappClientDir(fileURLToPath(new URL("../../../", import.meta.url))); - -/** - * The channels a server runs when it is not told otherwise. WhatsApp is opt-in twice: its adapter is an unofficial - * client that WhatsApp can answer with a ban of the linked account, so a server that was not asked for it neither - * loads nor offers it; and its client library is not part of the default install (it is GPL-3.0 through libsignal), - * so it is installed by the person who turns WhatsApp on. - */ -export function defaultChannelTypes(env: Record, clientDir: string = DEFAULT_CLIENT_DIR): ChannelType[] { - return [new TelegramChannelType(), ...(env[ENV.WHATSAPP] === "1" ? [new WhatsAppChannelType({ baileys: { clientDir } })] : [])]; -} - -/** - * Start the control plane and resolve once the API listens. Every resource - * taken before a failure is released again, so a failed start leaves no - * lock, open database or listening socket behind. - */ -export async function startServer(options: StartServerOptions = {}): Promise { - const env = options.env ?? process.env; - const debug = env.INVISIBLE_DOTS_DEBUG === "1"; - const logger = options.logger ?? serverLogger(env); - const paths = hostPaths(env); - const listen = parseListen(options.listen ?? (env[ENV.LISTEN] || undefined)); - - const cleanup: (() => Promise)[] = []; - const unwind = async () => { - for (const step of cleanup.splice(0).reverse()) { - await step().catch((error) => logger.error("cleanup step failed", { error: errorMessage(error) })); - } - }; - - try { - const lock = await acquireServerLock(paths); - cleanup.push(() => lock.release()); - await ensureDir(paths.logsDir, 0o700); - - const [token, masterKey] = await Promise.all([loadOrCreateApiToken(paths, env), loadOrGenerateMasterKey(paths)]); - if (token.created) logger.info("created the API token", { path: token.origin }); - - const db = await openControlPlaneDatabase({ env, masterKey: masterKey.value, logger }); - cleanup.push(() => db.close()); - if (masterKey.created) { - // Only an empty database may get a new key; see loadOrGenerateMasterKey. - await assertNothingEncrypted(db, masterKey.origin); - await saveMasterKey(paths, masterKey.value); - logger.info("created the master key", { path: masterKey.origin }); - } - - const driver = options.driver ?? createVmDriver({ env, paths, logger: prefixedStderrLogger("vm-manager", debug) }); - const scheduler = new Scheduler({ - ...options.scheduler, - db, - driver, - logger: options.logger ?? prefixedStderrLogger("scheduler", debug), - }); - cleanup.push(() => scheduler.close()); - const clientDir = options.whatsappClientDir ?? DEFAULT_CLIENT_DIR; - const clientProblem = !options.channelTypes && env[ENV.WHATSAPP] === "1" ? whatsappClientProblem(clientDir) : null; - if (clientProblem) logger.warn(`${ENV.WHATSAPP}=1 is set. ${clientProblem.message}`); - // Next to the Scheduler, using only what it offers; registered after it, so it stops first. - const channels = new ChannelHub({ - db, - host: scheduler, - types: options.channelTypes ?? defaultChannelTypes(env, clientDir), - logger: options.logger ?? prefixedStderrLogger("channels", debug), - }); - cleanup.push(() => channels.close()); - const doctorDeps: DoctorDeps = { - ...hostDoctorDeps({ - env, - home: paths.home, - images: createImageChecks(paths), - openRouterKey: async () => openRouterCheck((await scheduler.health()).openrouter_configured), - }), - ...options.doctor, - }; - const facts: HostFacts = { database_kind: db.kind, data_dir: paths.home, logs_dir: paths.logsDir }; - const app = buildServer({ scheduler, channels, doctor: () => runDoctor(doctorDeps), host: facts, token: token.value, logger }); - cleanup.push(() => app.close()); - - await scheduler.start(); - await channels.start(); - await app.listen({ host: listen.host, port: listen.port }); - const bound = app.server.address(); - const address: ListenAddress = typeof bound === "object" && bound ? { host: listen.host, port: bound.port } : listen; - const host = address.host.includes(":") ? `[${address.host}]` : address.host; - const url = `http://${host}:${address.port}`; - logger.info(`invisible-dots server ${API_VERSION} listening`, { url, home: paths.home, database: db.kind }); - - let closing: Promise | null = null; - return { - url, - address, - token: token.value, - paths, - database: db.kind, - created: { apiToken: token.created, masterKey: masterKey.created }, - app, - scheduler, - channels, - db, - close: () => (closing ??= unwind()), - }; - } catch (error) { - await unwind(); - throw error; - } -} - -/** - * Every way the person or the system asks a foreground server to stop: - * Ctrl+C, a service manager, a closed terminal (SIGHUP; Node raises it on - * Windows too when the console window is closed, a few seconds before - * Windows ends the process) and Ctrl+Break on Windows (SIGBREAK). All take - * the same clean path, which closes the embedded database and releases the - * lock. They are event names Node accepts on both hosts; one that a host - * never raises simply never fires. - */ -export const STOP_SIGNALS = ["SIGINT", "SIGTERM", "SIGHUP", "SIGBREAK"] as const; - -/** - * Resolve once `close` has finished after the first stop signal; a second - * signal exits at once. `signals` is the process, or a stand-in in tests. - */ -export function untilStopSignal(close: () => Promise, logger: Logger, signals: NodeJS.EventEmitter = process): Promise { - return new Promise((resolve) => { - let stopping = false; - const onSignal = (signal: NodeJS.Signals) => { - if (stopping) { - logger.warn(`second ${signal}, exiting without waiting`); - process.exit(1); - } - stopping = true; - logger.info(`${signal} received, shutting down (VMs keep running)`); - void close().finally(() => { - for (const name of STOP_SIGNALS) signals.off(name, onSignal); - logger.info("stopped"); - resolve(); - }); - }; - for (const name of STOP_SIGNALS) signals.on(name, onSignal); - }); -} diff --git a/apps/api/src/vm-driver.ts b/apps/api/src/vm-driver.ts deleted file mode 100644 index 22b08052..00000000 --- a/apps/api/src/vm-driver.ts +++ /dev/null @@ -1,137 +0,0 @@ -/** - * The real ComputerDriver: an adapter over vm-manager's VmManager (QEMU - * started directly, sections 3.4 and 3.5). It adds the one decision - * VmManager leaves to its caller, which images a computer uses: - * - * - a new computer gets the newest `golden-.qcow2`; its overlay - * stays on that golden image for its whole life (a backing file never - * changes under a disk); - * - every start uses the newest `runtime-.iso`, so a new version of - * our code reaches a Dot with a restart (section 3.3). - */ -import { readdir } from "node:fs/promises"; -import { join } from "node:path"; -import { type ComputerDriver, type ComputerSpecInput, type ComputerState, type CreatedComputer, type GuestApi, type GuestEndpoint, type StartedComputer, type WaitForHealthOptions } from "@invisible-dots/scheduler"; -import { type HealthAnswer, type HostPaths, type ImageKind, imageLabel, imageVersionOf, type Logger, pathExists } from "@invisible-dots/shared"; -import { VmManager, type StartVmResult, type VmSpec } from "@invisible-dots/vm-manager"; - -/** Compare versions such as "2026.10.1" and "2026.9.3" numerically, part by part. */ -export function compareVersions(a: string, b: string): number { - return a.localeCompare(b, "en", { numeric: true, sensitivity: "base" }); -} - -/** The image of that kind in `dir` with the highest version, as an absolute path. */ -export async function latestImage(dir: string, kind: ImageKind): Promise { - const label = imageLabel(kind); - let names: string[]; - try { - names = await readdir(dir); - } catch (error) { - throw new Error(`cannot list the images directory ${dir}: ${(error as Error).message}`, { cause: error }); - } - const candidates = names - .map((name) => ({ name, version: imageVersionOf(kind, name) })) - .filter((c): c is { name: string; version: string } => c.version !== undefined) - .sort((x, y) => compareVersions(x.version, y.version)); - const newest = candidates.at(-1); - if (!newest) throw new Error(`no ${label} in ${dir}: build one with "invisible-dots image build"`); - return join(dir, newest.name); -} - -export const latestGoldenImage = (imagesDir: string) => latestImage(imagesDir, "golden"); -export const latestRuntimeImage = (imagesDir: string) => latestImage(imagesDir, "runtime"); - -export class VmManagerDriver implements ComputerDriver { - readonly #vm: VmManager; - readonly #imagesDir: string; - - constructor(vm: VmManager, options: { imagesDir?: string } = {}) { - this.#vm = vm; - this.#imagesDir = options.imagesDir ?? vm.paths.imagesDir; - } - - latestGolden(): Promise { - return latestGoldenImage(this.#imagesDir); - } - - latestRuntime(): Promise { - return latestRuntimeImage(this.#imagesDir); - } - - #spec(input: ComputerSpecInput, goldenImage: string, runtimeImage: string): VmSpec { - return { - dotId: input.dotId, - token: input.token, - ...(input.proxy ? { proxy: input.proxy } : {}), - goldenImage, - runtimeImage, - cpus: input.resources.cpus, - memoryMiB: input.resources.memoryMiB, - diskBytes: input.resources.diskBytes, - }; - } - - #started(result: StartVmResult, runtimeImage: string): StartedComputer { - return { guestPort: result.guestPort, pid: result.pid, runtimeImage, alreadyRunning: result.alreadyRunning }; - } - - async create(input: ComputerSpecInput): Promise { - const [golden, runtime] = await Promise.all([this.latestGolden(), this.latestRuntime()]); - await this.#vm.create(this.#spec(input, golden, runtime)); - return { goldenImage: golden, runtimeImage: runtime }; - } - - async start(input: ComputerSpecInput & { goldenImage: string }): Promise { - const runtime = await this.latestRuntime(); - const spec = this.#spec(input, input.goldenImage, runtime); - // A bigger computer.disk from a PATCH: the overlay can only grow while the VM is off. - if ((await this.#vm.state(input.dotId)).state === "STOPPED") await this.#vm.resize(spec); - return this.#started(await this.#vm.start(spec), runtime); - } - - waitForHealth(endpoint: GuestEndpoint, token: string, options: WaitForHealthOptions): Promise { - return this.#vm.waitForGuestHealth(endpoint.dotId, endpoint.port, token, { - timeoutMs: options.timeoutMs, - intervalMs: options.intervalMs, - requestTimeoutMs: options.requestTimeoutMs, - }); - } - - async stop(dotId: string, token: string): Promise<{ forced: boolean }> { - const { forced } = await this.#vm.stop(dotId, token); - return { forced }; - } - - async reboot(input: ComputerSpecInput & { goldenImage: string }): Promise { - const runtime = await this.latestRuntime(); - return this.#started(await this.#vm.reboot(this.#spec(input, input.goldenImage, runtime)), runtime); - } - - destroy(dotId: string): Promise { - return this.#vm.destroy(dotId); - } - - async state(dotId: string): Promise { - const [info, exists] = await Promise.all([this.#vm.state(dotId), pathExists(this.#vm.paths.diskPath(dotId))]); - return { - exists, - state: info.state, - pid: info.pid, - guestPort: info.guestPort, - detail: info.detail ?? null, - }; - } - - guest(endpoint: GuestEndpoint, token: string): GuestApi { - return this.#vm.guestClient(endpoint.port, token); - } - - /** VmManager holds no connection between calls, so there is nothing to release. */ - async close(): Promise {} -} - - -/** The driver `invisible-dots server` uses: VmManager over this host's QEMU and INVISIBLE_DOTS_HOME. */ -export function createVmDriver(options: { env: Record; paths: HostPaths; logger: Logger }): VmManagerDriver { - return new VmManagerDriver(new VmManager({ env: options.env, paths: options.paths, logger: options.logger })); -} diff --git a/apps/api/test/api.test.ts b/apps/api/test/api.test.ts deleted file mode 100644 index 3aee7b84..00000000 --- a/apps/api/test/api.test.ts +++ /dev/null @@ -1,779 +0,0 @@ -import type { AddressInfo } from "node:net"; -import { ChannelHub } from "@invisible-dots/channels"; -import { FakeChannelType } from "@invisible-dots/channels/testing"; -import type { Database } from "@invisible-dots/database"; -import { createTestDatabase, testAdapters, type TestDatabase } from "@invisible-dots/database/testing"; -import { Scheduler } from "@invisible-dots/scheduler"; -import { FakeDriver, ManualClock, waitFor, waitUntilSettledReady } from "@invisible-dots/scheduler/testing"; -import { ApiError, InvisibleDotsClient } from "@invisible-dots/sdk"; -import { MAX_EVENT_PAGE, OPENROUTER_KEY_RULE, type DoctorCheck, type StoredEvent } from "@invisible-dots/shared"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { API_VERSION, buildServer, type FastifyInstance } from "../src/index.js"; -import { hostFacts } from "./host-facts.js"; - -const TOKEN = "test-token-0123456789abcdef"; - -/** What the host report answers on this fake host: one row that is not ok, with the command that fixes it. */ -const REPORT: DoctorCheck[] = [ - { id: "node", label: "Node", status: "ok", detail: "24.19.0" }, - { id: "qemu", label: "QEMU", status: "missing", detail: "qemu-system-x86_64 not found", fix: "invisible-dots setup" }, -]; - -const yaml = (name: string, idle = "15m") => - `name: ${name}\nmodel:\n provider: openrouter\n id: test/model\ncomputer:\n idle_timeout: ${idle}\n`; - -describe.each(testAdapters())("control-plane API (%s)", (kind) => { - let t: TestDatabase; - let db: Database; - let app: FastifyInstance; - let scheduler: Scheduler; - let channels: ChannelHub; - let driver: FakeDriver; - let clock: ManualClock; - let base: string; - let api: InvisibleDotsClient; - - beforeAll(async () => { - t = await createTestDatabase(kind); - db = t.db; - // A Dot reaches READY only with a key to push (lifecycle #push): every test that readies one needs it, so it is - // stored here and not by whichever test happened to run first. - await db.secrets.put("global", "openrouter_api_key", "sk-or-test"); - driver = new FakeDriver(); - clock = new ManualClock(); - scheduler = new Scheduler({ - db, - driver, - clock, - lifecycle: { healthPollMs: 5, readyTimeoutMs: 3_000, pumpRetryMs: 10 }, - dispatcher: { retryDelayMs: 0 }, - }); - channels = new ChannelHub({ db, host: scheduler, types: [new FakeChannelType()] }); - app = buildServer({ scheduler, channels, doctor: async () => REPORT, host: hostFacts(kind), token: TOKEN, heartbeatMs: 50 }); - await app.listen({ host: "127.0.0.1", port: 0 }); - base = `http://127.0.0.1:${(app.server.address() as AddressInfo).port}`; - api = new InvisibleDotsClient({ baseUrl: base, token: TOKEN }); - }); - - afterAll(async () => { - await app?.close(); - await channels?.close(); - await scheduler?.close(); - await t?.drop(); - }); - - async function readyDot(name: string, idle?: string) { - const dot = await api.createDot(yaml(name, idle)); - await waitUntilSettledReady(scheduler, driver, dot.id, name); - return dot; - } - - it("refuses requests without the right bearer token with 401 JSON", async () => { - const none = await fetch(`${base}/api/dots`); - expect(none.status).toBe(401); - expect(await none.json()).toMatchObject({ error: "unauthorized" }); - const wrong = await fetch(`${base}/api/health`, { headers: { authorization: "Bearer nope" } }); - expect(wrong.status).toBe(401); - const stream = await fetch(`${base}/api/stream`, { headers: { authorization: "Basic abc" } }); - expect(stream.status).toBe(401); - await expect(new InvisibleDotsClient({ baseUrl: base, token: "wrong-token-xxxxxxxx" }).health()).rejects.toMatchObject({ - status: 401, - code: "unauthorized", - }); - }); - - it("doctor: the host report with its verdict, only with the token", async () => { - expect((await fetch(`${base}/api/doctor`)).status).toBe(401); - expect(await api.doctor()).toEqual({ ok: false, checks: REPORT }); - }); - - it("health, unknown routes and malformed bodies answer {error, message}", async () => { - expect(await api.health()).toEqual({ status: "ok", database: "ok", version: API_VERSION, openrouter_configured: true, ...hostFacts(kind) }); - const missing = await fetch(`${base}/api/nope`, { headers: { authorization: `Bearer ${TOKEN}` } }); - expect(missing.status).toBe(404); - expect(await missing.json()).toMatchObject({ error: "not_found" }); - const broken = await fetch(`${base}/api/dots`, { - method: "POST", - headers: { authorization: `Bearer ${TOKEN}`, "content-type": "application/json" }, - body: "{not json", - }); - expect(broken.status).toBe(400); - expect(await broken.json()).toMatchObject({ error: "invalid_request" }); - }); - - it("usage: the spend the guest reported, since a moment, from the event log", async () => { - const dot = await readyDot("usage-one"); - const other = await readyDot("usage-two"); - expect(await api.usage(dot.id)).toEqual({ dot_id: dot.id, since: null, spent_usd: 0 }); - const guest = driver.guestOf(dot.id); - guest.onInbound = () => {}; - const task = await api.createTask(dot.id, { description: "costly" }); - await waitFor(async () => (await db.tasks.runs(task.id))[0]?.delivered_at, "delivered"); - guest.emit("task.progress", { task_id: task.id, text: "step", spent_usd: 0.25 }); - guest.emit("task.completed", { task_id: task.id, summary: "done", spent_usd: 0.5 }); - guest.emit("message.assistant", { text: "hello", spent_usd: 0.125 }); - driver.guestOf(other.id).emit("message.assistant", { text: "not mine", spent_usd: 9 }); - await waitFor(async () => (await api.getTask(task.id)).status === "COMPLETED", "completed"); - await waitFor(async () => (await api.usage(dot.id)).spent_usd === 0.625, "chat answer stored"); - expect(await api.usage(dot.name)).toEqual({ dot_id: dot.id, since: null, spent_usd: 0.625 }); - expect((await api.getTask(task.id)).spent_usd).toBe(0.5); - expect((await api.listTasks(dot.id))[0]?.spent_usd).toBe(0.5); - - const future = new Date(Date.now() + 3_600_000).toISOString(); - expect(await api.usage(dot.id, { since: future })).toEqual({ dot_id: dot.id, since: future, spent_usd: 0 }); - const past = new Date(Date.now() - 3_600_000).toISOString(); - expect((await api.usage(dot.id, { since: past })).spent_usd).toBe(0.625); - }); - - it("usage: a bad since is 400, an unknown Dot 404, and a deleted Dot's history stays readable by id", async () => { - const dot = await readyDot("usage-three"); - await expect(api.usage(dot.id, { since: "yesterday" })).rejects.toMatchObject({ status: 400, code: "invalid_request" }); - await expect(api.usage(dot.id, { since: "2026-13-45T00:00:00Z" })).rejects.toMatchObject({ status: 400 }); - await expect(api.usage("no-such-dot")).rejects.toMatchObject({ status: 404 }); - driver.guestOf(dot.id).emit("message.assistant", { text: "paid", spent_usd: 0.5 }); - await waitFor(async () => (await api.usage(dot.id)).spent_usd === 0.5, "spend stored"); - await api.deleteDot(dot.id); - await waitFor(async () => (await db.dots.get(dot.id)) === null, "deleted"); - expect(await api.usage(dot.id)).toEqual({ dot_id: dot.id, since: null, spent_usd: 0.5 }); - }); - - it("create validates the config: 400 with the issues, 409 for a taken name", async () => { - const invalid = await api.createDot({ name: "Bad Name", model: { provider: "openrouter", id: "m" } }).catch((e) => e); - expect(invalid).toBeInstanceOf(ApiError); - expect(invalid).toMatchObject({ status: 400, code: "invalid_config" }); - expect((invalid as ApiError).details).toEqual([expect.objectContaining({ path: "name" })]); - await readyDot("taken-name"); - await expect(api.createDot(yaml("taken-name"))).rejects.toMatchObject({ status: 409, code: "name_taken" }); - }); - - it("create -> READY -> task -> guest events -> COMPLETED, visible in tasks, events and the stream", async () => { - const controller = new AbortController(); - const seen: StoredEvent[] = []; - const streaming = (async () => { - for await (const event of api.stream({ signal: controller.signal })) { - seen.push(event); - if (event.type === "task.completed") break; - } - })(); - - const dot = await readyDot("flow-dot"); - expect((await api.listDots()).map((d) => d.name)).toContain("flow-dot"); - expect(await api.getDot("flow-dot")).toMatchObject({ id: dot.id, computer_state: "RUNNING" }); - const computer = await api.computer(dot.id); - expect(computer).toMatchObject({ state: "RUNNING", ready: true }); - - const task = await api.createTask("flow-dot", { description: "find the cheapest day", priority: 1 }); - expect(task.status).toBe("PENDING"); - await waitFor(async () => (await api.getTask(task.id)).status === "COMPLETED", "task COMPLETED"); - expect((await api.listTasks(dot.id)).map((x) => x.id)).toContain(task.id); - await streaming; - controller.abort(); - expect(seen.map((e) => e.type)).toEqual(expect.arrayContaining(["dot.created", "computer.started", "task.created", "task.completed"])); - - const events = await api.events(dot.id); - expect(events.find((e) => e.type === "task.completed")?.data).toMatchObject({ task_id: task.id }); - const after = events[1]!.id; - expect((await api.events(dot.id, { after, limit: 2 })).map((e) => e.id)).toEqual(events.slice(2, 4).map((e) => e.id)); - await expect(api.events(dot.id, { limit: MAX_EVENT_PAGE + 1 })).rejects.toMatchObject({ status: 400 }); - await expect(api.cancelTask(task.id)).rejects.toMatchObject({ status: 409, code: "task_finished" }); - }); - - it("events: filtered by type names and by task, together and with after and limit", async () => { - const dot = await readyDot("event-filters"); - const guest = driver.guestOf(dot.id); - const one = await api.createTask(dot.id, { description: "one" }); - const two = await api.createTask(dot.id, { description: "two" }); - // Two whole tasks, one after the other: 0.6 s on PGlite, 2.2 s on a PostgreSQL reached across WSL, past the - // default 5 s when that machine is loaded. The wait bounds liveness only. - await waitFor(async () => (await api.getTask(two.id)).status === "COMPLETED", "second task done", 15_000); - const call = { tool: "exec", permission: "exec.run", decision: "allow", ok: true, duration_ms: 5 } as const; - guest.emit("tool.called", { task_id: one.id, ...call, target: "ls" }); - guest.emit("tool.called", { ...call, target: "date" }); - guest.emit("browser.identity.created", { identity_id: "bi_shop", name: "shop" }); - await waitFor(async () => (await api.events(dot.id, { types: ["browser.identity.created"] })).length === 1, "events stored"); - - const types = (events: StoredEvent[]) => events.map((e) => e.type); - expect(types(await api.events(dot.id, { types: ["tool.called"] }))).toEqual(["tool.called", "tool.called"]); - expect(types(await api.events(dot.id, { types: ["tool.called", "browser.identity.created"] }))).toEqual(["tool.called", "tool.called", "browser.identity.created"]); - // A task's events: the host's own and the guest's, in id order; the chat's tool call belongs to no task. - const ofOne = await api.events(dot.id, { taskId: one.id }); - expect(types(ofOne)).toEqual(expect.arrayContaining(["task.created", "task.started", "task.completed", "tool.called"])); - expect(ofOne.every((e) => e.data.task_id === one.id)).toBe(true); - expect(ofOne.map((e) => e.id)).toEqual([...ofOne.map((e) => e.id)].sort((a, b) => a - b)); - expect(ofOne.filter((e) => e.type === "tool.called").map((e) => e.data.target)).toEqual(["ls"]); - expect((await api.events(dot.id, { taskId: two.id, types: ["tool.called"] })).length).toBe(0); - expect(types(await api.events(dot.id, { taskId: one.id, types: ["tool.called"] }))).toEqual(["tool.called"]); - expect((await api.events(dot.id, { taskId: one.id, limit: 1 })).length).toBe(1); - expect((await api.events(dot.id, { taskId: one.id, after: ofOne.at(-1)!.id })).length).toBe(0); - expect(await api.events(dot.id, { taskId: "task_nobody" })).toEqual([]); - - // The raw query: an empty types is no filter, a type nobody emits is refused, so is a repeated or empty parameter. - const get = async (query: string) => { - const response = await fetch(`${base}/api/dots/${dot.id}/events?${query}`, { headers: { authorization: `Bearer ${TOKEN}` } }); - return { status: response.status, body: (await response.json()) as { events?: StoredEvent[]; error?: string; message?: string } }; - }; - expect((await get("types=")).body.events!.length).toBeGreaterThan(5); - expect(await get("types=tool.called,tool.calls")).toMatchObject({ status: 400, body: { error: "invalid_request", message: "unknown event type: tool.calls" } }); - expect((await get("types=tool.called&types=browser.identity.created")).status).toBe(400); - // The Dot keeps its notes itself: a note written is no event, and asking for one is asking for a type nobody emits. - expect(await get("types=memory.written")).toMatchObject({ status: 400, body: { message: "unknown event type: memory.written" } }); - expect((await get("task_id=a&task_id=b")).status).toBe(400); - expect((await get("task_id=")).status).toBe(400); - - // Newest first, and a tool filter that narrows tool.called only: the newest browser call is found past the calls of other tools. - const browser = { ...call, tool: "browser_navigate", target: "x-1: https://example.com/" }; - guest.emit("tool.called", browser); - guest.emit("tool.called", { ...call, target: "pwd" }); - guest.emit("tool.called", { ...call, target: "whoami" }); - await waitFor(async () => (await api.events(dot.id, { types: ["tool.called"] })).length === 5, "more events stored"); - const newest = await api.events(dot.id, { types: ["tool.called"], order: "desc", limit: 2 }); - expect(newest.map((e) => e.data.target)).toEqual(["whoami", "pwd"]); - const browserCalls = await api.events(dot.id, { types: ["tool.called"], tools: ["browser_navigate", "browser_click"], order: "desc", limit: 1 }); - expect(browserCalls.map((e) => e.data.target)).toEqual(["x-1: https://example.com/"]); - expect((await api.events(dot.id, { types: ["tool.called", "browser.identity.created"], tools: ["browser_click"] })).map((e) => e.type)).toEqual(["browser.identity.created"]); - // `before` goes on, older, from the oldest event of a newest-first page, and only there. - const calls = await api.events(dot.id, { types: ["tool.called"], order: "desc", limit: 2 }); - expect((await api.events(dot.id, { types: ["tool.called"], order: "desc", limit: 2, before: calls.at(-1)!.id })).map((e) => e.data.target)).toEqual(["x-1: https://example.com/", "date"]); - expect(await get(`before=${calls.at(-1)!.id}`)).toMatchObject({ status: 400, body: { message: "before pages a list in order=desc" } }); - expect((await get("order=desc&before=-1")).status).toBe(400); - expect((await get("order=sideways")).status).toBe(400); - expect((await get("order=desc&order=asc")).status).toBe(400); - expect((await get("tools=a&tools=b")).status).toBe(400); - await expect(api.events("no-such-dot", { types: ["tool.called"] })).rejects.toMatchObject({ status: 404 }); - }); - - it("files: list and read what is under /home/dot, as the guest answers, and nothing else", async () => { - const dot = await readyDot("file-reader"); - const guest = driver.guestOf(dot.id); - const note = "# Fares\n\nCheapest day is Tuesday.\n"; - guest.putFile("/home/dot/memory/fares.md", note, new Date("2026-10-01T10:00:00Z")); - guest.putFile("/home/dot/memory/trips/rome.md", "Rome", new Date("2026-10-02T10:00:00Z")); - guest.putFile("/home/dot/workspace/shot.png", Uint8Array.from([0x89, 0x50, 0x4e, 0x47])); - guest.putFile("/home/dot/notes.txt", "hi"); - guest.putFile("/etc/passwd", "root:x:0:0"); - - expect(await api.listFiles(dot.id, "/home/dot/memory")).toEqual({ - path: "/home/dot/memory", - entries: [ - { name: "fares.md", type: "file", size: note.length, mtime: "2026-10-01T10:00:00.000Z" }, - { name: "trips", type: "dir", size: 0, mtime: "2026-10-02T10:00:00.000Z" }, - ], - }); - // Home by default, and relative or ~ paths mean the same as the absolute one. - const home = await api.listFiles(dot.id); - expect(home.path).toBe("/home/dot"); - expect(home.entries.map((e) => e.name)).toEqual(["memory", "notes.txt", "workspace"]); - expect(await api.listFiles(dot.id, "~/memory")).toEqual(await api.listFiles(dot.id, "memory/")); - - expect(new TextDecoder().decode(await api.readFile(dot.id, "/home/dot/memory/fares.md"))).toBe(note); - expect(new TextDecoder().decode(await api.readFile(dot.id, "memory/trips/rome.md"))).toBe("Rome"); - expect([...(await api.readFile(dot.id, "workspace/shot.png"))]).toEqual([0x89, 0x50, 0x4e, 0x47]); - - // Outside home, or with a .. segment: 400 before the guest is asked, whatever the guest holds. - const callsBefore = guest.calls.length; - for (const path of ["/etc/passwd", "/home", "/home/dotter/x", "../../etc/passwd", "memory/../../../etc/passwd", "/home/dot/memory/.."]) { - await expect(api.readFile(dot.id, path), path).rejects.toMatchObject({ status: 400, code: "invalid_path" }); - await expect(api.listFiles(dot.id, path), path).rejects.toMatchObject({ status: 400, code: "invalid_path" }); - } - expect(guest.calls.length).toBe(callsBefore); - const raw = (route: string, query: string) => fetch(`${base}/api/dots/${dot.id}/${route}${query}`, { headers: { authorization: `Bearer ${TOKEN}` } }); - expect((await raw("files", "")).status).toBe(400); - expect((await raw("files", "?path=")).status).toBe(400); - expect((await raw("files", "?path=a&path=b")).status).toBe(400); - expect((await raw("files/list", "?path=")).status).toBe(400); - expect((await fetch(`${base}/api/dots/${dot.id}/files?path=notes.txt`)).status).toBe(401); - - // The guest's own refusals pass through with their code. - await expect(api.readFile(dot.id, "memory/missing.md")).rejects.toMatchObject({ status: 404, code: "not_found" }); - await expect(api.listFiles(dot.id, "memory/missing")).rejects.toMatchObject({ status: 404, code: "not_found" }); - await expect(api.readFile(dot.id, "memory")).rejects.toMatchObject({ status: 400, code: "is_a_directory" }); - await expect(api.listFiles(dot.id, "notes.txt")).rejects.toMatchObject({ status: 400, code: "not_a_directory" }); - - // A link under home that leads out of it passes the path rule and is the guest's 403, with nothing of the target in the answer. - guest.putFile("/proc/4242/environ", "PROXY_PASSWORD=hunter2"); - guest.link("/home/dot/environ", "/proc/4242/environ"); - const leak = await raw("files", `?path=${encodeURIComponent("environ")}`); - expect(leak.status).toBe(403); - expect(await leak.json()).toMatchObject({ error: "outside_home" }); - await expect(api.readFile(dot.id, "environ")).rejects.toMatchObject({ status: 403, code: "outside_home" }); - }); - - it("files: a read says what it is, so the page never runs a Dot's file; a large one is refused", async () => { - const dot = await readyDot("file-types"); - const guest = driver.guestOf(dot.id); - guest.putFile("/home/dot/page.html", ""); - guest.putFile("/home/dot/pic.png", Uint8Array.from([1, 2, 3])); - guest.putFile("/home/dot/data.bin", Uint8Array.from([4, 5])); - guest.putFile("/home/dot/é \"q\".txt", "x"); - guest.putFile("/home/dot/big.bin", new Uint8Array(16 * 1024 * 1024 + 1)); - guest.putFile("/home/dot/limit.bin", new Uint8Array(16 * 1024 * 1024)); - const read = (path: string) => fetch(`${base}/api/dots/${dot.id}/files?path=${encodeURIComponent(path)}`, { headers: { authorization: `Bearer ${TOKEN}` } }); - - const html = await read("page.html"); - expect(html.headers.get("content-type")).toBe("text/plain; charset=utf-8"); - expect(html.headers.get("content-disposition")).toMatch(/^inline; filename="page\.html"/); - expect(html.headers.get("x-content-type-options")).toBe("nosniff"); - expect(html.headers.get("content-security-policy")).toBe("default-src 'none'; sandbox"); - expect(html.headers.get("cache-control")).toBe("no-store"); - expect(await html.text()).toBe(""); - expect((await read("pic.png")).headers.get("content-type")).toBe("image/png"); - const bin = await read("data.bin"); - expect(bin.headers.get("content-type")).toBe("application/octet-stream"); - expect(bin.headers.get("content-disposition")).toMatch(/^attachment; filename="data\.bin"/); - expect((await read("é \"q\".txt")).headers.get("content-disposition")).toBe( - "inline; filename=\"_ _q_.txt\"; filename*=UTF-8''%C3%A9%20%22q%22.txt", - ); - - expect((await read("limit.bin")).status).toBe(200); - const big = await read("big.bin"); - expect(big.status).toBe(413); - expect(await big.json()).toMatchObject({ error: "file_too_large" }); - await expect(api.readFile(dot.id, "big.bin")).rejects.toMatchObject({ status: 413, code: "file_too_large" }); - }); - - it("files need a running computer (409 computer_stopped) and a Dot that exists", async () => { - const dot = await readyDot("file-sleeper"); - await api.stopComputer(dot.id); - await scheduler.settle(); - await expect(api.listFiles(dot.id)).rejects.toMatchObject({ status: 409, code: "computer_stopped" }); - await expect(api.readFile(dot.id, "notes.txt")).rejects.toMatchObject({ status: 409, code: "computer_stopped" }); - await api.startComputer(dot.id); - await waitFor(async () => (await api.computer(dot.id)).ready, "started again"); - await expect(api.listFiles("no-such-dot")).rejects.toMatchObject({ status: 404 }); - await expect(api.readFile("no-such-dot", "a")).rejects.toMatchObject({ status: 404 }); - }); - - it("a tool table that follows the permissions, and no automations route: the jobs are the Dot's own", async () => { - const dot = await readyDot("automations"); - expect((await fetch(`${base}/api/dots/${dot.id}/automations`, { headers: { authorization: `Bearer ${TOKEN}` } })).status).toBe(404); - expect((await fetch(`${base}/api/dots/${dot.id}/tools`)).status).toBe(401); - - // Tools: the table, and what the model is offered follows the permissions the config pushed. - const { tools: none, mcp_servers } = await api.listTools(dot.id); - expect(mcp_servers).toEqual([]); - expect(none.map((t) => t.name)).toEqual([ - "exec", - "read_file", - "write_file", - "cron", - "browser_identity_list", - "browser_identity_create", - ]); - // The pushed config has the defaults filled in, none of them deny: every tool is offered. - expect(none.every((t) => t.offered)).toBe(true); - await api.updateDot(dot.id, `${yaml("automations")}permissions: - computer.exec: allow - files.read: ask - files.write: deny - automations: deny -`); - await waitFor(async () => (await api.listTools(dot.id)).tools.some((t) => !t.offered), "permissions pushed"); - expect(Object.fromEntries((await api.listTools(dot.id)).tools.map((t) => [t.name, t.offered]))).toEqual({ - exec: true, - read_file: true, - write_file: false, - cron: false, - browser_identity_list: true, - browser_identity_create: true, - }); - }); - - it("the tools need a running computer (409 computer_stopped) and a Dot that exists", async () => { - const dot = await readyDot("automation-sleeper"); - await api.stopComputer(dot.id); - await scheduler.settle(); - await expect(api.listTools(dot.id)).rejects.toMatchObject({ status: 409, code: "computer_stopped" }); - await expect(api.listSkills(dot.id)).rejects.toMatchObject({ status: 409, code: "computer_stopped" }); - await api.startComputer(dot.id); - await waitFor(async () => (await api.computer(dot.id)).ready, "started again"); - await expect(api.listTools("no-such-dot")).rejects.toMatchObject({ status: 404 }); - await expect(api.listSkills("no-such-dot")).rejects.toMatchObject({ status: 404 }); - }); - - it("the skills are the engine's, the built-in ones and the Dot's own, each with its whole file", async () => { - const dot = await readyDot("skilled"); - const guest = driver.guestOf(dot.id); - const own = { name: "shop-login", description: "Log in to the shop.", source: "dot" as const, path: "/home/dot/skills/shop-login/SKILL.md", content: "---\nname: shop-login\n---\n" }; - guest.skills = [...guest.skills, own]; - const skills = await api.listSkills(dot.id); - expect(skills.map((s) => [s.name, s.source])).toEqual([["invisible-playwright", "builtin"], ["shop-login", "dot"]]); - expect(skills[1]).toEqual(own); - expect((await fetch(`${base}/api/dots/${dot.id}/skills`)).status).toBe(401); - }); - - it("the stream filters by Dot and replays after an id without duplicates", async () => { - const a = await readyDot("stream-a"); - const b = await readyDot("stream-b"); - const history = await api.events(a.id); - const controller = new AbortController(); - const got: StoredEvent[] = []; - const reading = (async () => { - for await (const event of api.stream({ dotId: "stream-a", after: history[0]!.id, signal: controller.signal })) { - got.push(event); - if (event.type === "browser.identity.created") break; - } - })(); - driver.guestOf(b.id).emit("browser.identity.created", { identity_id: "bi_b", name: "b-only" }); - driver.guestOf(a.id).emit("browser.identity.created", { identity_id: "bi_a", name: "a-only" }); - await reading; - controller.abort(); - expect(got.every((e) => e.dot_id === a.id)).toBe(true); - expect(got.map((e) => e.id)).toEqual([...new Set(got.map((e) => e.id))]); - expect(got[0]?.id).toBe(history[1]?.id); - expect(got.at(-1)?.data).toMatchObject({ name: "a-only" }); - }); - - it("approval flow over HTTP", async () => { - const dot = await readyDot("asks-first"); - const guest = driver.guestOf(dot.id); - const original = guest.onInbound; - guest.onInbound = (event, g) => { - if (event.type === "task.created") g.requestApproval(event.data.task_id); - else return original(event, g); - }; - const task = await api.createTask(dot.id, { description: "delete an identity" }); - const pending = await waitFor( - async () => (await api.listApprovals("pending")).find((a) => a.dot_id === dot.id), - "approval pending", - ); - expect(pending.task_id).toBe(task.id); - await waitFor(async () => (await api.getTask(task.id)).status === "WAITING_APPROVAL", "task waiting"); - const rejected = await api.reject(pending.id, { note: "keep it" }); - expect(rejected).toMatchObject({ status: "rejected", note: "keep it" }); - await waitFor(async () => (await api.getTask(task.id)).status === "COMPLETED", "task resumed and completed"); - await expect(api.approve(pending.id)).rejects.toMatchObject({ status: 409, code: "already_resolved" }); - await expect(api.approve("apr_missing")).rejects.toMatchObject({ status: 404 }); - const raw = await fetch(`${base}/api/approvals?status=maybe`, { headers: { authorization: `Bearer ${TOKEN}` } }); - expect(raw.status).toBe(400); - }); - - it("approvals: several statuses at once, newest answer first, a limit that keeps the newest and a cursor", async () => { - const dot = await readyDot("asks-often"); - const guest = driver.guestOf(dot.id); - const ids = [guest.requestApproval(undefined), guest.requestApproval(undefined), guest.requestApproval(undefined)]; - await waitFor(async () => (await api.listApprovals("pending")).filter((a) => a.dot_id === dot.id).length === 3, "three pending"); - await api.reject(ids[1]!, {}); - await api.approve(ids[0]!, {}); - const mine = async (...args: Parameters) => (await api.listApprovals(...args)).filter((a) => a.dot_id === dot.id).map((a) => a.id); - // Answered, the last answered first (ids[0] after ids[1]); the one still pending is not among them. - expect(await mine(["approved", "rejected", "expired"], { order: "desc" })).toEqual([ids[0], ids[1]]); - expect(await mine(["pending", "rejected"])).toEqual([ids[1], ids[2]]); - const newestOne = await api.listApprovals(["approved", "rejected", "expired"], { order: "desc", limit: 1 }); - expect(newestOne.map((a) => a.id)).toEqual([ids[0]]); - expect((await api.listApprovals(["approved", "rejected", "expired"], { order: "desc", limit: 1, before: ids[0]! })).map((a) => a.id)).toEqual([ids[1]]); - - // Of one Dot, chosen in the database: by id or by name, and a Dot that does not exist has none. - const other = await readyDot("asks-rarely"); - const otherAsk = driver.guestOf(other.id).requestApproval(undefined); - await waitFor(async () => (await api.listApprovals("pending", { dot: other.id })).length === 1, "the other Dot's approval"); - expect((await api.listApprovals("pending", { dot: other.id })).map((a) => a.id)).toEqual([otherAsk]); - expect((await api.listApprovals("pending", { dot: "asks-rarely" })).map((a) => a.id)).toEqual([otherAsk]); - expect((await api.listApprovals(["approved", "rejected"], { dot: dot.id, order: "desc" })).map((a) => a.id)).toEqual([ids[0], ids[1]]); - expect(await api.listApprovals("pending", { dot: "dot_nobody" })).toEqual([]); - - const get = async (query: string) => (await fetch(`${base}/api/approvals?${query}`, { headers: { authorization: `Bearer ${TOKEN}` } })).status; - expect(await get("dot_id=")).toBe(400); - expect(await get(`dot_id=${dot.id}&dot_id=${other.id}`)).toBe(400); - expect(await get("status=approved,maybe")).toBe(400); - expect(await get("status=")).toBe(400); - expect(await get("status=approved&status=rejected")).toBe(400); - expect(await get("order=newest")).toBe(400); - expect(await get("limit=0")).toBe(400); - expect(await get("limit=501")).toBe(400); - expect(await get(`order=desc&before=${ids[0]}&limit=1`)).toBe(200); - expect(await get(`before=${ids[0]}`)).toBe(400); - expect(await get("order=desc&before=")).toBe(400); - }); - - it("approve with always over HTTP sets the permission in the Dot's config and pushes it", async () => { - const dot = await readyDot("always-http"); - const guest = driver.guestOf(dot.id); - const approvalId = guest.requestApproval(undefined); - await waitFor(async () => (await api.listApprovals("pending")).some((a) => a.id === approvalId), "approval pending"); - const post = (id: string, decision: string, body: unknown) => - fetch(`${base}/api/approvals/${id}/${decision}`, { - method: "POST", - headers: { authorization: `Bearer ${TOKEN}`, "content-type": "application/json" }, - body: JSON.stringify(body), - }); - expect((await post(approvalId, "approve", { always: "yes" })).status).toBe(400); - expect((await post(approvalId, "approve", { always: false })).status).toBe(400); - expect((await post(approvalId, "reject", { always: true })).status).toBe(400); - expect((await api.listApprovals("pending")).some((a) => a.id === approvalId)).toBe(true); - expect((await api.getDot(dot.id)).config.permissions).toEqual({}); - - expect(await api.approve(approvalId, { note: "yes", always: true })).toMatchObject({ status: "approved", note: "yes" }); - expect((await api.getDot(dot.id)).config.permissions["browser.identity.delete"]).toBe("allow"); - expect(guest.config?.permissions["browser.identity.delete"]).toBe("allow"); - const resolved = (await api.events(dot.id, { types: ["approval.resolved"] })).at(-1); - expect(resolved?.data).toEqual({ approval_id: approvalId, decision: "approve", note: "yes", always: true }); - await expect(api.approve(approvalId, { always: true })).rejects.toMatchObject({ status: 409, code: "already_resolved" }); - }); - - it("messages: delivered to a READY Dot and readable as a conversation", async () => { - const dot = await readyDot("talker"); - const sent = await api.sendMessage("talker", "what did you find?"); - expect(sent.delivery).toBe("delivered"); - await waitFor(async () => (await api.messages(dot.id)).length === 2, "reply"); - expect((await api.messages(dot.id)).map((m) => m.role)).toEqual(["user", "assistant"]); - await expect(api.sendMessage(dot.id, " ")).rejects.toMatchObject({ status: 400 }); - }); - - it("tasks: the newest first, a page at a time, and a cursor that goes on through the rest", async () => { - const dot = await readyDot("many-tasks"); - const other = await readyDot("other-tasks"); - const made: string[] = []; - for (const description of ["one", "two", "three", "four", "five"]) { - made.push((await api.createTask(dot.id, { description })).id); - await new Promise((resolve) => setTimeout(resolve, 5)); - } - const foreign = (await api.createTask(other.id, { description: "not mine" })).id; - const newestFirst = [...made].reverse(); - expect((await api.listTasks(dot.id)).map((t) => t.id)).toEqual(newestFirst); - const first = await api.listTasks(dot.id, { limit: 2 }); - expect(first.map((t) => t.id)).toEqual(newestFirst.slice(0, 2)); - const second = await api.listTasks(dot.id, { limit: 2, before: first.at(-1)!.id }); - expect(second.map((t) => t.id)).toEqual(newestFirst.slice(2, 4)); - expect((await api.listTasks(dot.id, { limit: 2, before: second.at(-1)!.id })).map((t) => t.id)).toEqual(newestFirst.slice(4)); - expect(await api.listTasks(dot.id, { before: made[0]! })).toEqual([]); - // Another Dot's task is no cursor here, and nothing of it is listed. - expect(await api.listTasks(dot.id, { before: foreign })).toEqual([]); - - const get = async (query: string) => (await fetch(`${base}/api/dots/${dot.id}/tasks?${query}`, { headers: { authorization: `Bearer ${TOKEN}` } })).status; - expect(await get("limit=0")).toBe(400); - expect(await get("limit=201")).toBe(400); - expect(await get("limit=x")).toBe(400); - expect(await get("before=")).toBe(400); - expect(await get("limit=200&before=nobody")).toBe(200); - }); - - it("every Dot in one read: the tasks that failed since a time, latest first, and every channel with its Dot", async () => { - const a = await readyDot("every-a"); - const b = await readyDot("every-b"); - const since = new Date(Date.now() - 1); - const failed = async (dotId: string, description: string) => { - const guest = driver.guestOf(dotId); - guest.onInbound = () => {}; - const task = await api.createTask(dotId, { description }); - await waitFor(async () => (await db.tasks.runs(task.id))[0]?.delivered_at, "delivered"); - guest.emit("task.failed", { task_id: task.id, error: "boom" }); - await waitFor(async () => (await api.getTask(task.id)).status === "FAILED", "failed"); - return task.id; - }; - const first = await failed(a.id, "first"); - const second = await failed(b.id, "second"); - const ours = (tasks: { id: string }[]) => tasks.map((t) => t.id).filter((id) => id === first || id === second); - expect(ours(await api.endedTasks("FAILED", since))).toEqual([second, first]); - expect(ours(await api.endedTasks("COMPLETED", since))).toEqual([]); - expect(await api.endedTasks("FAILED", new Date(Date.now() + 60_000))).toEqual([]); - const get = async (query: string) => (await fetch(`${base}/api/tasks?${query}`, { headers: { authorization: `Bearer ${TOKEN}` } })).status; - expect(await get("status=RUNNING&finished_after=2026-01-01T00:00:00Z")).toBe(400); - expect(await get("status=FAILED&finished_after=yesterday")).toBe(400); - expect(await get("status=FAILED")).toBe(400); - - await channels.add(a.id, "telegram"); - await channels.add(b.id, "telegram"); - const every = (await api.everyChannel()).filter((c) => c.dot_id === a.id || c.dot_id === b.id); - expect(every.map((c) => [c.dot_id, c.kind])).toEqual([ - [a.id, "telegram"], - [b.id, "telegram"], - ]); - expect(JSON.stringify(every)).not.toMatch(/token/i); - // The channels run until they are removed: this test stops what it started, so no later test or teardown inherits them. - await channels.remove(a.id, "telegram"); - await channels.remove(b.id, "telegram"); - expect((await api.everyChannel()).filter((c) => c.dot_id === a.id || c.dot_id === b.id)).toEqual([]); - }); - - it("messages: the newest page of a long conversation, and a cursor that goes back through the rest", async () => { - const dot = await readyDot("chatty"); - for (const [index, word] of ["one", "two", "three"].entries()) { - await api.sendMessage(dot.id, word); - await waitFor(async () => (await api.messages(dot.id)).length === 2 * (index + 1), `reply to ${word}`); - } - const all = await api.messages(dot.id); - expect(all.length).toBe(6); - // The default is the oldest first, so a limit keeps the oldest: only `order=desc` reaches the newest. - expect((await api.messages(dot.id, { limit: 2 })).map((m) => m.event_id)).toEqual(all.slice(0, 2).map((m) => m.event_id)); - const newest = await api.messages(dot.id, { order: "desc", limit: 2 }); - expect(newest.map((m) => m.event_id)).toEqual(all.slice(4).map((m) => m.event_id).reverse()); - const older = await api.messages(dot.id, { order: "desc", limit: 3, before: newest.at(-1)!.event_id }); - expect(older.map((m) => m.event_id)).toEqual(all.slice(1, 4).map((m) => m.event_id).reverse()); - expect(await api.messages(dot.id, { order: "desc", before: all[0]!.event_id })).toEqual([]); - - const get = async (query: string) => (await fetch(`${base}/api/dots/${dot.id}/messages?${query}`, { headers: { authorization: `Bearer ${TOKEN}` } })).status; - expect(await get("limit=0")).toBe(400); - expect(await get("limit=501")).toBe(400); - expect(await get("order=newest")).toBe(400); - expect(await get(`before=${all[0]!.event_id}`)).toBe(400); - expect(await get("order=desc&before=-1")).toBe(400); - expect(await get(`order=desc&before=${all[0]!.event_id}&limit=1`)).toBe(200); - }); - - it("messages: the HTTP route cannot claim a channel origin", async () => { - const dot = await readyDot("impostor"); - const origin = { channel: "telegram", binding_id: "chb_1", chat_id: "1", external_id: "2" }; - const raw = await fetch(`${base}/api/dots/${dot.id}/messages`, { - method: "POST", - headers: { authorization: `Bearer ${TOKEN}`, "content-type": "application/json" }, - body: JSON.stringify({ text: "I am Telegram", origin }), - }); - expect(raw.status).toBe(202); - await waitFor(async () => (await api.messages(dot.id)).length === 2, "reply"); - const [user] = await api.messages(dot.id); - expect(user).toMatchObject({ role: "user", text: "I am Telegram" }); - expect("origin" in user!).toBe(false); - }); - - it("browser identities and screenshots need a running computer (409 computer_stopped)", async () => { - const dot = await readyDot("browsing"); - const identity = await api.createIdentity(dot.id, { name: "Main account" }); - expect(identity.name).toBe("Main account"); - expect((await api.listIdentities(dot.id)).map((i) => i.id)).toEqual([identity.id]); - expect((await api.getIdentity(dot.id, identity.id)).id).toBe(identity.id); - await expect(api.getIdentity(dot.id, "nobody-abc123")).rejects.toMatchObject({ status: 404, code: "not_found" }); - const png = await api.screenshot(dot.id); - expect([...png.slice(0, 4)]).toEqual([0x89, 0x50, 0x4e, 0x47]); - await api.deleteIdentity(dot.id, identity.id); - - expect(await api.stopComputer(dot.id)).toEqual({ accepted: true }); - await scheduler.settle(); - expect((await api.computer(dot.id)).state).toBe("STOPPED"); - expect((await api.getDot(dot.id)).status).toBe("IDLE"); - await expect(api.listIdentities(dot.id)).rejects.toMatchObject({ status: 409, code: "computer_stopped" }); - await expect(api.createIdentity(dot.id, { name: "x" })).rejects.toMatchObject({ status: 409, code: "computer_stopped" }); - await expect(api.screenshot(dot.id)).rejects.toMatchObject({ status: 409, code: "computer_stopped" }); - - await api.startComputer(dot.id); - await waitFor(async () => (await api.computer(dot.id)).ready, "started again"); - }); - - it("an identity needs no proxy: a name alone, a null or a blank proxy make one with none, and a proxy is an explicit option", async () => { - const dot = await readyDot("no-proxy-needed"); - const create = async (body: Record) => { - const response = await fetch(`${base}/api/dots/${dot.id}/browser-identities`, { - method: "POST", - headers: { authorization: `Bearer ${TOKEN}`, "content-type": "application/json" }, - body: JSON.stringify(body), - }); - return { status: response.status, identity: (await response.json()) as Record }; - }; - for (const body of [{ name: "Plain" }, { name: "Null", proxy: null }, { name: "Blank", proxy: " " }]) { - const { status, identity } = await create(body); - expect(status, JSON.stringify(body)).toBe(201); - expect(identity, JSON.stringify(body)).toMatchObject({ hasProxy: false }); - } - const sdk = await api.createIdentity(dot.id, { name: "Through the SDK" }); - expect(sdk.hasProxy).toBe(false); - expect((await api.listIdentities(dot.id)).some((i) => i.hasProxy || "proxy" in i)).toBe(false); - - // The proxy is a secret: it is passed on as written, and the identity says only that it has one. - const own = await api.createIdentity(dot.id, { name: "Own exit", proxy: "socks5://user:hunter2@proxy.test:1080" }); - expect(own.hasProxy).toBe(true); - expect(own).not.toHaveProperty("proxy"); - const listed = JSON.stringify(await api.listIdentities(dot.id)); - expect(listed).not.toContain("hunter2"); - expect(listed).not.toContain("proxy.test"); - expect((await create({ name: "Number", proxy: 8080 })).status).toBe(400); - }); - - it("an MCP server's secrets are set and cleared through the API and the SDK, and no answer carries a value", async () => { - const dot = await api.createDot(`${yaml("mcp-api")}mcp_servers:\n web:\n url: https://web.example/mcp\n secrets: [Authorization]\n`); - await waitUntilSettledReady(scheduler, driver, dot.id, "mcp-api"); - const route = `${base}/api/dots/${dot.id}/mcp-secrets/web/Authorization`; - expect((await fetch(route, { method: "PUT", body: JSON.stringify({ value: "x" }), headers: { "content-type": "application/json" } })).status).toBe(401); - expect((await fetch(`${base}/api/dots/${dot.id}/mcp-secrets`)).status).toBe(401); - - const set = await api.setMcpSecret(dot.id, "web", "Authorization", "Bearer api-1"); - expect(set).toEqual({ dot_id: dot.id, secrets: [{ server: "web", name: "Authorization", set: true }] }); - expect(driver.guestOf(dot.id).mcpSecrets).toEqual({ web: { Authorization: "Bearer api-1" } }); - expect(JSON.stringify(await api.mcpSecrets(dot.id))).not.toContain("api-1"); - - // A PUT with no value is refused, not taken as a clear; the DELETE clears. - const empty = await fetch(route, { method: "PUT", headers: { authorization: `Bearer ${TOKEN}`, "content-type": "application/json" }, body: JSON.stringify({ value: null }) }); - expect(empty.status).toBe(400); - expect((await api.setMcpSecret(dot.id, "web", "Authorization", null)).secrets[0]!.set).toBe(false); - await expect(api.setMcpSecret(dot.id, "nope", "Authorization", "x")).rejects.toMatchObject({ status: 404 }); - }); - - it("an identity's frame is a JPEG of an open one, and close ends it, through the API and the SDK", async () => { - const dot = await readyDot("viewing"); - const identity = await api.createIdentity(dot.id, { name: "Main account" }); - await expect(api.getIdentityFrame(dot.id, identity.id)).rejects.toMatchObject({ status: 409, code: "not_open" }); - await expect(api.getIdentityFrame(dot.id, "nobody-abc123")).rejects.toMatchObject({ status: 404, code: "not_found" }); - - driver.guestOf(dot.id).launchIdentity(identity.id); - const jpeg = await api.getIdentityFrame(dot.id, identity.id); - expect([...jpeg]).toEqual([0xff, 0xd8, 0xff, 0xd9]); - const raw = await fetch(`${base}/api/dots/${dot.id}/browser-identities/${identity.id}/frame`, { - headers: { authorization: `Bearer ${TOKEN}` }, - }); - expect(raw.headers.get("content-type")).toBe("image/jpeg"); - expect(raw.headers.get("cache-control")).toBe("no-store"); - driver.guestOf(dot.id).identityBusy = true; - await expect(api.getIdentityFrame(dot.id, identity.id)).rejects.toMatchObject({ status: 503, code: "busy" }); - driver.guestOf(dot.id).identityBusy = false; - - await api.closeIdentity(dot.id, identity.id); - expect((await api.getIdentity(dot.id, identity.id)).status).toBe("available"); - await expect(api.getIdentityFrame(dot.id, identity.id)).rejects.toMatchObject({ status: 409, code: "not_open" }); - await expect(api.closeIdentity(dot.id, "nobody-abc123")).rejects.toMatchObject({ status: 404, code: "not_found" }); - - await api.stopComputer(dot.id); - await scheduler.settle(); - await expect(api.getIdentityFrame(dot.id, identity.id)).rejects.toMatchObject({ status: 409, code: "computer_stopped" }); - await expect(api.closeIdentity(dot.id, identity.id)).rejects.toMatchObject({ status: 409, code: "computer_stopped" }); - }); - - it("sleeps after idle_timeout (fake clock) and a new task wakes it", async () => { - const dot = await readyDot("napper", "5m"); - clock.advance(6 * 60_000); - expect(await scheduler.idleCheck()).toContain(dot.id); - await scheduler.settle(); - expect((await api.computer(dot.id)).state).toBe("STOPPED"); - const task = await api.createTask(dot.id, { description: "wake up" }); - await waitFor(async () => (await api.getTask(task.id)).status === "COMPLETED", "task after wake"); - expect((await api.computer(dot.id)).state).toBe("RUNNING"); - }); - - it("the secret route refuses a key that cannot travel in a header with 400, naming the rule and never the key", async () => { - const dot = await readyDot("key-check"); - expect(await api.setOpenRouterKey("sk-or-good", dot.id)).toEqual({ pushed: 1 }); - - for (const bad of ["sk-or-v1-SECRETHEAD\nSECRETTAIL", "sk-or-v1-SECRET HEAD", "sk-or-v1-SECRETüHEAD", "SECRET\u0000HEAD"]) { - const refused = await api.setOpenRouterKey(bad, dot.id).catch((error: unknown) => error); - expect(refused).toBeInstanceOf(ApiError); - expect(refused).toMatchObject({ status: 400, code: "invalid_request" }); - expect((refused as ApiError).message).toContain(OPENROUTER_KEY_RULE); - expect((refused as ApiError).message).not.toContain("SECRET"); - } - // Nothing was stored or pushed by the refusals. - expect(await db.secrets.get(dot.id, "openrouter_api_key")).toBe("sk-or-good"); - expect(driver.guestOf(dot.id).openrouterKey).toBe("sk-or-good"); - - // A pasted key with its ends trimmed is the key. - await api.setOpenRouterKey(" sk-or-padded\n", dot.id); - expect(driver.guestOf(dot.id).openrouterKey).toBe("sk-or-padded"); - }); - - it("PATCH updates the config; DELETE removes the Dot; the secret route stores the key", async () => { - const dot = await readyDot("to-patch"); - const updated = await api.updateDot(dot.id, `${yaml("to-patch")}instructions: short answers\n`); - expect(updated.config.instructions).toBe("short answers"); - expect(driver.guestOf(dot.id).config?.instructions).toBe("short answers"); - // A save from an old read is a 409 over the wire, and a malformed read is a 400. - const body = `${yaml("to-patch")}instructions: from a stale form\n`; - await expect(api.updateDot(dot.id, body, dot.config_version)).rejects.toMatchObject({ status: 409, code: "dot_changed" }); - expect((await api.updateDot(dot.id, body, updated.config_version)).config.instructions).toBe("from a stale form"); - await expect(api.updateDot(dot.id, body, 1.5)).rejects.toMatchObject({ status: 400, code: "invalid_request" }); - - expect(await api.setOpenRouterKey("sk-or-rotated", "to-patch")).toEqual({ pushed: 1 }); - expect(driver.guestOf(dot.id).openrouterKey).toBe("sk-or-rotated"); - await expect(api.setOpenRouterKey("")).rejects.toMatchObject({ status: 400 }); - - expect(await api.deleteDot("to-patch")).toEqual({ accepted: true }); - await scheduler.settle(); - await expect(api.getDot("to-patch")).rejects.toMatchObject({ status: 404 }); - expect((await api.events(dot.id)).at(-1)?.type).toBe("dot.deleted"); - }); -}); diff --git a/apps/api/test/channels.test.ts b/apps/api/test/channels.test.ts deleted file mode 100644 index 39a7cec0..00000000 --- a/apps/api/test/channels.test.ts +++ /dev/null @@ -1,234 +0,0 @@ -import type { AddressInfo } from "node:net"; -import { ChannelHub, TelegramChannelType } from "@invisible-dots/channels"; -import { FakeBotApi } from "@invisible-dots/channels/testing"; -import type { Database } from "@invisible-dots/database"; -import { createTestDatabase, testAdapters, type TestDatabase } from "@invisible-dots/database/testing"; -import { Scheduler } from "@invisible-dots/scheduler"; -import { FakeDriver, ManualClock, waitFor, waitUntilSettledReady } from "@invisible-dots/scheduler/testing"; -import { InvisibleDotsClient } from "@invisible-dots/sdk"; -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; -import { buildServer, type FastifyInstance } from "../src/index.js"; -import { hostFacts } from "./host-facts.js"; - -const API_TOKEN = "test-token-0123456789abcdef"; -const BOT_TOKEN = "123456:SECRET-TOKEN-VALUE"; -const OTHER_BOT_TOKEN = "654321:OTHER-SECRET-TOKEN-VALUE"; - -const yaml = (name: string) => `name: ${name}\nmodel:\n provider: openrouter\n id: test/model\n`; - -describe.each(testAdapters())("channel routes of the control-plane API (%s)", { timeout: 60_000 }, (kind) => { - let t: TestDatabase; - let db: Database; - let bots: FakeBotApi; - let app: FastifyInstance; - let scheduler: Scheduler; - let hub: ChannelHub; - let driver: FakeDriver; - let base: string; - let api: InvisibleDotsClient; - let seq = 0; - - beforeAll(async () => { - t = await createTestDatabase(kind); - db = t.db; - await db.secrets.put("global", "openrouter_api_key", "sk-or-test"); - bots = await FakeBotApi.start(); - driver = new FakeDriver(); - scheduler = new Scheduler({ - db, - driver, - clock: new ManualClock(), - lifecycle: { healthPollMs: 5, readyTimeoutMs: 3_000, pumpRetryMs: 10 }, - dispatcher: { retryDelayMs: 0 }, - }); - hub = new ChannelHub({ - db, - host: scheduler, - types: [new TelegramChannelType({ apiRoot: bots.apiRoot, pollSeconds: 30 })], - backoff: { initialMs: 1, maxMs: 5, jitter: 0 }, - }); - app = buildServer({ scheduler, channels: hub, doctor: async () => [], host: hostFacts(), token: API_TOKEN }); - await scheduler.start(); - await hub.start(); - await app.listen({ host: "127.0.0.1", port: 0 }); - base = `http://127.0.0.1:${(app.server.address() as AddressInfo).port}`; - api = new InvisibleDotsClient({ baseUrl: base, token: API_TOKEN }); - }, 60_000); - - beforeEach(() => { - bots.addBot(BOT_TOKEN, "dot_helper_bot"); - bots.addBot(OTHER_BOT_TOKEN, "second_bot"); - }); - - afterEach(async () => { - // A bot serves one Dot: what a test linked must not keep the next test from linking it. - for (const dot of await api.listDots()) { - for (const channel of await api.channels(dot.id)) await api.removeChannel(dot.id, channel.kind); - } - }); - - afterAll(async () => { - await app?.close(); - await hub?.close(); - await scheduler?.close(); - await bots?.close(); - await t?.drop(); - }); - - async function readyDot() { - const name = `chan-${++seq}`; - const dot = await api.createDot(yaml(name)); - await waitUntilSettledReady(scheduler, driver, dot.id, name); - return dot; - } - - /** A raw request, to see the status and the body exactly as sent. */ - async function raw(method: string, path: string, body?: unknown) { - const response = await fetch(`${base}${path}`, { - method, - headers: { authorization: `Bearer ${API_TOKEN}`, ...(body === undefined ? {} : { "content-type": "application/json" }) }, - body: body === undefined ? undefined : JSON.stringify(body), - }); - const text = await response.text(); - return { status: response.status, text, json: text ? (JSON.parse(text) as Record) : null }; - } - - it("lists nothing for a Dot without channels, and needs the API token like every route", async () => { - const dot = await readyDot(); - expect(await api.channels(dot.id)).toEqual([]); - for (const [method, path] of [ - ["GET", `/api/dots/${dot.id}/channels`], - ["PUT", `/api/dots/${dot.id}/channels/telegram`], - ["PATCH", `/api/dots/${dot.id}/channels/telegram`], - ["DELETE", `/api/dots/${dot.id}/channels/telegram`], - ["POST", `/api/dots/${dot.id}/channels/telegram/pairing`], - ["DELETE", `/api/dots/${dot.id}/channels/telegram/peers/10`], - ] as const) { - const response = await fetch(`${base}${path}`, { method }); - expect(response.status, `${method} ${path}`).toBe(401); - } - }); - - it("links a bot (201), shows its name, never returns the token, and replaces the token of a linked bot (200)", async () => { - const dot = await readyDot(); - const created = await raw("PUT", `/api/dots/${dot.name}/channels/telegram`, { token: ` ${BOT_TOKEN} ` }); - expect(created.status).toBe(201); - expect(created.json).toMatchObject({ kind: "telegram", enabled: true, account: "dot_helper_bot", peers: [], settings: { approvals: true, notify_tasks: true } }); - expect(created.text).not.toContain("SECRET-TOKEN"); - expect(await db.secrets.get(dot.id, "telegram_bot_token")).toBe(BOT_TOKEN); - - expect(JSON.stringify(await api.channels(dot.id))).not.toContain("SECRET-TOKEN"); - await waitFor(async () => (await api.channels(dot.name))[0]!.status === "connected", "connected"); - - const replaced = await raw("PUT", `/api/dots/${dot.id}/channels/telegram`, { token: OTHER_BOT_TOKEN }); - expect(replaced.status).toBe(200); - expect(replaced.json).toMatchObject({ kind: "telegram", account: "second_bot" }); - expect(replaced.text).not.toContain("SECRET-TOKEN"); - expect(await db.secrets.get(dot.id, "telegram_bot_token")).toBe(OTHER_BOT_TOKEN); - expect(await api.channels(dot.id)).toHaveLength(1); - }); - - it("pairs a person through the code, lists them, and revokes them", async () => { - const dot = await readyDot(); - await api.putTelegramChannel(dot.id, BOT_TOKEN); - await waitFor(() => bots.polling(BOT_TOKEN), "polling"); - const pairing = await api.pairChannel(dot.id, "telegram"); - expect(pairing.code).toMatch(/^[A-Z2-9]{8}$/); - expect(pairing.deep_link).toBe(`https://t.me/dot_helper_bot?start=${pairing.code}`); - expect(new Date(pairing.expires_at).getTime()).toBeGreaterThan(Date.now()); - expect((await raw("POST", `/api/dots/${dot.id}/channels/telegram/pairing`)).status).toBe(201); - - bots.say(BOT_TOKEN, `/start ${pairing.code}`, { id: 10, first_name: "Ann" }); - await waitFor(async () => (await api.channels(dot.id))[0]!.peers.length === 1, "the person to be paired"); - expect((await api.channels(dot.id))[0]!.peers[0]).toMatchObject({ peer_id: "10", role: "owner", label: "Ann" }); - - expect((await raw("DELETE", `/api/dots/${dot.id}/channels/telegram/peers/10`)).status).toBe(204); - expect((await api.channels(dot.id))[0]!.peers).toEqual([]); - await expect(api.removeChannelPeer(dot.id, "telegram", "10")).rejects.toMatchObject({ status: 404 }); - }); - - it("changes settings and pauses and resumes the channel, and refuses what is not a setting", async () => { - const dot = await readyDot(); - await api.putTelegramChannel(dot.id, BOT_TOKEN); - expect((await api.patchChannel(dot.id, "telegram", { settings: { notify_tasks: false } })).settings).toEqual({ approvals: true, notify_tasks: false, show_arguments: true }); - expect((await api.patchChannel(dot.id, "telegram", { settings: { show_arguments: false } })).settings).toEqual({ approvals: true, notify_tasks: false, show_arguments: false }); - const paused = await api.patchChannel(dot.id, "telegram", { enabled: false }); - expect(paused).toMatchObject({ enabled: false, settings: { approvals: true, notify_tasks: false } }); - await waitFor(() => !bots.polling(BOT_TOKEN), "polling to stop"); - const both = await api.patchChannel(dot.id, "telegram", { enabled: true, settings: { approvals: false } }); - expect(both).toMatchObject({ enabled: true, settings: { approvals: false, notify_tasks: false } }); - await waitFor(() => bots.polling(BOT_TOKEN), "polling again"); - - for (const body of [{}, { enabled: "yes" }, { settings: { colour: true } }, { settings: { approvals: "no" } }, { settings: [] }]) { - const refused = await raw("PATCH", `/api/dots/${dot.id}/channels/telegram`, body); - expect(refused.status, JSON.stringify(body)).toBe(400); - expect(refused.json).toMatchObject({ error: "invalid_request" }); - } - expect((await api.channels(dot.id))[0]!.settings).toEqual({ approvals: false, notify_tasks: false, show_arguments: false }); - }); - - it("unlinks: the channel stops, its token and people are deleted, and it is gone for a second try", async () => { - const dot = await readyDot(); - await api.putTelegramChannel(dot.id, BOT_TOKEN); - await waitFor(() => bots.polling(BOT_TOKEN), "polling"); - expect((await raw("DELETE", `/api/dots/${dot.id}/channels/telegram`)).status).toBe(204); - await waitFor(() => !bots.polling(BOT_TOKEN), "polling to stop"); - expect(await db.secrets.get(dot.id, "telegram_bot_token")).toBeNull(); - expect(await api.channels(dot.id)).toEqual([]); - await expect(api.removeChannel(dot.id, "telegram")).rejects.toMatchObject({ status: 404, code: "not_found" }); - await expect(api.patchChannel(dot.id, "telegram", { enabled: true })).rejects.toMatchObject({ status: 404 }); - await expect(api.pairChannel(dot.id, "telegram")).rejects.toMatchObject({ status: 404 }); - }); - - it("refuses with a status and a message that never carry the token: no token, a wrong one, a malformed one, an unreachable Telegram, a bot in use", async () => { - const dot = await readyDot(); - const other = await readyDot(); - const url = `/api/dots/${dot.id}/channels/telegram`; - const answers = [ - await raw("PUT", url, {}), - await raw("PUT", url, { token: 42 }), - await raw("PUT", url, { token: " " }), - await raw("PUT", url, { token: "999:UNKNOWN-TOKEN-VALUE" }), - await raw("PUT", url, { token: "bad token with spaces" }), - ]; - expect(answers.map((a) => a.status)).toEqual([400, 400, 400, 400, 400]); - expect(answers.map((a) => a.json?.error)).toEqual(["invalid_request", "invalid_request", "invalid_request", "invalid_credentials", "invalid_credentials"]); - expect(answers[3]!.json?.message).toMatch(/refused the bot token/); - - bots.addBot("777:UNREACHABLE-TOKEN-VALUE", "third_bot"); - bots.failNext("777:UNREACHABLE-TOKEN-VALUE", "getMe", "drop"); - const unreachable = await raw("PUT", url, { token: "777:UNREACHABLE-TOKEN-VALUE" }); - expect(unreachable.status).toBe(502); - expect(unreachable.json?.error).toBe("channel_unreachable"); - expect(unreachable.text).not.toContain("UNREACHABLE-TOKEN-VALUE"); - expect(await api.channels(dot.id)).toEqual([]); - expect(await db.secrets.get(dot.id, "telegram_bot_token")).toBeNull(); - - await api.putTelegramChannel(dot.id, BOT_TOKEN); - const inUse = await raw("PUT", `/api/dots/${other.id}/channels/telegram`, { token: BOT_TOKEN }); - expect(inUse.status).toBe(409); - expect(inUse.json?.error).toBe("account_in_use"); - expect(inUse.text).not.toContain("SECRET-TOKEN"); - expect(await api.channels(other.id)).toEqual([]); - - for (const answer of [...answers, unreachable, inUse]) expect(answer.text).not.toContain(BOT_TOKEN.split(":")[1]); - }); - - it("answers 404 for a Dot that does not exist and 400 for a channel kind that does not", async () => { - const dot = await readyDot(); - await expect(api.channels("no-such-dot")).rejects.toMatchObject({ status: 404 }); - await expect(api.putTelegramChannel("no-such-dot", BOT_TOKEN)).rejects.toMatchObject({ status: 404 }); - for (const [method, path] of [ - ["PATCH", `/api/dots/${dot.id}/channels/carrier-pigeon`], - ["DELETE", `/api/dots/${dot.id}/channels/carrier-pigeon`], - ["POST", `/api/dots/${dot.id}/channels/carrier-pigeon/pairing`], - ["DELETE", `/api/dots/${dot.id}/channels/carrier-pigeon/peers/1`], - ] as const) { - const answer = await raw(method, path, method === "PATCH" ? { enabled: true } : undefined); - expect(answer.status, `${method} ${path}`).toBe(400); - expect(answer.json?.message).toMatch(/the channels are telegram, whatsapp/); - } - // A channel this server has no adapter for yet answers like one that is not linked. - await expect(api.pairChannel(dot.id, "whatsapp")).rejects.toMatchObject({ status: 404 }); - }); -}); diff --git a/apps/api/test/config.test.ts b/apps/api/test/config.test.ts deleted file mode 100644 index 9154ec57..00000000 --- a/apps/api/test/config.test.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { hostPaths, type HostPaths } from "@invisible-dots/shared"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { - acquireServerLock, - loadOrCreateApiToken, - loadOrGenerateMasterKey, - parseListen, - saveMasterKey, -} from "../src/index.js"; - -let dir: string; -let paths: HostPaths; - -beforeEach(async () => { - dir = await mkdtemp(join(tmpdir(), "idots-config-")); - paths = hostPaths({ INVISIBLE_DOTS_HOME: dir }); -}); - -afterEach(async () => { - await rm(dir, { recursive: true, force: true }); -}); - -describe("server settings", () => { - it("parses the listen address", () => { - expect(parseListen()).toEqual({ host: "127.0.0.1", port: 8787 }); - expect(parseListen("0.0.0.0:9000")).toEqual({ host: "0.0.0.0", port: 9000 }); - expect(parseListen("[::1]:8787")).toEqual({ host: "::1", port: 8787 }); - expect(parseListen("8080")).toEqual({ host: "127.0.0.1", port: 8080 }); - expect(() => parseListen("localhost:http")).toThrow(/INVISIBLE_DOTS_LISTEN/); - expect(parseListen("127.0.0.1:0")).toEqual({ host: "127.0.0.1", port: 0 }); - expect(() => parseListen("localhost:http")).toThrow('INVISIBLE_DOTS_LISTEN must look like "127.0.0.1:8787", got "localhost:http"'); - }); - - it("names the setting it was asked for, with its own example, and refuses port 0 only where the setting needs a fixed port", () => { - const web = { variable: "INVISIBLE_DOTS_WEB_LISTEN", example: "127.0.0.1:3000", fixedPort: true }; - expect(parseListen("127.0.0.1:3100", web)).toEqual({ host: "127.0.0.1", port: 3100 }); - expect(() => parseListen("localhost:http", web)).toThrow('INVISIBLE_DOTS_WEB_LISTEN must look like "127.0.0.1:3000", got "localhost:http"'); - expect(() => parseListen("70000", web)).toThrow(/must look like/); - expect(() => parseListen("0", web)).toThrow('INVISIBLE_DOTS_WEB_LISTEN needs a fixed port, got "0"'); - expect(() => parseListen("127.0.0.1:0", web)).toThrow(/needs a fixed port/); - }); - - it("creates the API token once and reads it back; the environment wins; short tokens are refused", async () => { - const created = await loadOrCreateApiToken(paths, {}); - expect(created).toMatchObject({ created: true, origin: paths.apiTokenPath }); - expect(created.value).toMatch(/^[0-9a-f]{64}$/); - expect(await loadOrCreateApiToken(paths, {})).toEqual({ ...created, created: false }); - - expect(await loadOrCreateApiToken(paths, { INVISIBLE_DOTS_TOKEN: "x".repeat(32) })).toMatchObject({ - value: "x".repeat(32), - origin: "INVISIBLE_DOTS_TOKEN", - }); - await expect(loadOrCreateApiToken(paths, { INVISIBLE_DOTS_TOKEN: "short" })).rejects.toThrow(/shorter/); - await writeFile(paths.apiTokenPath, "tiny\r\n"); - await expect(loadOrCreateApiToken(paths, {})).rejects.toThrow(/shorter/); - }); - - it("generates a master key without writing it; saves it on request; accepts raw or hex keys, refuses others", async () => { - const generated = await loadOrGenerateMasterKey(paths); - expect(generated.created).toBe(true); - expect(generated.value.length).toBe(32); - await expect(readFile(paths.masterKeyPath)).rejects.toMatchObject({ code: "ENOENT" }); - - await saveMasterKey(paths, generated.value); - expect(await loadOrGenerateMasterKey(paths)).toEqual({ value: generated.value, origin: paths.masterKeyPath, created: false }); - - await writeFile(paths.masterKeyPath, `${"ab".repeat(32)}\n`); - expect((await loadOrGenerateMasterKey(paths)).value).toEqual(Buffer.from("ab".repeat(32), "hex")); - await writeFile(paths.masterKeyPath, "not a key"); - await expect(loadOrGenerateMasterKey(paths)).rejects.toThrow(/must hold 32 bytes/); - }); -}); - -describe("server lock", () => { - it("is exclusive while held, released on release, and taken over from a dead pid", async () => { - const lock = await acquireServerLock(paths); - expect(JSON.parse(await readFile(paths.serverLockPath, "utf8"))).toMatchObject({ pid: process.pid }); - await expect(acquireServerLock(paths)).rejects.toThrow(/already holds/); - await lock.release(); - await expect(readFile(paths.serverLockPath)).rejects.toMatchObject({ code: "ENOENT" }); - - await writeFile(paths.serverLockPath, "4242\n"); - await expect(acquireServerLock(paths, () => "ours")).rejects.toThrow(/pid 4242.*remove .*server\.lock/); - const taken = await acquireServerLock(paths, () => "gone"); - expect(JSON.parse(await readFile(paths.serverLockPath, "utf8"))).toMatchObject({ pid: process.pid }); - await taken.release(); - }); - - it("release leaves a lock that another server took over in place", async () => { - const lock = await acquireServerLock(paths); - await writeFile(paths.serverLockPath, "4242\n"); - await lock.release(); - expect(await readFile(paths.serverLockPath, "utf8")).toBe("4242\n"); - }); -}); diff --git a/apps/api/test/doctor.test.ts b/apps/api/test/doctor.test.ts deleted file mode 100644 index dba4f037..00000000 --- a/apps/api/test/doctor.test.ts +++ /dev/null @@ -1,78 +0,0 @@ -/** The doctor's image rows: the verdict on an image is kept for as long as the image and its manifest are the same files. */ -import { mkdir, mkdtemp, rm, utimes, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import type { ImageCheck } from "@invisible-dots/image-builder"; -import { hostPaths } from "@invisible-dots/shared"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { createImageChecks } from "../src/index.js"; - -let home: string; - -beforeEach(async () => { - home = await mkdtemp(join(tmpdir(), "idots-doctor-")); -}); - -afterEach(async () => { - await rm(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 }); -}); - -/** Both images in the home's images directory, and a `verify` that counts the images it is asked about. */ -async function setup(answer: ImageCheck = { ok: true, manifest: {} as never }) { - const paths = hostPaths({ INVISIBLE_DOTS_HOME: home }); - await mkdir(paths.imagesDir, { recursive: true }); - const golden = join(paths.imagesDir, "golden-1.qcow2"); - const runtime = join(paths.imagesDir, "runtime-1.iso"); - for (const file of [golden, runtime]) await writeFile(file, "image"); - await writeFile(join(paths.imagesDir, "golden-1.json"), "{}"); - await writeFile(join(paths.imagesDir, "runtime-1.json"), "{}"); - const verified: string[] = []; - const checks = createImageChecks(paths, async (image) => { - verified.push(image); - return answer; - }); - return { checks, verified, golden, runtime, paths }; -} - -describe("the image rows", () => { - it("verify each image once while its files stay the same", async () => { - const { checks, verified, golden, runtime } = await setup(); - const first = await checks(); - expect(first.map((row) => [row.id, row.status])).toEqual([["golden-image", "ok"], ["runtime-image", "ok"]]); - expect(await checks()).toEqual(first); - expect(await checks()).toEqual(first); - expect(verified).toEqual([golden, runtime]); - }); - - it("verify again when the image changes, and when its manifest does", async () => { - const { checks, verified, golden, runtime, paths } = await setup(); - await checks(); - await writeFile(golden, "a different image"); - await checks(); - expect(verified).toEqual([golden, runtime, golden]); - const manifest = join(paths.imagesDir, "runtime-1.json"); - await utimes(manifest, new Date(), new Date(Date.now() + 60_000)); - await checks(); - expect(verified).toEqual([golden, runtime, golden, runtime]); - }); - - it("keep a failed verdict too, with the command that fixes it", async () => { - const { checks, verified } = await setup({ ok: false, reason: "it hashes to the wrong digest", fix: "delete it and run invisible-dots image build" }); - const rows = await checks(); - expect(rows[0]).toMatchObject({ id: "golden-image", status: "failed", detail: "it hashes to the wrong digest", fix: "delete it and run invisible-dots image build" }); - await checks(); - expect(verified).toHaveLength(2); - }); - - it("answer missing without hashing anything when there is no image", async () => { - const paths = hostPaths({ INVISIBLE_DOTS_HOME: home }); - await mkdir(paths.imagesDir, { recursive: true }); - const verified: string[] = []; - const rows = await createImageChecks(paths, async (image) => { - verified.push(image); - return { ok: false, reason: "unreachable", fix: "" }; - })(); - expect(rows.map((row) => row.status)).toEqual(["missing", "missing"]); - expect(verified).toEqual([]); - }); -}); diff --git a/apps/api/test/file-types.test.ts b/apps/api/test/file-types.test.ts deleted file mode 100644 index d1cb6eef..00000000 --- a/apps/api/test/file-types.test.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { serveFile } from "../src/file-types.js"; - -describe("serveFile: what a browser may be told a Dot's file is", () => { - it("serves images as images and text as plain text, inline", () => { - expect(serveFile("/home/dot/a.png")).toMatchObject({ contentType: "image/png" }); - expect(serveFile("/home/dot/a.JPG")).toMatchObject({ contentType: "image/jpeg" }); - expect(serveFile("/home/dot/a.jpeg")).toMatchObject({ contentType: "image/jpeg" }); - expect(serveFile("/home/dot/a.gif")).toMatchObject({ contentType: "image/gif" }); - expect(serveFile("/home/dot/a.webp")).toMatchObject({ contentType: "image/webp" }); - for (const name of ["a.md", "a.txt", "a.json", "a.py", "a.CSV", "a.yaml", "memory/trips/a.log"]) { - expect(serveFile(`/home/dot/${name}`), name).toMatchObject({ contentType: "text/plain; charset=utf-8" }); - expect(serveFile(`/home/dot/${name}`).disposition, name).toMatch(/^inline;/); - } - }); - - it("never serves markup or script as what it is: html, svg, xml and javascript are text", () => { - for (const name of ["a.html", "a.htm", "a.svg", "a.xml", "a.js", "a.mjs"]) { - expect(serveFile(`/home/dot/${name}`).contentType, name).toBe("text/plain; charset=utf-8"); - } - }); - - it("makes everything else a download", () => { - for (const name of ["a.bin", "a.pdf", "a.zip", "Makefile", ".bashrc", "a.", "archive.tar.gz", "a.png.exe"]) { - const served = serveFile(`/home/dot/${name}`); - expect(served.contentType, name).toBe("application/octet-stream"); - expect(served.disposition, name).toMatch(/^attachment;/); - } - }); - - it("names the file in both forms, with nothing in the plain one that could end the header value", () => { - expect(serveFile("/home/dot/memory/fares.md").disposition).toBe(`inline; filename="fares.md"; filename*=UTF-8''fares.md`); - expect(serveFile('/home/dot/é "q"\\100%.txt').disposition).toBe( - `inline; filename="_ _q__100_.txt"; filename*=UTF-8''%C3%A9%20%22q%22%5C100%25.txt`, - ); - expect(serveFile("/home/dot/a\r\nx-evil: 1.txt").disposition).not.toMatch(/[\r\n]/); - }); -}); diff --git a/apps/api/test/guest-http.ts b/apps/api/test/guest-http.ts deleted file mode 100644 index bec85977..00000000 --- a/apps/api/test/guest-http.ts +++ /dev/null @@ -1,156 +0,0 @@ -/** - * dot-agentd's HTTP surface (architecture sections 5.1 to 5.3) in front of - * the scheduler's FakeGuest, so the real GuestClient of vm-manager talks to - * it over real TCP exactly as it talks to a guest through QEMU's port - * forward. Only the routes the control plane calls are served, including - * the unauthenticated proof route, computed the guest's way. - */ -import { createHmac } from "node:crypto"; -import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; -import type { AddressInfo, Socket } from "node:net"; -import type { FakeGuest } from "@invisible-dots/scheduler/testing"; -import { GUEST_PROOF_CONTEXT, type SecretsRequest } from "@invisible-dots/shared"; - -export interface FakeGuestServer { - port: number; - close(): Promise; -} - -async function readJson(request: IncomingMessage): Promise { - const chunks: Buffer[] = []; - for await (const chunk of request) chunks.push(chunk as Buffer); - const text = Buffer.concat(chunks).toString("utf8"); - return text ? JSON.parse(text) : undefined; -} - -function send(response: ServerResponse, status: number, body?: unknown): void { - if (body === undefined) { - response.writeHead(status).end(); - return; - } - response.writeHead(status, { "content-type": "application/json" }).end(JSON.stringify(body)); -} - -const IDENTITIES = "/v1/agent/browser-identities"; - -/** - * Serve `guest()` (looked up per request: a test may swap the guest) on - * `port` of 127.0.0.1 (0 picks one). Every request is appended to - * `requests` as "METHOD /path". `onPowerOff` is what the guest's poweroff - * ends in: the test's QEMU exits. - */ -export async function serveFakeGuest( - guest: () => FakeGuest | undefined, - options: { port?: number; requests?: string[]; onPowerOff?: () => void } = {}, -): Promise { - const requests = options.requests ?? []; - const sockets = new Set(); - const server: Server = createServer(async (request, response) => { - const url = new URL(request.url ?? "/", "http://guest"); - requests.push(`${request.method} ${url.pathname}`); - const current = guest(); - // A powered-off VM: QEMU's forward accepts and drops, which the client sees as a reset. - if (!current?.running) { - request.socket.destroy(); - return; - } - if (request.method === "GET" && url.pathname === "/v1/proof") { - const nonce = url.searchParams.get("nonce") ?? ""; - return send(response, 200, { proof: createHmac("sha256", current.token).update(`${GUEST_PROOF_CONTEXT}${nonce}`).digest("hex") }); - } - if (request.headers.authorization !== `Bearer ${current.token}`) { - send(response, 401, { error: "unauthorized", message: "bad token" }); - return; - } - try { - const route = `${request.method} ${url.pathname}`; - if (url.pathname.startsWith(`${IDENTITIES}/`)) { - const [rawId, action] = url.pathname.slice(IDENTITIES.length + 1).split("/"); - const id = decodeURIComponent(rawId!); - if (action === "frame" && request.method === "GET") { - response.writeHead(200, { "content-type": "image/jpeg" }).end(Buffer.from(await current.getBrowserIdentityFrame(id))); - return; - } - if (action === "close" && request.method === "POST") { - await current.closeBrowserIdentity(id); - return send(response, 204); - } - if (action === undefined && request.method === "GET") return send(response, 200, await current.getBrowserIdentity(id)); - if (action === undefined && request.method === "DELETE") { - await current.deleteBrowserIdentity(id); - return send(response, 204); - } - } - switch (route) { - case "GET /v1/health": - return send(response, 200, await current.health()); - case "GET /v1/system": - return send(response, 200, await current.system()); - case "POST /v1/system/poweroff": - send(response, 202, { status: "powering_off" }); - options.onPowerOff?.(); - return; - case "GET /v1/screenshot": - response.writeHead(200, { "content-type": "image/png" }).end(Buffer.from(await current.screenshot())); - return; - case "POST /v1/agent/secrets": - await current.pushSecrets((await readJson(request)) as SecretsRequest); - return send(response, 204); - case "PUT /v1/agent/config": - await current.putConfig((await readJson(request)) as never); - return send(response, 204); - case "POST /v1/agent/events": - return send(response, 202, await current.postEvent((await readJson(request)) as never)); - case "GET /v1/agent/state": - return send(response, 200, await current.state()); - case "POST /v1/agent/prepare-sleep": - await current.prepareSleep(); - return send(response, 204); - case `GET ${IDENTITIES}`: - return send(response, 200, await current.listBrowserIdentities()); - case `POST ${IDENTITIES}`: - return send(response, 201, await current.createBrowserIdentity((await readJson(request)) as never)); - case "GET /v1/agent/events/stream": { - const controller = new AbortController(); - request.on("close", () => controller.abort()); - response.writeHead(200, { "content-type": "text/event-stream" }); - response.write(": open\n\n"); - try { - for await (const event of current.events({ after: Number(url.searchParams.get("after") ?? 0), signal: controller.signal })) { - response.write(`id: ${event.seq}\ndata: ${JSON.stringify(event)}\n\n`); - } - } catch { - // A disconnect inside the fake ends the stream; the client reconnects. - } - response.end(); - return; - } - default: - return send(response, 404, { error: "not_found", message: route }); - } - } catch (error) { - const status = (error as { status?: number }).status; - if (!status) { - request.socket.destroy(); - return; - } - send(response, status, { error: (error as { code?: string }).code ?? "error", message: (error as Error).message }); - } - }); - server.on("connection", (socket) => { - sockets.add(socket); - socket.on("close", () => sockets.delete(socket)); - }); - await new Promise((resolve, reject) => { - server.once("error", reject); - server.listen(options.port ?? 0, "127.0.0.1", () => resolve()); - }); - return { - port: (server.address() as AddressInfo).port, - close: () => - new Promise((resolve) => { - for (const socket of sockets) socket.destroy(); - server.close(() => resolve()); - }), - }; -} diff --git a/apps/api/test/host-facts.ts b/apps/api/test/host-facts.ts deleted file mode 100644 index a757ed5a..00000000 --- a/apps/api/test/host-facts.ts +++ /dev/null @@ -1,8 +0,0 @@ -import type { HostFacts } from "@invisible-dots/sdk/types"; - -/** Where a fake host keeps its state, as `GET /api/health` is to say it. */ -export const hostFacts = (kind: HostFacts["database_kind"] = "pglite"): HostFacts => ({ - database_kind: kind, - data_dir: "/srv/dots-home", - logs_dir: "/srv/dots-home/logs", -}); diff --git a/apps/api/test/start.test.ts b/apps/api/test/start.test.ts deleted file mode 100644 index 307d07c1..00000000 --- a/apps/api/test/start.test.ts +++ /dev/null @@ -1,303 +0,0 @@ -/** - * The control plane started the way `invisible-dots server` starts it, on a - * real INVISIBLE_DOTS_HOME with the embedded PGlite; only the VM layer is a - * fake. Each test gets an empty home in the system's temp directory. - */ -import { EventEmitter } from "node:events"; -import { existsSync } from "node:fs"; -import { mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { FakeChannelType } from "@invisible-dots/channels/testing"; -import { FakeDriver, waitFor } from "@invisible-dots/scheduler/testing"; -import { InvisibleDotsClient } from "@invisible-dots/sdk"; -import { newId, parseDotConfig, permissionBitsEnforced } from "@invisible-dots/shared"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { healthyDoctor } from "../../vm-manager/test/doctor-fakes.js"; -import { startServer, STOP_SIGNALS, untilStopSignal, type RunningServer, type StartServerOptions } from "../src/index.js"; - -const quiet = { debug() {}, info() {}, warn() {}, error() {} }; - -let home: string; -const running: RunningServer[] = []; - -beforeEach(async () => { - home = await mkdtemp(join(tmpdir(), "idots-home-")); -}); - -afterEach(async () => { - await Promise.all(running.splice(0).map((s) => s.close())); - await rm(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 }); -}); - -/** An environment with nothing but the home: no DATABASE_URL, so PGlite; no token variable. */ -function envFor(dir: string, extra: Record = {}): Record { - return { INVISIBLE_DOTS_HOME: dir, ...extra }; -} - -async function start(options: StartServerOptions = {}): Promise { - const server = await startServer({ - env: envFor(home), - listen: "127.0.0.1:0", - logger: quiet, - driver: new FakeDriver(), - scheduler: { dispatchIntervalMs: 60_000, idleCheckIntervalMs: 60_000 }, - ...options, - }); - running.push(server); - return server; -} - -describe("stopping on a signal", { timeout: 120_000 }, () => { - it("closes the database and releases the lock on every way a terminal or a service stops it", async () => { - // Closing the console window (SIGHUP, also on Windows) and Ctrl+Break (SIGBREAK) included. - expect([...STOP_SIGNALS]).toEqual(["SIGINT", "SIGTERM", "SIGHUP", "SIGBREAK"]); - for (const signal of STOP_SIGNALS) { - const server = await startServer({ - env: envFor(home), - listen: "127.0.0.1:0", - logger: quiet, - driver: new FakeDriver(), - scheduler: { dispatchIntervalMs: 60_000, idleCheckIntervalMs: 60_000 }, - }); - const signals = new EventEmitter(); - const done = untilStopSignal(() => server.close(), quiet, signals); - expect(existsSync(join(home, "server.lock"))).toBe(true); - signals.emit(signal, signal); - await done; - expect(existsSync(join(home, "server.lock")), signal).toBe(false); - expect(signals.eventNames()).toEqual([]); - } - }); -}); - -describe("stopping while a page is open", { timeout: 120_000 }, () => { - it("ends the live streams a browser holds and closes: a stop does not wait for the person to close the tab", async () => { - const server = await start(); - const controller = new AbortController(); - const headers = { authorization: `Bearer ${server.token}` }; - const stream = await fetch(`${server.url}/api/stream`, { headers, signal: controller.signal }); - expect(stream.status).toBe(200); - const reading = stream.body!.getReader().read(); - - const closed = server.close().then(() => "closed" as const); - const outcome = await Promise.race([closed, new Promise<"waiting">((resolve) => setTimeout(() => resolve("waiting"), 15_000))]); - controller.abort(); - expect(outcome).toBe("closed"); - // The stream the client held ended: its read answers (the end of the body) and does not hang. - await expect(Promise.race([reading.then(() => "ended"), new Promise((resolve) => setTimeout(() => resolve("hung"), 2_000))])).resolves.toBe("ended"); - running.splice(running.indexOf(server), 1); - }); -}); - -async function stopped(server: RunningServer): Promise { - running.splice(running.indexOf(server), 1); - await server.close(); -} - -// A first PGlite start runs initdb on disk, which takes seconds on a slow machine. -describe("startServer on an empty INVISIBLE_DOTS_HOME", { timeout: 120_000 }, () => { - it("creates api.token and master.key, opens PGlite in db/, and serves the API with that token", async () => { - const server = await start(); - expect(server.database).toBe("pglite"); - expect(server.created).toEqual({ apiToken: true, masterKey: true }); - - const tokenFile = await readFile(join(home, "config", "api.token"), "utf8"); - expect(tokenFile).toBe(`${server.token}\n`); - expect(server.token).toMatch(/^[0-9a-f]{64}$/); - const key = await readFile(join(home, "config", "master.key")); - expect(key.length).toBe(32); - expect((await readdir(join(home, "db"))).length).toBeGreaterThan(0); - if (permissionBitsEnforced()) { - expect((await stat(join(home, "config", "api.token"))).mode & 0o777).toBe(0o600); - expect((await stat(join(home, "config", "master.key"))).mode & 0o777).toBe(0o600); - expect((await stat(join(home, "config"))).mode & 0o777).toBe(0o700); - } - - const api = new InvisibleDotsClient({ baseUrl: server.url, token: server.token }); - expect(await api.health()).toMatchObject({ - status: "ok", - database: "ok", - openrouter_configured: false, - database_kind: "pglite", - data_dir: home, - logs_dir: join(home, "logs"), - }); - await api.setOpenRouterKey("sk-or-first"); - expect(await api.health()).toMatchObject({ openrouter_configured: true }); - const refused = await fetch(`${server.url}/api/health`, { headers: { authorization: "Bearer wrong-token-0123456789" } }); - expect(refused.status).toBe(401); - - await stopped(server); - await expect(stat(server.paths.serverLockPath)).rejects.toMatchObject({ code: "ENOENT" }); - }); - - it("serves the host report: the machine's rows, this home's images, and the OpenRouter row from its own Scheduler", async () => { - // The machine (QEMU, the accelerator, the disk) is faked; the images, the home and the key are this server's own. - const { findQemu, run, accelerator, acceleratorAccess, freeSpace } = healthyDoctor().deps; - const server = await start({ doctor: { findQemu, run, accelerator, acceleratorAccess, freeSpace } }); - const api = new InvisibleDotsClient({ baseUrl: server.url, token: server.token }); - - const before = await api.doctor(); - expect(before.checks.map((c) => c.id)).toEqual(["node", "qemu", "qemu-img", "accelerator", "accelerator-probe", "disk", "golden-image", "runtime-image", "openrouter"]); - expect(before.ok).toBe(false); - const row = (id: string) => before.checks.find((c) => c.id === id); - expect(row("qemu")?.status).toBe("ok"); - expect(row("golden-image")).toMatchObject({ status: "missing", detail: `none in ${server.paths.imagesDir}`, fix: "invisible-dots image build" }); - expect(row("runtime-image")?.status).toBe("missing"); - expect(row("openrouter")).toMatchObject({ status: "missing", detail: "no key stored", fix: "invisible-dots secret openrouter" }); - - await api.setOpenRouterKey("sk-or-first"); - expect((await api.doctor()).checks.find((c) => c.id === "openrouter")).toEqual({ id: "openrouter", label: "OpenRouter key", status: "ok", detail: "stored" }); - }); - - it("runs one host report at a time: requests that arrive while it runs share its answer", async () => { - // The accelerator probe is the slow row; the onboarding checklist asks again and again. - const base = healthyDoctor().deps; - let probes = 0; - let release: () => void = () => {}; - const gate = new Promise((resolve) => { - release = resolve; - }); - const acceleratorAccess = async () => { - probes += 1; - await gate; - return base.acceleratorAccess(); - }; - const server = await start({ doctor: { ...base, acceleratorAccess } }); - const api = new InvisibleDotsClient({ baseUrl: server.url, token: server.token }); - let arrived = 0; - server.app.server.on("request", (request: { url?: string }) => { - if (request.url === "/api/doctor") arrived += 1; - }); - - const together = Promise.all([api.doctor(), api.doctor(), api.doctor()]); - await waitFor(() => probes === 1 && arrived === 3, "all three requests at the server while the probe is held"); - // A request's hooks and handler run on promises once it has arrived: one turn of the event loop has them joined. - await new Promise((resolve) => setImmediate(resolve)); - release(); - const [first, second, third] = await together; - expect(probes).toBe(1); - expect(second).toEqual(first); - expect(third).toEqual(first); - - // Once it has answered, the next request runs it again. - await api.doctor(); - expect(probes).toBe(2); - }); - - it("a restart keeps the token, the key and the data", async () => { - const first = await start(); - await first.db.secrets.put("global", "openrouter_api_key", "sk-or-kept"); - const token = first.token; - await stopped(first); - - const second = await start(); - expect(second.created).toEqual({ apiToken: false, masterKey: false }); - expect(second.token).toBe(token); - expect(await second.db.secrets.openRouterKey("dot_none")).toBe("sk-or-kept"); - }); - - it("starts the channels stored in the database with the server and stops them when it closes", async () => { - const first = await start({ channelTypes: [new FakeChannelType()] }); - const dotId = newId("dot"); - await first.db.dots.insert({ - id: dotId, - config: parseDotConfig("name: channeled\nmodel:\n provider: openrouter\n id: test/model\n"), - status: "IDLE", - }); - await first.channels.add(dotId, "telegram"); - await stopped(first); - - const type = new FakeChannelType(); - const second = await start({ channelTypes: [type] }); - const channel = await waitFor(() => type.channels.at(-1)?.sink && type.channels.at(-1), "the stored channel to run"); - expect(await second.channels.list(dotId)).toHaveLength(1); - await stopped(second); - expect(channel.sink).toBeNull(); - }); - - it("refuses a second server on the same home while the first runs", async () => { - const first = await start(); - // In one process the second start is caught by the lock's own record of what it holds. - await expect(start()).rejects.toThrow(/already holds the invisible-dots server lock/); - const api = new InvisibleDotsClient({ baseUrl: first.url, token: first.token }); - expect(await api.health()).toMatchObject({ status: "ok" }); - }); - - it("takes over the lock of a server that died", async () => { - await writeFile(join(home, "server.lock"), "999999999\n"); - const server = await start(); - expect(JSON.parse(await readFile(server.paths.serverLockPath, "utf8"))).toMatchObject({ pid: process.pid }); - }); - - it("refuses to start when master.key is gone but the database holds encrypted values, and writes no new key", async () => { - const first = await start(); - await first.db.secrets.put("global", "openrouter_api_key", "sk-or-locked"); - await stopped(first); - await rm(join(home, "config", "master.key")); - - await expect(start()).rejects.toThrow(/master\.key/); - await expect(stat(join(home, "config", "master.key"))).rejects.toMatchObject({ code: "ENOENT" }); - // The failed start released everything it took: a later start can take the lock. - await expect(stat(join(home, "server.lock"))).rejects.toMatchObject({ code: "ENOENT" }); - }); - - it("INVISIBLE_DOTS_TOKEN replaces the token file and none is written", async () => { - const token = "env-token-0123456789abcdef"; - const server = await start({ env: envFor(home, { INVISIBLE_DOTS_TOKEN: token }) }); - expect(server.token).toBe(token); - expect(server.created.apiToken).toBe(false); - await expect(stat(join(home, "config", "api.token"))).rejects.toMatchObject({ code: "ENOENT" }); - }); - - it("a failed start releases the lock and the database", async () => { - await expect(start({ listen: "not-an-address:http" })).rejects.toThrow(/INVISIBLE_DOTS_LISTEN/); - await expect(start({ env: envFor(home, { INVISIBLE_DOTS_TOKEN: "short" }) })).rejects.toThrow(/shorter/); - await expect(stat(join(home, "server.lock"))).rejects.toMatchObject({ code: "ENOENT" }); - const server = await start(); - expect(server.database).toBe("pglite"); - }); - - it("with INVISIBLE_DOTS_WHATSAPP=1 and no WhatsApp client installed, starts anyway and says in the log how to install it", async () => { - const warnings: string[] = []; - const logger = { ...quiet, warn: (message: string) => void warnings.push(message) }; - const server = await start({ env: envFor(home, { INVISIBLE_DOTS_WHATSAPP: "1" }), logger, whatsappClientDir: join(home, "no-client") }); - expect(warnings.filter((m) => m.includes("WhatsApp client is not installed"))).toHaveLength(1); - expect(warnings.join("\n")).toContain("npm run whatsapp:install"); - expect(server.channels.kinds).toEqual(["telegram", "whatsapp"]); - const api = new InvisibleDotsClient({ baseUrl: server.url, token: server.token }); - expect(await api.health()).toMatchObject({ status: "ok" }); - }); - - it("with INVISIBLE_DOTS_WHATSAPP=1 and a client that is not the pinned release, starts anyway and says in the log which releases differ", async () => { - const client = join(home, "client"); - await mkdir(join(client, "node_modules", "baileys"), { recursive: true }); - await writeFile(join(client, "package.json"), JSON.stringify({ dependencies: { baileys: "1.2.4" } })); - await writeFile(join(client, "node_modules", "baileys", "package.json"), JSON.stringify({ name: "baileys", version: "1.2.3", main: "index.js" })); - await writeFile(join(client, "node_modules", "baileys", "index.js"), ""); - const warnings: string[] = []; - const logger = { ...quiet, warn: (message: string) => void warnings.push(message) }; - const server = await start({ env: envFor(home, { INVISIBLE_DOTS_WHATSAPP: "1" }), logger, whatsappClientDir: client }); - const line = warnings.filter((m) => m.includes("WhatsApp")); - expect(line).toHaveLength(1); - expect(line[0]).toContain("1.2.3"); - expect(line[0]).toContain("1.2.4"); - expect(line[0]).toContain("npm run whatsapp:install"); - expect(server.channels.kinds).toEqual(["telegram", "whatsapp"]); - }); - - it("says nothing of the WhatsApp client when WhatsApp is not asked for, or when the client is there", async () => { - const client = join(home, "client"); - await mkdir(join(client, "node_modules", "baileys"), { recursive: true }); - await writeFile(join(client, "package.json"), JSON.stringify({ dependencies: { baileys: "1.2.3" } })); - await writeFile(join(client, "node_modules", "baileys", "package.json"), JSON.stringify({ name: "baileys", version: "1.2.3", main: "index.js" })); - await writeFile(join(client, "node_modules", "baileys", "index.js"), ""); - const warnings: string[] = []; - const logger = { ...quiet, warn: (message: string) => void warnings.push(message) }; - const first = await start({ logger, whatsappClientDir: join(home, "no-client") }); - await stopped(first); - await start({ env: envFor(home, { INVISIBLE_DOTS_WHATSAPP: "1" }), logger, whatsappClientDir: client }); - expect(warnings.filter((m) => m.includes("WhatsApp"))).toEqual([]); - }); -}); diff --git a/apps/api/test/vm-driver.test.ts b/apps/api/test/vm-driver.test.ts deleted file mode 100644 index 7532acb7..00000000 --- a/apps/api/test/vm-driver.test.ts +++ /dev/null @@ -1,272 +0,0 @@ -/** - * VmManagerDriver over the real VmManager. QEMU is vm-manager's own fake at - * the process boundary (it parses the real argv); qemu-img is a fake runner; - * the guest is a FakeGuest behind real HTTP on the port VmManager picked and - * passed to the "QEMU", listening only while that QEMU runs, as QEMU's - * forward does; its poweroff stops the QEMU. The last test runs the whole - * Scheduler on top of it. - */ -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { createTestDatabase, type TestDatabase } from "@invisible-dots/database/testing"; -import { Scheduler } from "@invisible-dots/scheduler"; -import { FAKE_MAX_IDENTITIES, FakeGuest, waitFor } from "@invisible-dots/scheduler/testing"; -import { computerResources, hostPaths, type HostPaths, parseDotConfig, prefixedStderrLogger, silentLogger } from "@invisible-dots/shared"; -import { VmManager } from "@invisible-dots/vm-manager"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { FakeQemuHost, FakeRunner, type FakeVm } from "../../vm-manager/test/fakes.js"; -import { compareVersions, latestImage, VmManagerDriver } from "../src/index.js"; -import { serveFakeGuest, type FakeGuestServer } from "./guest-http.js"; - -const DOT = "dot_01k6h3w2ze8m4qv7r1xk9bntc5"; -const QEMU = { system: "/opt/qemu/bin/qemu-system-x86_64", img: "/opt/qemu/bin/qemu-img" }; -const TOKEN = "dot-token-0123456789"; - -/** - * The fake QEMU host whose forward is a fake guest server on the guest port - * of the argv, up from the spawn until the process ends: the guest boots and - * powers off together with its VM, with the token of its seed. - */ -class GuestQemuHost extends FakeQemuHost { - /** Guest requests of every VM, as "METHOD /path", in order. */ - readonly requests: string[] = []; - readonly #servers = new Map(); - - constructor( - private readonly guest: () => FakeGuest | undefined, - private readonly replaceGuest: (guest: FakeGuest) => void, - ) { - super(); - } - - protected override async openForward(vm: FakeVm): Promise { - if (this.guest()?.token !== vm.token) this.replaceGuest(new FakeGuest(vm.token)); - this.guest()?.boot(); - const server = await serveFakeGuest(() => this.guest(), { - port: vm.guestPort, - requests: this.requests, - onPowerOff: () => setTimeout(() => this.terminate(vm), 5), - }); - this.#servers.set(vm.pid, server); - } - - protected override closeForward(vm: FakeVm): void { - this.guest()?.powerOff(); - void this.#servers.get(vm.pid)?.close(); - this.#servers.delete(vm.pid); - } - - async closeAll(): Promise { - await Promise.all([...this.#servers.values()].map((s) => s.close())); - this.#servers.clear(); - } -} - -let root: string; -let paths: HostPaths; -let diskSize: number; -let runner: FakeRunner; -let guest: FakeGuest | undefined; -let host: GuestQemuHost; - -function manager(): VmManager { - return new VmManager({ - paths, - logger: silentLogger, - qemu: QEMU, - accelerator: "kvm", - runner, - processes: host, - pollIntervalMs: 1, - startSettleMs: 1, - shutdownTimeoutMs: 200, - killTimeoutMs: 100, - startTimeoutMs: 1_000, - }); -} - -function input(dotId = DOT, disk = "40gb") { - const config = parseDotConfig({ name: "drv", model: { provider: "openrouter", id: "m" }, computer: { disk } }); - return { dotId, token: TOKEN, resources: computerResources(config) }; -} - -beforeEach(async () => { - root = await mkdtemp(join(tmpdir(), "idots-drv-")); - paths = hostPaths({ INVISIBLE_DOTS_HOME: root }); - await mkdir(paths.imagesDir, { recursive: true }); - for (const name of ["golden-2026.9.0.qcow2", "golden-2026.10.0.qcow2", "runtime-1.2.iso", "runtime-1.10.iso", "noise.txt"]) { - await writeFile(join(paths.imagesDir, name), name); - } - diskSize = 0; - runner = new FakeRunner(async (_command, args) => { - if (args[0] === "create") { - diskSize = Number(args.at(-1)); - await writeFile(args.at(-2)!, "qcow2"); - } - if (args[0] === "info") return { stdout: JSON.stringify({ "virtual-size": diskSize, format: "qcow2" }), stderr: "" }; - if (args[0] === "resize") diskSize = Number(args.at(-1)); - return undefined; - }); - guest = new FakeGuest(TOKEN); - host = new GuestQemuHost( - () => guest, - (next) => { - guest = next; - }, - ); -}); - -afterEach(async () => { - guest?.powerOff(); - await host.closeAll(); - await rm(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); -}); - -describe("image selection", () => { - it("compares versions numerically", () => { - expect(compareVersions("2026.10.1", "2026.9.3")).toBeGreaterThan(0); - expect(compareVersions("1.2", "1.10")).toBeLessThan(0); - }); - - it("picks the newest golden image and runtime ISO, and names the command that builds one", async () => { - const driver = new VmManagerDriver(manager()); - expect(await driver.latestGolden()).toBe(join(paths.imagesDir, "golden-2026.10.0.qcow2")); - expect(await driver.latestRuntime()).toBe(join(paths.imagesDir, "runtime-1.10.iso")); - await expect(latestImage(join(root, "missing"), "golden")).rejects.toThrow(/cannot list/); - await rm(join(paths.imagesDir, "runtime-1.2.iso")); - await rm(join(paths.imagesDir, "runtime-1.10.iso")); - await expect(driver.latestRuntime()).rejects.toThrow(/no runtime ISO .*invisible-dots image build/); - }); -}); - -describe("VmManagerDriver over VmManager", () => { - it("create, start, health, stop, start again on a grown disk, reboot to a new process, destroy", async () => { - const driver = new VmManagerDriver(manager()); - expect(await driver.state(DOT)).toEqual({ exists: false, state: "STOPPED", pid: null, guestPort: null, detail: null }); - - const created = await driver.create(input()); - expect(created).toEqual({ - goldenImage: join(paths.imagesDir, "golden-2026.10.0.qcow2"), - runtimeImage: join(paths.imagesDir, "runtime-1.10.iso"), - }); - const started = await driver.start({ ...input(), goldenImage: created.goldenImage }); - expect(started).toMatchObject({ alreadyRunning: false, runtimeImage: created.runtimeImage }); - expect(await driver.state(DOT)).toMatchObject({ exists: true, state: "RUNNING", pid: started.pid, guestPort: started.guestPort }); - const argv = host.spawned.at(-1)!.args; - // The newest runtime ISO is the one attached. - expect(argv.some((arg) => arg.includes("runtime-1.10.iso"))).toBe(true); - - const endpoint = { dotId: DOT, port: started.guestPort }; - const health = await driver.waitForHealth(endpoint, TOKEN, { timeoutMs: 2_000, intervalMs: 5, requestTimeoutMs: 1_000 }); - expect(health).toMatchObject({ agentd: "ok", agent: { status: "ok" } }); - await driver.guest(endpoint, TOKEN).pushSecrets({ openrouter_api_key: "sk-or-x", mcp_secrets: { time: { TOKEN: "t-1" } } }); - expect(guest?.mcpSecrets).toEqual({ time: { TOKEN: "t-1" } }); - expect(guest?.openrouterKey).toBe("sk-or-x"); - // Another token cannot check the guest's proof, so it is never sent; waiting does not help. - await expect(driver.guest(endpoint, "wrong-token-0000").health()).rejects.toMatchObject({ code: "guest_unproven" }); - await expect(driver.waitForHealth(endpoint, "wrong-token-0000", { timeoutMs: 2_000, intervalMs: 5, requestTimeoutMs: 1_000 })).rejects.toMatchObject({ - code: "guest_unproven", - }); - - // Identities through the real client, with the agent's own rules: a bad request and the limit come back as the guest's 4xx. - const client = driver.guest(endpoint, TOKEN); - const identity = await client.createBrowserIdentity({ name: "Shop" }); - expect((await client.listBrowserIdentities()).identities.map((i) => i.id)).toEqual([identity.id]); - expect((await client.getBrowserIdentity(identity.id)).name).toBe("Shop"); - await expect(client.createBrowserIdentity({ name: "proxied", proxy: 8080 as never })).rejects.toMatchObject({ status: 400, code: "invalid" }); - const more = []; - for (let n = 1; n < FAKE_MAX_IDENTITIES; n++) more.push(await client.createBrowserIdentity({ name: `extra-${n}` })); - await expect(client.createBrowserIdentity({ name: "one-too-many" })).rejects.toMatchObject({ status: 409, code: "limit" }); - for (const made of [identity, ...more]) await client.deleteBrowserIdentity(made.id); - - expect(await driver.stop(DOT, TOKEN)).toEqual({ forced: false }); - expect(host.requests).toContain("POST /v1/system/poweroff"); - // Like QEMU's forward, the guest port is gone with the process. - await expect(driver.guest(endpoint, TOKEN).health()).rejects.toMatchObject({ code: "ECONNREFUSED" }); - expect(await driver.state(DOT)).toMatchObject({ exists: true, state: "STOPPED", pid: null, guestPort: null }); - - // A PATCH made computer.disk bigger: the next start grows the overlay first. - await driver.start({ ...input(DOT, "60gb"), goldenImage: created.goldenImage }); - expect(runner.lines().some((line) => line.includes(" resize ") && line.endsWith(String(60 * 1024 ** 3)))).toBe(true); - - const before = (await driver.state(DOT)).pid; - const rebooted = await driver.reboot({ ...input(DOT, "60gb"), goldenImage: created.goldenImage }); - expect(rebooted.pid).not.toBe(before); - expect(rebooted.alreadyRunning).toBe(false); - - await driver.destroy(DOT); - expect(await driver.state(DOT)).toMatchObject({ exists: false, state: "STOPPED" }); - }); - - it("waitForHealth fails at once when QEMU is gone instead of waiting out the timeout", async () => { - const driver = new VmManagerDriver(manager()); - const created = await driver.create(input()); - const started = await driver.start({ ...input(), goldenImage: created.goldenImage }); - guest!.bootPolls = Number.MAX_SAFE_INTEGER; - const waiting = driver.waitForHealth({ dotId: DOT, port: started.guestPort }, TOKEN, { timeoutMs: 60_000, intervalMs: 5, requestTimeoutMs: 1_000 }); - host.terminate([...host.vms.values()][0]!); - await expect(waiting).rejects.toThrow(/not running any more/); - }); -}); - -describe("Scheduler over VmManagerDriver", () => { - let t: TestDatabase; - - beforeEach(async () => { - t = await createTestDatabase("pglite"); - await t.db.secrets.put("global", "openrouter_api_key", "sk-or-real-path"); - }); - - afterEach(async () => { - await t.drop(); - }); - - function scheduler(): Scheduler { - return new Scheduler({ - db: t.db, - driver: new VmManagerDriver(manager()), - lifecycle: { healthPollMs: 5, readyTimeoutMs: 5_000, pumpRetryMs: 10, pumpMaxRetryMs: 50 }, - dispatcher: { retryDelayMs: 0 }, - dispatchIntervalMs: 60_000, - idleCheckIntervalMs: 60_000, - ...(process.env.IDOTS_TEST_DEBUG ? { logger: prefixedStderrLogger("scheduler", true) } : {}), - }); - } - - it("create -> READY -> task over HTTP -> sleep -> restart of the control plane -> wake", { timeout: 30_000 }, async () => { - let s = scheduler(); - const yaml = "name: real-path\nmodel:\n provider: openrouter\n id: m\n"; - const dot = await s.createDot(yaml); - await waitFor(async () => (await t.db.dots.get(dot.id))?.status === "READY", "READY", 10_000); - const computer = await t.db.computers.get(dot.id); - expect(computer).toMatchObject({ state: "RUNNING", vm_name: `invisible-dot-${dot.id}` }); - expect(computer?.guest_port).toBe((await new VmManagerDriver(manager()).state(dot.id)).guestPort); - expect(computer?.pid).toEqual(expect.any(Number)); - expect(guest?.openrouterKey).toBe("sk-or-real-path"); - expect(guest?.config?.name).toBe("real-path"); - - const task = await s.createTask(dot.id, { description: "over http" }); - await waitFor(async () => (await t.db.tasks.get(task.id))?.status === "COMPLETED", "task COMPLETED over HTTP", 10_000); - - // A control plane restart: QEMU keeps running and the new process adopts it from its pid file. - await s.close(); - const spawnedBefore = host.spawned.length; - s = scheduler(); - await s.start(); - await waitFor(() => s.lifecycle.isReady(dot.id), "READY after restart", 10_000); - expect(host.spawned.length).toBe(spawnedBefore); - - await s.lifecycle.stop(dot.id, "idle"); - expect(host.requests).toContain("POST /v1/agent/prepare-sleep"); - expect(host.requests).toContain("POST /v1/system/poweroff"); - expect(await t.db.computers.get(dot.id)).toMatchObject({ state: "STOPPED", guest_port: null, pid: null }); - expect([...host.vms.values()]).toHaveLength(0); - expect(host.killed).toEqual([]); - - const wake = await s.createTask(dot.id, { description: "after sleep" }); - await waitFor(async () => (await t.db.tasks.get(wake.id))?.status === "COMPLETED", "task after wake", 10_000); - expect(host.spawned.length).toBe(spawnedBefore + 1); - await s.close(); - }); -}); diff --git a/apps/api/test/whatsapp.test.ts b/apps/api/test/whatsapp.test.ts deleted file mode 100644 index d56c3fa6..00000000 --- a/apps/api/test/whatsapp.test.ts +++ /dev/null @@ -1,209 +0,0 @@ -/** - * The WhatsApp routes of the control-plane API: starting a link, the stream of codes to scan, and what a server - * that does not run WhatsApp (the default: it is opt-in) says. The channel is the real one on a fake connection. - */ -import type { AddressInfo } from "node:net"; -import { ChannelHub, WhatsAppChannelType } from "@invisible-dots/channels"; -import { FakeChannelType, FakeWhatsAppConnector } from "@invisible-dots/channels/testing"; -import type { Database } from "@invisible-dots/database"; -import { createTestDatabase, testAdapters, type TestDatabase } from "@invisible-dots/database/testing"; -import { Scheduler } from "@invisible-dots/scheduler"; -import { FakeDriver, ManualClock, waitFor, waitUntilSettledReady } from "@invisible-dots/scheduler/testing"; -import { InvisibleDotsClient } from "@invisible-dots/sdk"; -import type { ChannelLinkFrame } from "@invisible-dots/shared"; -import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; -import { buildServer, defaultChannelTypes, type FastifyInstance } from "../src/index.js"; -import { hostFacts } from "./host-facts.js"; - -const API_TOKEN = "test-token-0123456789abcdef"; -const NUMBER = "15550001111"; -const yaml = (name: string) => `name: ${name}\nmodel:\n provider: openrouter\n id: test/model\n`; - -describe.each(testAdapters())("WhatsApp routes of the control-plane API (%s)", { timeout: 60_000 }, (kind) => { - let t: TestDatabase; - let db: Database; - let app: FastifyInstance; - let plain: FastifyInstance; - let scheduler: Scheduler; - let hub: ChannelHub; - let plainHub: ChannelHub; - let driver: FakeDriver; - let connector: FakeWhatsAppConnector; - let base: string; - let plainBase: string; - let api: InvisibleDotsClient; - let seq = 0; - - beforeAll(async () => { - t = await createTestDatabase(kind); - db = t.db; - await db.secrets.put("global", "openrouter_api_key", "sk-or-test"); - driver = new FakeDriver(); - scheduler = new Scheduler({ - db, - driver, - clock: new ManualClock(), - lifecycle: { healthPollMs: 5, readyTimeoutMs: 3_000, pumpRetryMs: 10 }, - dispatcher: { retryDelayMs: 0 }, - }); - connector = new FakeWhatsAppConnector(); - hub = new ChannelHub({ - db, - host: scheduler, - types: [new FakeChannelType(), new WhatsAppChannelType({ connector: () => connector, pauseMs: () => 0 })], - backoff: { initialMs: 1, maxMs: 5, jitter: 0 }, - }); - app = buildServer({ scheduler, channels: hub, doctor: async () => [], host: hostFacts(), token: API_TOKEN, heartbeatMs: 50 }); - // The server as it is when WhatsApp was not asked for: Telegram only. It never starts a binding. - plainHub = new ChannelHub({ db, host: scheduler, types: [new FakeChannelType()] }); - plain = buildServer({ scheduler, channels: plainHub, doctor: async () => [], host: hostFacts(), token: API_TOKEN }); - await scheduler.start(); - await hub.start(); - await app.listen({ host: "127.0.0.1", port: 0 }); - await plain.listen({ host: "127.0.0.1", port: 0 }); - base = `http://127.0.0.1:${(app.server.address() as AddressInfo).port}`; - plainBase = `http://127.0.0.1:${(plain.server.address() as AddressInfo).port}`; - api = new InvisibleDotsClient({ baseUrl: base, token: API_TOKEN }); - }, 60_000); - - afterEach(async () => { - for (const dot of await api.listDots()) { - for (const channel of await api.channels(dot.id)) await api.removeChannel(dot.id, channel.kind); - } - connector.connections.length = 0; - }); - - afterAll(async () => { - await app?.close(); - await plain?.close(); - await hub?.close(); - await scheduler?.close(); - await t?.drop(); - }); - - async function readyDot() { - const name = `wa-${++seq}`; - const dot = await api.createDot(yaml(name)); - await waitUntilSettledReady(scheduler, driver, dot.id, name); - return dot; - } - - async function raw(origin: string, method: string, path: string) { - const response = await fetch(`${origin}${path}`, { method, headers: { authorization: `Bearer ${API_TOKEN}` } }); - const text = await response.text(); - return { status: response.status, text, json: text ? (JSON.parse(text) as Record) : null }; - } - - async function linkedDot() { - const dot = await readyDot(); - await api.linkWhatsApp(dot.id); - await waitFor(() => connector.connections.length === 1, "the connection"); - connector.current.open(NUMBER); - await waitFor(async () => (await api.channels(dot.id))[0]?.status === "connected", "connected"); - return dot; - } - - it("needs the API token like every route", async () => { - const dot = await readyDot(); - for (const [method, path] of [ - ["POST", `/api/dots/${dot.id}/channels/whatsapp/link`], - ["GET", `/api/dots/${dot.id}/channels/whatsapp/qr`], - ] as const) { - expect((await fetch(`${base}${path}`, { method })).status, `${method} ${path}`).toBe(401); - } - }); - - it("lists the kinds this server runs next to the Dot's channels", async () => { - const dot = await readyDot(); - expect(await api.channelsOverview(dot.id)).toEqual({ channels: [], available: ["telegram", "whatsapp"] }); - const plainAnswer = await raw(plainBase, "GET", `/api/dots/${dot.id}/channels`); - expect(plainAnswer.json).toEqual({ channels: [], available: ["telegram"] }); - }); - - it("is off unless the server was started with it, and says how to turn it on", async () => { - const dot = await readyDot(); - const link = await raw(plainBase, "POST", `/api/dots/${dot.id}/channels/whatsapp/link`); - expect(link.status).toBe(400); - expect(link.json).toMatchObject({ error: "invalid_request", message: expect.stringContaining("INVISIBLE_DOTS_WHATSAPP=1") }); - expect((await raw(plainBase, "GET", `/api/dots/${dot.id}/channels/whatsapp/qr`)).status).toBe(400); - }); - - it("starts a link with 202 and the record of a channel that waits, and streams the codes and the end", async () => { - const dot = await readyDot(); - const started = await raw(base, "POST", `/api/dots/${dot.id}/channels/whatsapp/link`); - expect(started.status).toBe(202); - expect(started.json).toMatchObject({ kind: "whatsapp", enabled: true, status: "connecting", account: null, peers: [] }); - - const frames: ChannelLinkFrame[] = []; - const controller = new AbortController(); - const reading = (async () => { - for await (const frame of api.whatsappLink(dot.id, { signal: controller.signal })) frames.push(frame); - })(); - await waitFor(() => connector.connections.length === 1, "the connection"); - await waitFor(() => frames.length === 1, "the first frame"); - connector.current.showCode("2@first"); - await waitFor(() => frames.length === 2, "the first code"); - connector.current.showCode("2@second"); - connector.current.open(NUMBER); - await reading; - expect(frames).toEqual([{ state: "waiting" }, { state: "code", code: "2@first" }, { state: "code", code: "2@second" }, { state: "linked", account: NUMBER }]); - expect(await api.channels(dot.id)).toMatchObject([{ kind: "whatsapp", status: "connected", account: NUMBER }]); - }); - - it("answers the stream as an uncached event stream with a heartbeat, and never writes a code anywhere else", async () => { - const dot = await readyDot(); - await api.linkWhatsApp(dot.id); - await waitFor(() => connector.connections.length === 1, "the connection"); - connector.current.showCode("2@SECRET-CODE-TO-SCAN"); - const controller = new AbortController(); - const response = await fetch(`${base}/api/dots/${dot.id}/channels/whatsapp/qr`, { headers: { authorization: `Bearer ${API_TOKEN}` }, signal: controller.signal }); - expect(response.status).toBe(200); - expect(response.headers.get("content-type")).toMatch(/^text\/event-stream/); - expect(response.headers.get("cache-control")).toBe("no-store"); - const reader = response.body!.pipeThrough(new TextDecoderStream()).getReader(); - let text = ""; - while (!text.includes(": ping")) text += (await reader.read()).value ?? ""; - expect(text).toContain(": connected"); - expect(text).toContain(`data: ${JSON.stringify({ state: "code", code: "2@SECRET-CODE-TO-SCAN" })}`); - controller.abort(); - await reader.cancel().catch(() => {}); - expect(JSON.stringify(await api.channels(dot.id))).not.toContain("SECRET-CODE-TO-SCAN"); - expect(JSON.stringify(await api.events(dot.id))).not.toContain("SECRET-CODE-TO-SCAN"); - }); - - it("is a 404 before anything is linked, as JSON and not as a stream", async () => { - const dot = await readyDot(); - const qr = await raw(base, "GET", `/api/dots/${dot.id}/channels/whatsapp/qr`); - expect(qr.status).toBe(404); - expect(qr.json).toMatchObject({ error: "not_found" }); - expect((await raw(base, "GET", `/api/dots/nothing-here/channels/whatsapp/qr`)).status).toBe(404); - expect((await raw(base, "POST", `/api/dots/nothing-here/channels/whatsapp/link`)).status).toBe(404); - }); - - it("refuses to link a channel that is linked", async () => { - const dot = await linkedDot(); - const again = await raw(base, "POST", `/api/dots/${dot.id}/channels/whatsapp/link`); - expect(again.status).toBe(409); - expect(again.json).toMatchObject({ error: "already_linked" }); - const frames: ChannelLinkFrame[] = []; - for await (const frame of api.whatsappLink(dot.id)) frames.push(frame); - expect(frames).toEqual([{ state: "linked", account: NUMBER }]); - }); - - it("pairs with a link and the words to send, and removes the channel with everything it kept", async () => { - const dot = await linkedDot(); - await db.secrets.put(dot.id, "whatsapp_creds", "identity"); - const pairing = await api.pairChannel(dot.id, "whatsapp"); - expect(pairing.deep_link).toBe(`https://wa.me/${NUMBER}?text=pair%20${pairing.code}`); - expect(pairing.message).toBe(`pair ${pairing.code}`); - await api.removeChannel(dot.id, "whatsapp"); - expect(await api.channels(dot.id)).toEqual([]); - expect(await db.secrets.get(dot.id, "whatsapp_creds")).toBeNull(); - }); - - it("runs WhatsApp only when INVISIBLE_DOTS_WHATSAPP=1, and Telegram always", () => { - expect(defaultChannelTypes({}).map((type) => type.kind)).toEqual(["telegram"]); - expect(defaultChannelTypes({ INVISIBLE_DOTS_WHATSAPP: "0" }).map((type) => type.kind)).toEqual(["telegram"]); - expect(defaultChannelTypes({ INVISIBLE_DOTS_WHATSAPP: "1" }).map((type) => type.kind)).toEqual(["telegram", "whatsapp"]); - }); -}); diff --git a/apps/api/tsconfig.json b/apps/api/tsconfig.json deleted file mode 100644 index 31a061f4..00000000 --- a/apps/api/tsconfig.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "include": [ - "src/**/*.ts", - "test/**/*.ts" - ] -} diff --git a/apps/cli/package.json b/apps/cli/package.json deleted file mode 100644 index ad1c22b8..00000000 --- a/apps/cli/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@invisible-dots/cli", - "version": "0.0.0", - "private": true, - "description": "The invisible-dots command: setup, doctor, image build, server, and the API client commands.", - "license": "MIT", - "type": "module", - "exports": { - ".": "./src/index.ts" - }, - "scripts": { - "typecheck": "tsc -p tsconfig.json", - "build": "node scripts/build.mjs", - "dev": "tsx src/main.ts" - }, - "dependencies": { - "@invisible-dots/api": "0.0.0", - "@invisible-dots/image-builder": "0.0.0", - "@invisible-dots/sdk": "0.0.0", - "@invisible-dots/shared": "0.0.0", - "@invisible-dots/vm-manager": "0.0.0", - "qrcode": "1.5.4" - }, - "bin": { - "invisible-dots": "dist/invisible-dots.mjs" - }, - "devDependencies": { - "@types/qrcode": "1.5.6" - } -} diff --git a/apps/cli/scripts/build.mjs b/apps/cli/scripts/build.mjs deleted file mode 100644 index 901c9a7f..00000000 --- a/apps/cli/scripts/build.mjs +++ /dev/null @@ -1,25 +0,0 @@ -// Bundles the invisible-dots command into one ESM file that plain node runs -// (the workspace packages are TypeScript sources). `invisible-dots server` -// is in the same bundle, so the database's run-time files go next to it. -// The pinned Windows QEMU installer (virtualization/qemu/windows.json) is -// imported as JSON and so is inside the bundle. -import { mkdirSync, rmSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; -import { build } from "esbuild"; -import { copyDatabaseRuntimeFiles } from "../../../packages/database/scripts/runtime-files.mjs"; -import { BUNDLE_OPTIONS } from "./bundle-options.mjs"; - -const root = dirname(dirname(fileURLToPath(import.meta.url))); -const dist = join(root, "dist"); -rmSync(dist, { recursive: true, force: true }); -mkdirSync(dist, { recursive: true }); - -await build({ - ...BUNDLE_OPTIONS, - entryPoints: [join(root, "src", "main.ts")], - outfile: join(dist, "invisible-dots.mjs"), - logLevel: "info", -}); - -copyDatabaseRuntimeFiles(dist); diff --git a/apps/cli/scripts/bundle-options.d.mts b/apps/cli/scripts/bundle-options.d.mts deleted file mode 100644 index 4fbb3d13..00000000 --- a/apps/cli/scripts/bundle-options.d.mts +++ /dev/null @@ -1,4 +0,0 @@ -import type { BuildOptions } from "esbuild"; - -/** The esbuild options of the invisible-dots bundle (bundle-options.mjs). */ -export declare const BUNDLE_OPTIONS: BuildOptions; diff --git a/apps/cli/scripts/bundle-options.mjs b/apps/cli/scripts/bundle-options.mjs deleted file mode 100644 index 5b85646e..00000000 --- a/apps/cli/scripts/bundle-options.mjs +++ /dev/null @@ -1,22 +0,0 @@ -// The esbuild options of the invisible-dots bundle, shared by scripts/build.mjs and tests/repo/cli-bundle.test.ts, -// which bundles a real Telegram call with them: what breaks only once bundled is caught there, not by a person. -export const BUNDLE_OPTIONS = { - bundle: true, - platform: "node", - format: "esm", - target: "node24", - sourcemap: true, - // Classes and functions keep their names. esbuild renames one whose name another module also uses (AbortSignal - // became AbortSignal2), and node-fetch, under grammY, recognizes a signal by its constructor's name: renamed, every - // Telegram call failed as "Network request for 'getMe' failed!", in the bundle only. - keepNames: true, - // pg loads its optional native binding with require(); it is never installed here. - // The opt-in WhatsApp client (Baileys, which depends on libsignal, GPL-3.0) is not in the bundle either: the adapter - // loads it at run time from optional/whatsapp/ by path (packages/channels/src/whatsapp-baileys/client.ts), so no - // import of it is in the bundle's graph, and a bundle that held it would be a GPL work. - external: ["pg-native"], - banner: { - // CommonJS dependencies inside an ESM bundle still call require(). - js: "#!/usr/bin/env node\nimport { createRequire as __createRequire } from 'node:module';\nconst require = __createRequire(import.meta.url);", - }, -}; diff --git a/apps/cli/src/api-client.ts b/apps/cli/src/api-client.ts deleted file mode 100644 index cf01bbbd..00000000 --- a/apps/cli/src/api-client.ts +++ /dev/null @@ -1,32 +0,0 @@ -/** How the CLI reaches the API: its URL and its token, from the environment or INVISIBLE_DOTS_HOME. */ -import { InvisibleDotsClient } from "@invisible-dots/sdk"; -import { ApiTokenError, DEFAULT_LISTEN, ENV, hostPaths, readApiToken } from "@invisible-dots/shared"; - -export const DEFAULT_URL = `http://${DEFAULT_LISTEN}`; - -/** No usable token: the person has to start the server once or set the variable. */ -export class AuthSetupError extends Error {} - -export function apiUrl(env: Record): string { - return env[ENV.URL]?.trim() || DEFAULT_URL; -} - -/** The token as the server reads it (shared readApiToken): INVISIBLE_DOTS_TOKEN, else the first line of api.token. */ -export async function resolveToken(env: Record): Promise { - let token; - try { - token = await readApiToken(env); - } catch (error) { - if (error instanceof ApiTokenError) throw new AuthSetupError(`no usable API token: ${error.message}`); - throw error; - } - if (!token) { - const path = hostPaths(env).apiTokenPath; - throw new AuthSetupError(`no API token: ${path} does not exist yet; start the server once (invisible-dots server) or set ${ENV.TOKEN}`); - } - return token.value; -} - -export async function connectApi(env: Record, fetchImpl?: typeof fetch): Promise { - return new InvisibleDotsClient({ baseUrl: apiUrl(env), token: await resolveToken(env), fetch: fetchImpl }); -} diff --git a/apps/cli/src/cli.ts b/apps/cli/src/cli.ts deleted file mode 100644 index 7b657cde..00000000 --- a/apps/cli/src/cli.ts +++ /dev/null @@ -1,755 +0,0 @@ -/** - * The `invisible-dots` command line. `run` takes its arguments and its world - * (output streams, stdin, environment, fetch, the host commands) as - * parameters, so the tests drive it in-process against a fake server and - * fake host commands. - * - * Two kinds of command live here: the host commands of architecture - * section 11 (setup, doctor, image build, server), which act on this - * machine, and the API client commands, which talk to a running server. - */ -import { readFile, writeFile } from "node:fs/promises"; -import { resolve } from "node:path"; -import { parseArgs } from "node:util"; -import QRCode from "qrcode"; -import { ApiError, readRecentEvents, type ApprovalRecord, type DotSummary, type InvisibleDotsClient, type TaskRecord } from "@invisible-dots/sdk"; -import { - accountLabel, - CHANNEL_KINDS, - COMPUTER_STOPPED, - DEFAULT_WEB_LISTEN, - ENV, - MAX_EVENT_PAGE, - permissionPhrase, - toolLabel, - type ChannelKind, - type ChannelRecord, - type ComputerAnswer, - type McpSecretsAnswer, - type StoredEvent, - type ToolListAnswer, -} from "@invisible-dots/shared"; -import { STORE_OPENROUTER_KEY } from "@invisible-dots/vm-manager"; -import cliPackage from "../package.json" with { type: "json" }; -import { apiUrl, AuthSetupError, connectApi, DEFAULT_URL } from "./api-client.js"; -import { addTelegramChannel } from "./commands.js"; -import { EXIT } from "./exit.js"; - -export { DEFAULT_URL } from "./api-client.js"; -export { EXIT } from "./exit.js"; - -/** The version of this package, which the bundle carries inside it (esbuild inlines the JSON). */ -export const CLI_VERSION: string = cliPackage.version; - -/** How many tasks `status` shows: the newest, which is all it asks the server for. */ -const STATUS_TASKS = 10; - -/** - * The commands that act on this host. The real ones (host.ts) load QEMU - * discovery, the image builder and the whole control plane, so they are - * imported only when one of them runs; tests pass fakes. - */ -/** `image build`: whether the golden image may be downloaded instead of built, and whether it is written compressed. */ -export interface ImageBuildOptions { - download: boolean; - compress: boolean; -} - -export interface HostCommands { - doctor(options: { json: boolean }, io: CliIo): Promise; - setup(io: CliIo): Promise; - /** `setup --all`: setup, the builds and `image build` in one run that can be run again (setup/all.ts). */ - setupAll(io: CliIo): Promise; - imageBuild(io: CliIo, options?: ImageBuildOptions): Promise; - server(io: CliIo, options: { web: boolean }): Promise; -} - -export interface CliIo { - stdout: (text: string) => void; - stderr: (text: string) => void; - /** All of standard input, as text: for input that is piped in. */ - readStdin: () => Promise; - /** - * One line of a secret typed in the terminal, with the echo off, so it stays out of the scrollback. Enter ends it - * on every host; the end-of-input key differs (Ctrl-D on Linux, Ctrl-Z then Enter on Windows), so nothing a person - * types depends on it. It rejects when the person cancels (Ctrl-C). - */ - readSecret: () => Promise; - stdinIsTTY: boolean; - env: Record; - cwd: string; - fetch?: typeof fetch; - /** Aborted on Ctrl-C, to end `logs` and stop `image build`. */ - signal?: AbortSignal; - /** Default: the real host commands. */ - host?: HostCommands; -} - -class UsageError extends Error {} - -export const USAGE = `invisible-dots - control your Dots - -Getting this host ready (the same commands on Linux and Windows): - invisible-dots setup --all everything in one run, in order: the guest daemon, QEMU and its accelerator, the web client, the images; - run it again after a restart or a failure and it carries on - invisible-dots setup just QEMU and its accelerator (may ask for administrator rights once) - invisible-dots doctor [--json] check everything; one line per check and the command that fixes a failure - invisible-dots image build [--no-download] [--compress] - the runtime ISO, and the golden image: downloaded when one is published for - these inputs, built here otherwise (--no-download: always build; --compress: - a compressed qcow2, as the published one is) - invisible-dots server [--no-web] run the control plane and the web client in the foreground (--no-web: the control plane only) - -Using the server: - invisible-dots init [file] [--force] write a sample Dot config (default dot.yaml), check the server - invisible-dots create create a Dot from a YAML config - invisible-dots update replace the Dot's config with a YAML config; refused if it was changed since this command read it - invisible-dots delete --yes delete the Dot: its computer, its disk with everything on it, and its channels (cannot be undone) - invisible-dots list list Dots - invisible-dots status show a Dot, its computer and its recent tasks - invisible-dots message send a chat message - invisible-dots task [--priority N] [--at ISO-8601] - queue a task - invisible-dots tasks list a Dot's tasks - invisible-dots cancel cancel a task that has not finished - invisible-dots computer start|stop|reboot - invisible-dots browser identities list the Dot's browser identities - invisible-dots mcp list the MCP servers the Dot's config declares: where each is on its computer, and which secrets are set - invisible-dots approvals [--all] list pending approvals (--all: every approval, the newest first) - invisible-dots approve [--note text] [--always] - --always also allows that permission for the Dot from now on - invisible-dots reject [--note text] - invisible-dots secret openrouter [--dot ] - store the OpenRouter key: asked for in a terminal, read from stdin when piped (never from arguments) - invisible-dots secret proxy --dot [--clear] - set the Dot's VM proxy (socks5://user:pass@host:port), used from its next start; asked for like the key; --clear removes it - invisible-dots secret mcp --dot [--clear] - set a secret an MCP server of the Dot names (an environment variable or a header, e.g. "Bearer "); asked for like the key; --clear removes it - invisible-dots channel add telegram --dot - link the Dot to a Telegram bot: the token from @BotFather is asked for in a terminal, read from stdin when piped (never from arguments) - invisible-dots channel link whatsapp --dot - link a WhatsApp number (opt-in on the server, unofficial, a ban of the account is possible): shows a code to scan in WhatsApp > Linked devices - invisible-dots channel list [--dot ] list channels (of every Dot without --dot) with their status and paired people - invisible-dots channel pair telegram|whatsapp --dot - print a one-time link (valid 10 minutes) that pairs your Telegram or WhatsApp account to the Dot - invisible-dots channel remove telegram|whatsapp --dot - unlink: the token (or the linked device's keys) and the paired people are deleted - invisible-dots logs [--tail N] [--no-follow] - print recent events, then follow new ones - - is a Dot name or id. Add --json for machine-readable output. invisible-dots --version prints the version. - -Environment: - ${ENV.HOME} the data directory (default ~/.invisible-dots) - ${ENV.WEB_LISTEN} where the web client listens, host:port (default ${DEFAULT_WEB_LISTEN}) - ${ENV.QEMU_DIR} the one directory QEMU is looked for in, when set (otherwise the official installer's directory, then PATH) - ${ENV.URL} server URL (default ${DEFAULT_URL}) - ${ENV.TOKEN} API token (default: the first line of <${ENV.HOME}>/config/api.token) - -Exit codes: 0 ok; 1 the server reported an error, a doctor check is not ok or a setup step failed; -2 usage error; 3 server unreachable; 4 missing or refused token; 5 restart the computer, then run doctor (or setup --all again). -`; - -export const SAMPLE_DOT = `# A Dot configuration (docs/architecture.md, section 7). -name: my-first-dot # lowercase letters, digits and '-', up to 40 -instructions: > - Be concise. Write findings to files in ~/workspace. -model: - provider: openrouter - id: z-ai/glm-5.3-flash # any OpenRouter model id -computer: - cpu: 2 - memory: 4gb - disk: 40gb - idle_timeout: 15m # sleep after 15 minutes with nothing to do; 0 = never -permissions: - computer.exec: allow - browser.identity.delete: ask -limits: - max_steps_per_task: 60 - max_cost_per_task_usd: 1.00 -`; - -const OPTIONS = { - json: { type: "boolean" }, - force: { type: "boolean" }, - priority: { type: "string" }, - at: { type: "string" }, - note: { type: "string" }, - always: { type: "boolean" }, - dot: { type: "string" }, - tail: { type: "string" }, - "no-follow": { type: "boolean" }, - "no-web": { type: "boolean" }, - "no-download": { type: "boolean" }, - compress: { type: "boolean" }, - all: { type: "boolean" }, - clear: { type: "boolean" }, - yes: { type: "boolean" }, - help: { type: "boolean", short: "h" }, - version: { type: "boolean", short: "v" }, -} as const; - -/** - * Whether Ctrl-C asks the command to stop (through `io.signal`) instead of ending the process at once: `logs` ends - * cleanly, and `image build`, alone or as the last step of `setup --all`, kills its builder VM first. `server` installs - * its own handlers. - */ -export function interruptIsAsked(argv: string[]): boolean { - try { - const { positionals, values } = parseArgs({ args: argv, options: OPTIONS, allowPositionals: true, strict: false }); - return positionals[0] === "logs" || positionals[0] === "image" || (positionals[0] === "setup" && values.all === true); - } catch { - return false; - } -} - -/** The command word of an argument list, so main.ts can decide what Ctrl-C does before `run` starts. */ -export function commandOf(argv: string[]): string | undefined { - try { - return parseArgs({ args: argv, options: OPTIONS, allowPositionals: true, strict: false }).positionals[0]; - } catch { - return undefined; - } -} - -function noArguments(args: string[], command: string): void { - if (args.length > 0) throw new UsageError(`${command} takes no arguments, got "${args.join(" ")}"`); -} - -function need(args: string[], index: number, what: string): string { - const value = args[index]; - if (value === undefined || value === "") throw new UsageError(`missing ${what}`); - return value; -} - -function pad(rows: string[][]): string { - const widths: number[] = []; - for (const row of rows) row.forEach((cell, i) => (widths[i] = Math.max(widths[i] ?? 0, cell.length))); - return rows.map((row) => row.map((cell, i) => (i === row.length - 1 ? cell : cell.padEnd(widths[i]!))).join(" ")).join("\n") + "\n"; -} - -function oneLine(text: string, max = 60): string { - const flat = text.replace(/\s+/g, " ").trim(); - return flat.length > max ? `${flat.slice(0, max - 3)}...` : flat; -} - -/** What `status` says about the Dot's automations: when the next one is due, or that the person's stop has paused them (when one was due: a Dot with none has nothing to pause). */ -function nextAutomation(computer: ComputerAnswer): string { - if (computer.stop_reason === "user" && computer.next_automation_at !== null) return "paused: the computer was stopped by you (start it to resume)"; - return computer.next_automation_at ?? "none due"; -} - -export function formatEvent(event: StoredEvent): string { - const { guest_event_id: _id, guest_ts: _ts, ...data } = event.data; - const detail = Object.keys(data).length > 0 ? ` ${JSON.stringify(data)}` : ""; - return `${event.created_at} #${event.id} ${event.type}${detail}`; -} - -function taskRows(tasks: TaskRecord[]): string { - if (tasks.length === 0) return "no tasks\n"; - return pad([ - ["ID", "STATUS", "PRIORITY", "CREATED", "DESCRIPTION"], - ...tasks.map((t) => [t.id, t.status, String(t.priority), t.created_at, oneLine(t.description)]), - ]); -} - -function dotRows(dots: DotSummary[]): string { - if (dots.length === 0) return "no Dots yet: create one with invisible-dots create \n"; - return pad([ - ["NAME", "STATUS", "COMPUTER", "MODEL", "ID"], - ...dots.map((d) => [d.name, d.status, d.computer_state ?? "-", d.config.model.id, d.id]), - ]); -} - -/** How a person is told to finish a pairing on each channel: Telegram's link opens the bot with a Start button, WhatsApp's opens a chat with the words ready to send. */ -const PAIRING_WORDS: Record = { - telegram: { action: "press Start", to: "the bot" }, - whatsapp: { action: "press Send", to: "the number" }, -}; - -function channelRows(rows: { dot: string; channel: ChannelRecord }[]): string { - if (rows.length === 0) return "no channels: link one with invisible-dots channel add telegram --dot \n"; - return pad([ - ["DOT", "CHANNEL", "STATUS", "ACCOUNT", "PEOPLE", "NOTE"], - ...rows.map(({ dot, channel: c }) => [ - dot, - c.kind, - c.enabled ? c.status : "paused", - accountLabel(c.kind, c.account) ?? "-", - c.peers.length === 0 ? "-" : c.peers.map((p) => p.label).join(", "), - c.status_detail ? oneLine(c.status_detail, 80) : "", - ]), - ]); -} - -/** The approvals in the words of the web client's approval card: the Dot by its name, what it wants to do, the permission with its risk. */ -function approvalRows(approvals: ApprovalRecord[], dotNames: ReadonlyMap): string { - return pad([ - ["ID", "STATUS", "DOT", "WANTS TO", "PERMISSION", "REASON"], - // A Dot that was deleted has no name any more: its id is what is left of it. - ...approvals.map((a) => [a.id, a.status, dotNames.get(a.dot_id) ?? a.dot_id, toolLabel(a.tool).ask, permissionPhrase(a.permission), oneLine(a.reason)]), - ]); -} - -/** A Dot config read from a YAML file named on the command line. */ -async function readConfigFile(io: CliIo, path: string): Promise { - const file = resolve(io.cwd, path); - try { - return await readFile(file, "utf8"); - } catch (error) { - throw new UsageError(`cannot read ${file}: ${(error as NodeJS.ErrnoException).code ?? (error as Error).message}`); - } -} - -/** The newest `count` events of the Dot's log, oldest first: read from the end, a page at a time, never from its start. */ -async function newestEvents(client: InvisibleDotsClient, dotId: string, count: number): Promise { - const found: StoredEvent[] = []; - let before: number | undefined; - while (found.length < count) { - const want = Math.min(count - found.length, MAX_EVENT_PAGE); - const page = await readRecentEvents(client, dotId, { count: want, before }); - found.unshift(...page); - if (page.length < want) break; - before = page[0]!.id; - } - return found; -} - -/** One line of a secret, from the terminal (echo off) or from stdin: never from the arguments, which end up in shell history and ps. */ -async function secretFromInput(io: CliIo, what: { prompt: string; noun: string; command: string; spaces?: true }): Promise { - if (io.stdinIsTTY) io.stderr(`paste the ${what.prompt}, then press Enter:\n`); - const value = (io.stdinIsTTY ? await io.readSecret() : await io.readStdin()).trim(); - if (!value) throw new UsageError(`no ${what.noun} given; run "${what.command}" in a terminal and paste it, or pipe it in`); - // A header's value may be "Bearer "; a key or a proxy never has a space, and a line break is never part of one. - if (what.spaces ? /[\r\n]/.test(value) : /\s/.test(value)) { - throw new UsageError(`the ${what.noun} on stdin contains ${what.spaces ? "a line break" : "whitespace"}; pass only the ${what.noun}`); - } - return value; -} - -function mcpRows(tools: ToolListAnswer | null, secrets: McpSecretsAnswer): string { - const servers = new Set([...(tools?.mcp_servers ?? []).map((server) => server.name), ...secrets.secrets.map((secret) => secret.server)]); - if (servers.size === 0 && tools !== null) return "no MCP servers\n"; - const state = (name: string) => { - if (tools === null) return "computer stopped"; - const server = tools.mcp_servers.find((s) => s.name === name); - if (server === undefined) return "-"; - return server.state === "connected" ? `connected, ${server.tools} tools` : server.state === "failed" ? `failed: ${oneLine(server.error ?? "", 80)}` : server.state; - }; - const secretsOf = (name: string) => - secrets.secrets.filter((secret) => secret.server === name).map((secret) => `${secret.name}${secret.set ? "" : " (not set)"}`).join(", ") || "-"; - return pad([["SERVER", "STATE", "SECRETS"], ...[...servers].map((name) => [name, state(name), secretsOf(name)])]); -} - -function channelKind(value: string): ChannelKind { - if (!(CHANNEL_KINDS as readonly string[]).includes(value)) throw new UsageError(`unknown channel "${value}": use ${CHANNEL_KINDS.join(" or ")}`); - return value as ChannelKind; -} - -export async function run(argv: string[], io: CliIo): Promise { - let parsed; - try { - parsed = parseArgs({ args: argv, options: OPTIONS, allowPositionals: true, strict: true }); - } catch (error) { - io.stderr(`invisible-dots: ${(error as Error).message}\nRun "invisible-dots --help" for usage.\n`); - return EXIT.usage; - } - const { values, positionals } = parsed; - if (values.version) { - io.stdout(`${CLI_VERSION}\n`); - return EXIT.ok; - } - const [command, ...args] = positionals; - if (values.help || !command || command === "help") { - io.stdout(USAGE); - return command || values.help ? EXIT.ok : EXIT.usage; - } - - const baseUrl = apiUrl(io.env); - const out = (value: unknown, text: string) => io.stdout(values.json ? `${JSON.stringify(value, null, 2)}\n` : text); - let client: InvisibleDotsClient | undefined; - const api = async () => (client ??= await connectApi(io.env, io.fetch)); - const host = async (): Promise => io.host ?? (await import("./host.js")).realHostCommands(); - - try { - switch (command) { - case "setup": - noArguments(args, "setup"); - return values.all === true ? await (await host()).setupAll(io) : await (await host()).setup(io); - case "doctor": - noArguments(args, "doctor"); - return await (await host()).doctor({ json: values.json === true }, io); - case "image": { - const what = need(args, 0, "build"); - if (what !== "build") throw new UsageError(`unknown image subcommand "${what}": use build`); - noArguments(args.slice(1), "image build"); - return await (await host()).imageBuild(io, { download: values["no-download"] !== true, compress: values.compress === true }); - } - case "server": - noArguments(args, "server"); - return await (await host()).server(io, { web: values["no-web"] !== true }); - case "init": { - const file = resolve(io.cwd, args[0] ?? "dot.yaml"); - try { - await writeFile(file, SAMPLE_DOT, { flag: values.force ? "w" : "wx" }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "EEXIST") { - io.stderr(`invisible-dots: ${file} already exists; use --force to overwrite it\n`); - return EXIT.failed; - } - throw error; - } - io.stdout(`wrote ${file}\n`); - const health = await (await api()).health(); - io.stdout(`server ${baseUrl} is reachable (version ${health.version}, database ${health.database})\n`); - io.stdout(`edit the file, then: invisible-dots create ${args[0] ?? "dot.yaml"}\n`); - return EXIT.ok; - } - case "create": { - const yaml = await readConfigFile(io, need(args, 0, "")); - const dot = await (await api()).createDot(yaml); - out(dot, `created Dot ${dot.name} (${dot.id}); its computer is being provisioned.\nfollow it with: invisible-dots logs ${dot.name}\n`); - return EXIT.ok; - } - case "update": { - const name = need(args, 0, ""); - const yaml = await readConfigFile(io, need(args, 1, "")); - if (args.length > 2) throw new UsageError("update takes a Dot and one YAML file"); - const c = await api(); - // The version read here makes the save conditional: a config changed meanwhile (in the web client, by an - // "always allow") is a 409 and nothing is saved, instead of being overwritten unseen. - const current = await c.getDot(name); - const dot = await c.updateDot(current.id, yaml, current.config_version); - out(dot, `updated Dot ${dot.name} (config version ${dot.config_version})\n`); - return EXIT.ok; - } - case "delete": { - const name = need(args, 0, ""); - noArguments(args.slice(1), "delete "); - if (values.yes !== true) { - throw new UsageError( - `delete destroys Dot ${name}'s computer and its disk, with every file, memory note and browser identity on them, and its channels; it cannot be undone. To do it: invisible-dots delete ${name} --yes`, - ); - } - const c = await api(); - const dot = await c.getDot(name); - const answer = await c.deleteDot(dot.id); - out(answer, `deleting Dot ${dot.name}; what it did stays in its log: invisible-dots logs ${dot.id}\n`); - return EXIT.ok; - } - case "list": { - const dots = await (await api()).listDots(); - out(dots, dotRows(dots)); - return EXIT.ok; - } - case "status": { - const name = need(args, 0, ""); - const c = await api(); - const [dot, computer, tasks] = await Promise.all([c.getDot(name), c.computer(name), c.listTasks(name, { limit: STATUS_TASKS })]); - const text = - pad([ - ["name", dot.name], - ["id", dot.id], - ["status", dot.status + (dot.error ? ` (${dot.error})` : "")], - ["model", dot.config.model.id], - ["computer", `${computer.state}${computer.ready ? ", ready" : ""}${computer.last_error ? ` (last error: ${computer.last_error})` : ""}`], - ["resources", `${dot.config.computer.cpu} cpu, ${dot.config.computer.memory} memory, ${dot.config.computer.disk} disk, idle_timeout ${dot.config.computer.idle_timeout}`], - ["last active", computer.last_active_at ?? "never"], - ["next automation", nextAutomation(computer)], - ]) + - "\n" + - taskRows(tasks); - out({ dot, computer, tasks }, text); - return EXIT.ok; - } - case "message": { - const name = need(args, 0, ""); - const text = args.slice(1).join(" "); - if (!text.trim()) throw new UsageError("missing "); - const answer = await (await api()).sendMessage(name, text); - out( - answer, - answer.delivery === "delivered" - ? `message delivered (${answer.message_id})\n` - : `message queued (${answer.message_id}): the Dot's computer is starting and gets it once READY\n`, - ); - return EXIT.ok; - } - case "task": { - const name = need(args, 0, ""); - const description = args.slice(1).join(" "); - if (!description.trim()) throw new UsageError("missing "); - let priority: number | undefined; - if (values.priority !== undefined) { - priority = Number(values.priority); - if (!Number.isInteger(priority)) throw new UsageError("--priority must be an integer"); - } - if (values.at !== undefined && Number.isNaN(new Date(values.at).getTime())) { - throw new UsageError("--at must be an ISO 8601 time, e.g. 2026-10-03T08:00:00Z"); - } - const task = await (await api()).createTask(name, { - description, - ...(priority !== undefined ? { priority } : {}), - ...(values.at !== undefined ? { scheduled_at: new Date(values.at).toISOString() } : {}), - }); - out(task, `queued task ${task.id}\n`); - return EXIT.ok; - } - case "tasks": { - const tasks = await (await api()).listTasks(need(args, 0, "")); - out(tasks, taskRows(tasks)); - return EXIT.ok; - } - case "cancel": { - const id = need(args, 0, ""); - noArguments(args.slice(1), "cancel "); - const task = await (await api()).cancelTask(id); - out(task, `task ${task.id} ${task.status}\n`); - return EXIT.ok; - } - case "computer": { - const name = need(args, 0, ""); - const action = need(args, 1, "start|stop|reboot"); - const c = await api(); - if (action === "start") await c.startComputer(name); - else if (action === "stop") await c.stopComputer(name); - else if (action === "reboot") await c.rebootComputer(name); - else throw new UsageError(`unknown computer action "${action}": use start, stop or reboot`); - const note = action === "stop" ? `automations are paused while the computer is stopped; resume them with: invisible-dots computer ${name} start\n` : ""; - out({ accepted: true }, `${action} requested; follow it with: invisible-dots logs ${name}\n${note}`); - return EXIT.ok; - } - case "browser": { - const name = need(args, 0, ""); - const what = need(args, 1, "identities"); - if (what !== "identities") throw new UsageError(`unknown browser subcommand "${what}": use identities`); - const identities = await (await api()).listIdentities(name); - out( - identities, - identities.length === 0 - ? "no browser identities\n" - : pad([ - ["ID", "NAME", "STATUS", "LAST USED", "PROXY"], - ...identities.map((i) => [i.id, i.name, i.status, i.lastUsedAt ?? "never", i.hasProxy ? "yes" : "-"]), - ]), - ); - return EXIT.ok; - } - case "approvals": { - const c = await api(); - // With --all the newest come first, as in the web client's history, so the cap of the list drops the oldest and - // never the newest; the pending ones come oldest first, the one that has waited longest on top. - const [approvals, dots] = await Promise.all([values.all ? c.listApprovals(undefined, { order: "desc" }) : c.listApprovals("pending"), c.listDots()]); - out( - approvals, - approvals.length === 0 ? (values.all ? "no approvals\n" : "no pending approvals\n") : approvalRows(approvals, new Map(dots.map((d) => [d.id, d.name]))), - ); - return EXIT.ok; - } - case "approve": - case "reject": { - const id = need(args, 0, ""); - const c = await api(); - if (values.always && command === "reject") throw new UsageError("--always applies to approve, not to reject"); - const note = values.note === undefined ? {} : { note: values.note }; - const approval = - command === "approve" - ? await c.approve(id, { ...note, ...(values.always ? { always: true as const } : {}) }) - : await c.reject(id, note); - out(approval, `approval ${approval.id} ${approval.status}\n`); - return EXIT.ok; - } - case "mcp": { - const name = need(args, 0, ""); - const c = await api(); - const secrets = await c.mcpSecrets(name); - let tools: ToolListAnswer | null = null; - try { - tools = await c.listTools(name); - } catch (error) { - // A stopped computer has no engine to ask where the servers are; their secrets are the host's. - if (!(error instanceof ApiError && error.code === COMPUTER_STOPPED)) throw error; - } - out({ mcp_servers: tools?.mcp_servers ?? null, secrets: secrets.secrets }, mcpRows(tools, secrets)); - return EXIT.ok; - } - case "secret": { - const kind = need(args, 0, "openrouter|proxy|mcp"); - if (kind === "mcp") { - if (values.dot === undefined || values.dot === "") throw new UsageError("missing --dot "); - const server = need(args, 1, ""); - const secretName = need(args, 2, ""); - if (args.length > 3) throw new UsageError("the secret is read from stdin, never from arguments (they end up in shell history and ps)"); - const command = `invisible-dots secret mcp --dot ${values.dot} ${server} ${secretName}`; - const value = values.clear ? null : await secretFromInput(io, { prompt: `value of ${secretName} for the MCP server ${server}`, noun: "secret", command, spaces: true }); - const result = await (await api()).setMcpSecret(values.dot, server, secretName, value); - out(result, `${secretName} of the MCP server ${server} ${value === null ? "cleared" : "stored"} for Dot ${result.dot_id}; a running Dot starts the server again with it\n`); - return EXIT.ok; - } - if (kind === "proxy") { - if (values.dot === undefined || values.dot === "") throw new UsageError("missing --dot "); - if (args.length > 1) throw new UsageError("the proxy is read from stdin, never from arguments (they end up in shell history and ps)"); - const proxy = values.clear ? null : await secretFromInput(io, { prompt: "VM proxy (socks5://user:pass@host:port)", noun: "proxy", command: `invisible-dots secret proxy --dot ${values.dot}` }); - const result = await (await api()).setVmProxy(values.dot, proxy); - out(result, result.proxy ? `VM proxy stored for Dot ${result.dot_id}; it is used from the Dot's next start\n` : `Dot ${result.dot_id} goes out directly from its next start\n`); - return EXIT.ok; - } - if (kind !== "openrouter") throw new UsageError(`unknown secret "${kind}": openrouter, proxy or mcp`); - if (args.length > 1) { - throw new UsageError("the key is read from stdin, never from arguments (they end up in shell history and ps)"); - } - const key = await secretFromInput(io, { prompt: "OpenRouter API key", noun: "key", command: STORE_OPENROUTER_KEY }); - const result = await (await api()).setOpenRouterKey(key, values.dot); - out( - result, - `OpenRouter key stored ${values.dot ? `for Dot ${values.dot}` : "for every Dot"}; pushed to ${result.pushed} running Dot${result.pushed === 1 ? "" : "s"}\n`, - ); - return EXIT.ok; - } - case "channel": { - const what = need(args, 0, "add|link|list|pair|remove"); - const dotName = () => { - if (values.dot === undefined || values.dot === "") throw new UsageError("missing --dot "); - return values.dot; - }; - switch (what) { - case "add": { - const kind = need(args, 1, "telegram"); - if (kind !== "telegram") { - throw new UsageError(`cannot add "${kind}" with a token: only telegram is supported there; WhatsApp is linked with "invisible-dots channel link whatsapp --dot "`); - } - if (args.length > 2) throw new UsageError("the token is read from stdin, never from arguments (they end up in shell history and ps)"); - const name = dotName(); - const token = await secretFromInput(io, { prompt: "Telegram bot token from @BotFather", noun: "token", command: addTelegramChannel(name) }); - const channel = await (await api()).putTelegramChannel(name, token); - out( - channel, - `linked Telegram bot @${channel.account ?? "?"} to Dot ${name}\n` + - `next: invisible-dots channel pair telegram --dot ${name}\n` + - "note: Telegram chats are not end-to-end encrypted; Telegram can read what you and the Dot write there.\n", - ); - return EXIT.ok; - } - case "link": { - const kind = need(args, 1, "whatsapp"); - if (kind !== "whatsapp") throw new UsageError(`cannot link "${kind}" by scanning a code: only whatsapp is linked that way; telegram takes a token ("invisible-dots channel add telegram --dot ")`); - if (args.length > 2) throw new UsageError("channel link takes only the channel and --dot "); - const name = dotName(); - const c = await api(); - await c.linkWhatsApp(name); - io.stderr( - "WhatsApp: this links the number as a device of a personal account through an unofficial client, which WhatsApp can answer by banning the account. Use a number of its own, not the one you live on.\n" + - "On the phone: WhatsApp > Settings > Linked devices > Link a device, then scan the code below (it is replaced every few seconds).\n", - ); - for await (const frame of c.whatsappLink(name, { signal: io.signal })) { - if (values.json) io.stdout(`${JSON.stringify(frame)}\n`); - if (frame.state === "code") { - if (!values.json) io.stdout(`${await QRCode.toString(frame.code, { type: "terminal", small: true })}\n`); - } else if (frame.state === "linked") { - if (!values.json) { - io.stdout(`linked WhatsApp${frame.account ? ` number ${accountLabel("whatsapp", frame.account)}` : ""} to Dot ${name}\nnext: invisible-dots channel pair whatsapp --dot ${name}\n`); - } - return EXIT.ok; - } else if (frame.state === "failed") { - io.stderr(`invisible-dots: ${frame.detail}\n`); - return EXIT.failed; - } else if (frame.detail && !values.json) { - io.stderr(`${frame.detail}; trying again\n`); - } - } - io.stderr("invisible-dots: stopped before the link finished\n"); - return EXIT.failed; - } - case "list": { - noArguments(args.slice(1), "channel list"); - const c = await api(); - const dots = values.dot === undefined ? await c.listDots() : [await c.getDot(values.dot)]; - const rows = (await Promise.all(dots.map(async (d) => (await c.channels(d.id)).map((channel) => ({ dot: d.name, channel }))))).flat(); - out(rows, channelRows(rows)); - return EXIT.ok; - } - case "pair": { - const kind = channelKind(need(args, 1, "telegram")); - const name = dotName(); - const pairing = await (await api()).pairChannel(name, kind); - out( - pairing, - pairing.deep_link - ? `open this link on the device where you use ${kind}, then ${PAIRING_WORDS[kind].action} (valid until ${pairing.expires_at}):\n ${pairing.deep_link}\nor send ${PAIRING_WORDS[kind].to}: ${pairing.message}\n` - : `pairing code ${pairing.code}, valid until ${pairing.expires_at}\n`, - ); - return EXIT.ok; - } - case "remove": { - const kind = channelKind(need(args, 1, "telegram")); - const name = dotName(); - await (await api()).removeChannel(name, kind); - out({ removed: true }, `unlinked ${kind} from Dot ${name}: ${kind === "whatsapp" ? "the linked device's keys" : "its token"} and paired people are deleted\n`); - return EXIT.ok; - } - default: - throw new UsageError(`unknown channel subcommand "${what}": use add, link, list, pair or remove`); - } - } - case "logs": { - const name = need(args, 0, ""); - const tail = values.tail === undefined ? 20 : Number(values.tail); - if (!Number.isInteger(tail) || tail < 0) throw new UsageError("--tail must be a non-negative integer"); - const c = await api(); - const dot = await c.getDot(name).catch((error: unknown) => { - // A deleted Dot's events stay readable by id. - if (error instanceof ApiError && error.status === 404 && name.includes("_")) return { id: name }; - throw error; - }); - // The newest `tail` events, and at least the newest one, which following goes on from. - const recent = await newestEvents(c, dot.id, Math.max(tail, 1)); - const last = recent.at(-1)?.id ?? 0; - for (const event of recent.slice(recent.length - tail)) io.stdout(values.json ? `${JSON.stringify(event)}\n` : `${formatEvent(event)}\n`); - if (values["no-follow"]) return EXIT.ok; - for await (const event of c.stream({ - dotId: dot.id, - after: last, - signal: io.signal, - onReconnect: ({ error }) => io.stderr(`connection lost (${error.message}), reconnecting...\n`), - })) { - io.stdout(values.json ? `${JSON.stringify(event)}\n` : `${formatEvent(event)}\n`); - } - return EXIT.ok; - } - default: - throw new UsageError(`unknown command "${command}"`); - } - } catch (error) { - if (error instanceof UsageError) { - io.stderr(`invisible-dots: ${error.message}\nRun "invisible-dots --help" for usage.\n`); - return EXIT.usage; - } - if (error instanceof AuthSetupError) { - io.stderr(`invisible-dots: ${error.message}\n`); - return EXIT.auth; - } - if (error instanceof ApiError) { - if (error.status === 0) { - io.stderr(`invisible-dots: ${error.message}\nIs the server running (invisible-dots server)? Set ${ENV.URL} if it listens elsewhere.\n`); - return EXIT.unreachable; - } - if (error.status === 401) { - io.stderr(`invisible-dots: the server refused the API token (401); check ${ENV.TOKEN} or api.token\n`); - return EXIT.auth; - } - io.stderr(`invisible-dots: ${error.message} [${error.status} ${error.code}]\n`); - const details = error.details; - if (Array.isArray(details)) { - for (const issue of details as { path?: string; message?: string }[]) { - io.stderr(` ${issue.path ? `${issue.path}: ` : ""}${issue.message ?? ""}\n`); - } - } - return EXIT.failed; - } - // Ctrl-C is how `logs` ends; for every other command it is an interruption. - if (io.signal?.aborted && command === "logs") return EXIT.ok; - io.stderr(`invisible-dots: ${(error as Error).message}\n`); - return EXIT.failed; - } -} diff --git a/apps/cli/src/commands.ts b/apps/cli/src/commands.ts deleted file mode 100644 index e4e96877..00000000 --- a/apps/cli/src/commands.ts +++ /dev/null @@ -1,10 +0,0 @@ -/** - * Command lines this command prints as the fix for something, kept in one - * place so doctor, setup and the usage errors never tell the person two - * different things. Each works as printed in a POSIX shell and in Windows - * PowerShell alike: no redirection, nothing to quote. The ones the vm-manager's - * doctor report names too (setup, the OpenRouter key) are in its errors.ts. - */ - -/** Links a Dot to a Telegram bot: in a terminal it asks for the token, one line; piped, it reads standard input. */ -export const addTelegramChannel = (dot: string): string => `invisible-dots channel add telegram --dot ${dot}`; diff --git a/apps/cli/src/doctor/command.ts b/apps/cli/src/doctor/command.ts deleted file mode 100644 index dca2642d..00000000 --- a/apps/cli/src/doctor/command.ts +++ /dev/null @@ -1,10 +0,0 @@ -/** `invisible-dots doctor`: the report on stdout, exit code 0 only when every check is ok (section 11.1). */ -import { EXIT } from "../exit.js"; -import { allOk, doctorAnswer, runDoctor, type DoctorDeps } from "@invisible-dots/vm-manager"; -import { renderReport } from "./render.js"; - -export async function doctorCommand(deps: DoctorDeps, options: { json: boolean }, out: (text: string) => void): Promise { - const results = await runDoctor(deps); - out(options.json ? `${JSON.stringify(doctorAnswer(results), null, 2)}\n` : renderReport(results)); - return allOk(results) ? EXIT.ok : EXIT.failed; -} diff --git a/apps/cli/src/doctor/render.ts b/apps/cli/src/doctor/render.ts deleted file mode 100644 index 93f726b2..00000000 --- a/apps/cli/src/doctor/render.ts +++ /dev/null @@ -1,22 +0,0 @@ -/** The doctor report as text: one line per check, a fix line under each that is not ok, and a count. */ -import type { DoctorCheck } from "@invisible-dots/shared"; - -const STATUS_WIDTH = "missing".length + 2; - -export function renderReport(results: readonly DoctorCheck[]): string { - const labelWidth = Math.max(...results.map((r) => r.label.length)) + 2; - const indent = " ".repeat(STATUS_WIDTH + labelWidth); - const lines: string[] = []; - for (const r of results) { - lines.push(`${r.status.padEnd(STATUS_WIDTH)}${r.label.padEnd(labelWidth)}${r.detail}`); - if (r.status !== "ok" && r.fix) lines.push(`${indent}fix: ${r.fix}`); - } - const count = (status: DoctorCheck["status"]) => results.filter((r) => r.status === status).length; - const ok = count("ok"); - lines.push( - ok === results.length - ? `all ${results.length} checks ok` - : `${results.length} checks: ${ok} ok, ${count("missing")} missing, ${count("failed")} failed`, - ); - return `${lines.join("\n")}\n`; -} diff --git a/apps/cli/src/exit.ts b/apps/cli/src/exit.ts deleted file mode 100644 index 9fbd4ea5..00000000 --- a/apps/cli/src/exit.ts +++ /dev/null @@ -1,14 +0,0 @@ -/** Exit codes of `invisible-dots`: what went wrong, for scripts. */ -export const EXIT = { - ok: 0, - /** The server answered with an error, a doctor check is not ok, or a setup step failed. */ - failed: 1, - /** Wrong command line. */ - usage: 2, - /** The server could not be reached. */ - unreachable: 3, - /** No API token, or the server refused it. */ - auth: 4, - /** setup enabled something that works only after Windows restarts. */ - restart: 5, -} as const; diff --git a/apps/cli/src/host.ts b/apps/cli/src/host.ts deleted file mode 100644 index 920acebf..00000000 --- a/apps/cli/src/host.ts +++ /dev/null @@ -1,187 +0,0 @@ -/** - * The real host commands: the doctor, setup, image build and server of - * architecture section 11, wired to the real machine. Everything decided - * here comes from the package that owns it: QEMU discovery, the accelerator - * and the doctor report from the vm-manager, image verification and building - * from the image builder (the doctor's image rows through the API), the - * control plane from the API, paths from shared. This file only connects - * them; it holds no platform branch (those are in setup/install.ts and the - * vm-manager's accelerator-access.ts). - * - * cli.ts imports this module only when a host command runs, so the API - * client commands never load QEMU discovery, the image builder or the - * database. - */ -import { randomBytes } from "node:crypto"; -import { mkdtemp, readFile, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { fileURLToPath } from "node:url"; -import { createImageChecks, parseListen, serverLogger, startServer, untilStopSignal } from "@invisible-dots/api"; -import { - buildGoldenImage, - buildRuntimeIso, - defaultRuntimeInputs, - fetchVerified, - type ProcessRunner, -} from "@invisible-dots/image-builder"; -import { allowlistedEnvironment, currentUserSid, hostPaths, type DoctorCheck } from "@invisible-dots/shared"; -import { - accelerator, - findQemu, - firstLine, - hostAccessDeps, - hostDoctorDeps, - NodeCommandRunner, - openRouterCheck, - runProcess, - startProcess, - type DoctorDeps, -} from "@invisible-dots/vm-manager"; -import windowsQemuPinJson from "../../../virtualization/qemu/windows.json" with { type: "json" }; -import { connectApi } from "./api-client.js"; -import type { CliIo, HostCommands, ImageBuildOptions } from "./cli.js"; -import { doctorCommand } from "./doctor/command.js"; -import { EXIT } from "./exit.js"; -import { runSetupAll } from "./setup/all.js"; -import { buildAgent, buildWeb, type BuildDeps } from "./setup/build.js"; -import { currentUserIsRoot, type InstallDeps } from "./setup/install.js"; -import { parseWindowsQemuPin } from "./setup/qemu-pin.js"; -import { serve } from "./serve.js"; -import { prepareHost, runSetup, type SetupDeps } from "./setup/setup.js"; -import { locateWebBuild, startWebServer } from "./web.js"; - -/** - * Whether the running server holds an OpenRouter key. The key lives in the - * database, and the embedded PGlite database belongs to the one server - * process that has it open (a second opener would corrupt it), so doctor - * asks the server instead of reading the database. - */ -async function openRouterKey(env: Record, fetchImpl?: typeof fetch): Promise { - const base = { id: "openrouter", label: "OpenRouter key" } as const; - let health: unknown; - try { - health = await (await connectApi(env, fetchImpl)).health(); - } catch (error) { - return { - ...base, - status: "failed", - detail: `cannot tell while the server is not reachable (${firstLine((error as Error).message)})`, - fix: "start it with: invisible-dots server, then run invisible-dots doctor again", - }; - } - const configured = (health as { openrouter_configured?: unknown }).openrouter_configured; - if (typeof configured === "boolean") return openRouterCheck(configured); - return { ...base, status: "failed", detail: "the server's /api/health does not report openrouter_configured", fix: "update the server" }; -} - -function doctorDeps(io: CliIo): DoctorDeps { - const paths = hostPaths(io.env); - return { - ...hostDoctorDeps({ env: io.env, home: paths.home, images: createImageChecks(paths), openRouterKey: () => openRouterKey(io.env, io.fetch) }), - webBuild: () => locateWebBuild(REPO_ROOT), - }; -} - -function setupDeps(io: CliIo): SetupDeps { - return { doctor: doctorDeps(io), install: installDeps(io), out: io.stdout }; -} - -function buildDeps(io: CliIo): BuildDeps { - return { - run: runProcess, - env: io.env, - repoRoot: REPO_ROOT, - node: process.execPath, - webBuilt: async () => (await locateWebBuild(REPO_ROOT)).missing === undefined, - }; -} - -function installDeps(io: CliIo): InstallDeps { - const log = (line: string) => io.stdout(`${line}\n`); - return { - ...hostAccessDeps(io.env), - log, - readText: (path) => readFile(path, "utf8"), - isRoot: currentUserIsRoot(), - windowsQemuPin: () => parseWindowsQemuPin(windowsQemuPinJson), - userSid: () => currentUserSid(io.env), - uniqueName: () => randomBytes(16).toString("hex"), - makeTempDir: () => mkdtemp(join(tmpdir(), "invisible-dots-setup-")), - removeDir: (path) => rm(path, { recursive: true, force: true }), - download: async (pin, dest) => { - await fetchVerified({ url: pin.url, sha256: pin.sha256 }, dest, { log, ...(io.fetch ? { fetch: io.fetch } : {}) }); - }, - }; -} - -/** The image builder's process boundary: the host's one runner, the same one the vm-manager uses for qemu-img. */ -const commands = new NodeCommandRunner(); -const processRunner: ProcessRunner = { - run: (command, args, options) => commands.run(command, args, options), - spawn: (command, args, options) => startProcess(command, args, { cwd: options.cwd, env: allowlistedEnvironment(process.env) }), -}; - -/** - * The repository root. apps/cli/src/host.ts and the bundle - * apps/cli/dist/invisible-dots.mjs sit at the same depth, so this holds for - * both. The image builder's own default would point inside apps/cli once - * bundled, so its guest files are named here (the engine's source is named by - * defaultRuntimeInputs, from this same root). - */ -const REPO_ROOT = fileURLToPath(new URL("../../../", import.meta.url)); -const IMAGE_BUILDER_DIR = join(REPO_ROOT, "guest", "image-builder"); - -/** - * The runtime ISO first: it takes seconds and fails at once when the agent - * bundle or dot-agentd has not been built, before an hour of golden image - * provisioning. - */ -async function imageBuild(io: CliIo, options: ImageBuildOptions = { download: true, compress: false }): Promise { - const log = (line: string) => io.stdout(`${line}\n`); - const paths = hostPaths(io.env); - const runtime = await buildRuntimeIso({ paths, log, assetRoot: IMAGE_BUILDER_DIR, inputs: defaultRuntimeInputs(REPO_ROOT) }); - log(`${runtime.created ? "built" : "already built"}: ${runtime.iso}`); - const qemu = await findQemu(io.env); - const golden = await buildGoldenImage({ - qemu, - accelerator: accelerator(), - runner: processRunner, - paths, - assetRoot: IMAGE_BUILDER_DIR, - log, - ...(options.download ? {} : { prebuilt: false as const }), - ...(options.compress ? { compress: true } : {}), - ...(io.fetch ? { fetch: io.fetch } : {}), - ...(io.signal ? { signal: io.signal } : {}), - }); - log(`${golden.created ? "ready" : "already there"}: ${golden.image}`); - return EXIT.ok; -} - -export function realHostCommands(): HostCommands { - return { - doctor: (options, io) => doctorCommand(doctorDeps(io), options, io.stdout), - setup: (io) => runSetup(setupDeps(io)), - setupAll: (io) => { - const builds = buildDeps(io); - return runSetupAll({ - out: io.stdout, - agent: () => buildAgent(builds), - host: () => prepareHost(setupDeps(io)), - web: () => buildWeb(builds), - images: () => imageBuild(io), - env: io.env, - ...(io.signal ? { signal: io.signal } : {}), - }); - }, - imageBuild, - server: async (io, options) => { - await serve( - { env: io.env, logger: serverLogger(io.env), web: options.web, repoRoot: REPO_ROOT }, - { startServer, untilStopSignal, parseListen, startWebServer }, - ); - return EXIT.ok; - }, - }; -} diff --git a/apps/cli/src/index.ts b/apps/cli/src/index.ts deleted file mode 100644 index e4029e93..00000000 --- a/apps/cli/src/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./cli.js"; diff --git a/apps/cli/src/main.ts b/apps/cli/src/main.ts deleted file mode 100644 index a969256d..00000000 --- a/apps/cli/src/main.ts +++ /dev/null @@ -1,43 +0,0 @@ -/** The `invisible-dots` executable: wires `run` to the real process. */ -import { commandOf, interruptIsAsked, run } from "./cli.js"; -import { readSecretLine } from "./secret-input.js"; - -async function readStdin(): Promise { - const chunks: Buffer[] = []; - for await (const chunk of process.stdin) chunks.push(chunk as Buffer); - return Buffer.concat(chunks).toString("utf8"); -} - - -const argv = process.argv.slice(2); -const command = commandOf(argv); -const controller = new AbortController(); - -// What Ctrl-C does depends on the command. `logs` ends cleanly. `image build` -// (and `setup --all`, which ends with it) is asked to stop, so it can kill its -// builder VM, and a second Ctrl-C exits at once. `server` installs its own -// handlers (it closes the database before exiting). Every other command keeps -// Node's default: exit immediately. -if (interruptIsAsked(argv)) { - process.on("SIGINT", () => { - if (controller.signal.aborted) process.exit(130); - controller.abort(); - }); -} - -const code = await run(argv, { - stdout: (text) => process.stdout.write(text), - stderr: (text) => process.stderr.write(text), - readStdin, - readSecret: () => readSecretLine(process.stdin, process.stderr), - stdinIsTTY: Boolean(process.stdin.isTTY), - env: process.env, - cwd: process.cwd(), - signal: controller.signal, -}); -process.exitCode = code; -// After a clean stop the server has nothing left to say, but a database or -// VM-manager handle may still hold the event loop; the server's own bin -// exits the same way. Other commands end naturally, which lets piped output -// flush on Windows, where pipe writes are asynchronous. -if (command === "server") process.exit(code); diff --git a/apps/cli/src/secret-input.ts b/apps/cli/src/secret-input.ts deleted file mode 100644 index a6e81378..00000000 --- a/apps/cli/src/secret-input.ts +++ /dev/null @@ -1,49 +0,0 @@ -/** - * One line a person types in the terminal that must not show: an API key or a bot token pasted at a prompt. The - * terminal's own echo would leave it in the scrollback and in any session recording, so the line is read through a - * readline whose output is a stream that swallows what it is given: readline still handles the keys (backspace, - * Ctrl-U, paste) and the terminal is in raw mode, but nothing of the value is written back to the screen. - */ -import { createInterface } from "node:readline"; -import { Writable, type Readable } from "node:stream"; - -/** What a terminal's input is to this reader: a stream that may be put in raw mode. */ -export type TerminalInput = Readable & { isTTY?: boolean; setRawMode?: (mode: boolean) => unknown }; - -/** The person pressed Ctrl-C at the prompt. */ -export class SecretInputCancelled extends Error { - constructor() { - super("cancelled"); - } -} - -/** - * Read one line from `input` with the echo off, then write one newline to `output` (Enter was not echoed either, - * and the next line of output must not follow the prompt on the same line). The line is returned as typed; an end - * of input before Enter returns what was typed so far. - */ -export function readSecretLine(input: TerminalInput, output: Pick): Promise { - return new Promise((resolve, reject) => { - const swallowed = new Writable({ - write(_chunk, _encoding, done) { - done(); - }, - }); - const lines = createInterface({ input, output: swallowed, terminal: true, historySize: 0 }); - let line = ""; - let settled = false; - const finish = (settle: () => void) => { - if (settled) return; - settled = true; - lines.close(); - output.write("\n"); - settle(); - }; - lines.once("line", (text) => { - line = text; - finish(() => resolve(line)); - }); - lines.once("SIGINT", () => finish(() => reject(new SecretInputCancelled()))); - lines.once("close", () => finish(() => resolve(line))); - }); -} diff --git a/apps/cli/src/serve.ts b/apps/cli/src/serve.ts deleted file mode 100644 index 8ac11ecb..00000000 --- a/apps/cli/src/serve.ts +++ /dev/null @@ -1,77 +0,0 @@ -/** - * `invisible-dots server`: the control plane and, next to it, the web client, - * in the foreground until a stop signal. The order is the contract: the - * signal handlers are installed as soon as the control plane listens (a - * Ctrl+C while the web client is starting must still close the database), - * and a stop closes the web client first, then the control plane. - */ -import type { ListenAddress, ListenSetting, StartServerOptions } from "@invisible-dots/api"; -import { DEFAULT_WEB_LISTEN, ENV, type HostPaths, type Logger } from "@invisible-dots/shared"; -import { locateWebBuild, type WebServer, type WebServerOptions } from "./web.js"; - -/** What `serve` needs from the packages that own the processes; tests pass fakes. */ -export interface ServeDeps { - startServer(options: StartServerOptions): Promise<{ url: string; paths: HostPaths; close(): Promise }>; - untilStopSignal(close: () => Promise, logger: Logger, signals?: NodeJS.EventEmitter): Promise; - parseListen(value: string, setting: ListenSetting): ListenAddress; - startWebServer(options: WebServerOptions): Promise; -} - -export interface ServeOptions { - env: Record; - logger: Logger; - /** Serve the web client too (`--no-web` turns it off). */ - web: boolean; - /** The repository (or install) root the web build is found under. */ - repoRoot: string; - /** The process, or a stand-in in tests. */ - signals?: NodeJS.EventEmitter; -} - -/** The web client's listen address: its own default shown in the error, and a port the person can be told. */ -const WEB_LISTEN: ListenSetting = { variable: ENV.WEB_LISTEN, example: DEFAULT_WEB_LISTEN, fixedPort: true }; - -export async function serve(options: ServeOptions, deps: ServeDeps): Promise { - const { env, logger } = options; - // Parsed before anything starts, so a bad INVISIBLE_DOTS_WEB_LISTEN fails the command at once. - const webListen = options.web ? deps.parseListen(env[ENV.WEB_LISTEN]?.trim() || DEFAULT_WEB_LISTEN, WEB_LISTEN) : undefined; - const server = await deps.startServer({ env, logger }); - logger.info(`API token in ${server.paths.apiTokenPath}; stop with Ctrl+C (Dots keep running)`); - - const starting = new AbortController(); - let web: Promise = Promise.resolve(undefined); - const stopped = deps.untilStopSignal( - async () => { - starting.abort(); - try { - await (await web)?.stop(); - } finally { - await server.close(); - } - }, - logger, - options.signals, - ); - - if (webListen) { - // Assigned in the same tick the handlers were installed, so a stop always finds the start it has to end. - web = (async () => { - try { - const started = await deps.startWebServer({ - build: await locateWebBuild(options.repoRoot), - listen: webListen, - apiUrl: server.url, - env, - signal: starting.signal, - onUnexpectedExit: (message) => logger.warn(`${message}; the control plane keeps running`), - }); - logger.info(`web client at ${started.url}`); - return started; - } catch (error) { - if (!starting.signal.aborted) logger.warn(`web client not started: ${(error as Error).message}; the control plane keeps running`); - return undefined; - } - })(); - } - await stopped; -} diff --git a/apps/cli/src/setup/all.ts b/apps/cli/src/setup/all.ts deleted file mode 100644 index 4bb2f69a..00000000 --- a/apps/cli/src/setup/all.ts +++ /dev/null @@ -1,91 +0,0 @@ -/** - * `invisible-dots setup --all`: everything the quick start did by hand after `npm ci`, in order, as one command that - * can be run again. Each step is the code of the command that owns it (the guest daemon and web client builds in - * build.ts, QEMU and its accelerator in setup.ts, `image build` in host.ts) and skips what is already done, so a run - * that stopped, for a restart, a new login or a failure, is continued by running the same command again. - * - * The one step that needs administrator rights stays one step: it is `setup`'s, which asks once. It comes second, - * after the quick daemon build that finds a missing Go before anything is changed, and before the long builds, so a - * restart it asks for costs nothing already done. - */ -import { DEFAULT_WEB_LISTEN, ENV } from "@invisible-dots/shared"; -import { EXIT } from "../exit.js"; -import type { StepResult } from "./build.js"; -import type { HostOutcome } from "./setup.js"; - -/** The command this prints wherever the person has to come back: the same one, which continues. */ -export const SETUP_ALL_COMMAND = "invisible-dots setup --all"; - -export interface SetupAllDeps { - out(text: string): void; - /** Build dot-agentd. */ - agent(): Promise; - /** Get QEMU and its accelerator ready (`setup`). */ - host(): Promise; - /** Build the web client unless it is built. */ - web(): Promise; - /** `image build`: the exit code, or a rejection with the reason. */ - images(): Promise; - env: Record; - /** Aborted by Ctrl-C. Checked between steps: only `image build` can stop in the middle of one. */ - signal?: AbortSignal; -} - -/** The address the web client listens on: the setting, or its default. */ -export function webAddress(env: Record): string { - return `http://${env[ENV.WEB_LISTEN]?.trim() || DEFAULT_WEB_LISTEN}`; -} - -const STEPS = 4; - -export async function runSetupAll(deps: SetupAllDeps): Promise { - const { out } = deps; - const heading = (n: number, text: string) => out(`\n[${n}/${STEPS}] ${text}\n`); - const interrupted = () => deps.signal?.aborted === true; - const stopped = (lines: readonly string[]) => { - for (const line of lines) out(` ${line}\n`); - out(`\nStopped. Nothing after this step was run. Once it is fixed, run ${SETUP_ALL_COMMAND} again: it skips what is done.\n`); - return EXIT.failed; - }; - out(`${SETUP_ALL_COMMAND}: ${STEPS} steps. Each skips what is already done, so run it again after anything that stops it.\n`); - - heading(1, "the guest daemon dot-agentd (Go builds it)"); - const agent = await deps.agent(); - if (!agent.ok) return stopped(agent.lines); - for (const line of agent.lines) out(` ${line}\n`); - if (interrupted()) return stopped(["interrupted"]); - - heading(2, "QEMU and its accelerator (the `setup` command: it asks for administrator rights once, and only if something is missing)"); - const host = await deps.host(); - if (!host.ready) { - if (host.code === EXIT.restart) { - out(`\nRestart the computer now, then run ${SETUP_ALL_COMMAND} again from this folder: it carries on with step 3.\n`); - } else if (host.code !== EXIT.usage) { - out(`\nStopped. When what is listed above is done (log out and in again if it says so), run ${SETUP_ALL_COMMAND} again: it skips what is done.\n`); - } - return host.code; - } - - if (interrupted()) return stopped(["interrupted"]); - heading(3, "the web client"); - const web = await deps.web(); - if (!web.ok) return stopped(web.lines); - for (const line of web.lines) out(` ${line}\n`); - if (interrupted()) return stopped(["interrupted"]); - - heading(4, "the guest images (what `invisible-dots image build` does: the golden image is downloaded when one is published for these inputs)"); - try { - const code = await deps.images(); - if (code !== EXIT.ok) return stopped([`image build ended with exit code ${code}`]); - } catch (error) { - return stopped([`image build stopped: ${(error as Error).message}`]); - } - - out(` -This computer is ready. To use it: - 1. start the server and leave it running: invisible-dots server - 2. open ${webAddress(deps.env)} in your browser - 3. the page asks for your OpenRouter key, then lets you create your first Dot. -`); - return EXIT.ok; -} diff --git a/apps/cli/src/setup/build.ts b/apps/cli/src/setup/build.ts deleted file mode 100644 index 9cadfb49..00000000 --- a/apps/cli/src/setup/build.ts +++ /dev/null @@ -1,84 +0,0 @@ -/** - * The two builds the quick start used to ask for by hand, as steps of `invisible-dots setup --all`: the guest - * daemon dot-agentd (Go) and the web client (Next). What each one runs is owned elsewhere (the image builder names the - * Go command, apps/web/scripts/build.mjs is the web build); this file runs it and says what happened. Both go - * through the host's one process runner, with an argument array and never a shell. - */ -import { agentdBuildCommand, defaultRuntimeInputs } from "@invisible-dots/image-builder"; -import { WEB_BUILD_COMMAND, type Runner, type RunResult } from "@invisible-dots/vm-manager"; -import { webBuildScript } from "../web.js"; - -/** A step either went through (and says what it did) or stopped the run (and says what to do). */ -export interface StepResult { - ok: boolean; - lines: string[]; -} - -export interface BuildDeps { - run: Runner; - /** The environment the builds start from; the Go build adds its own variables. */ - env: Record; - repoRoot: string; - /** The program that runs the web build script: the node running this command. */ - node: string; - /** Whether the web client is built completely (`locateWebBuild`). */ - webBuilt(): Promise; -} - -/** - * How to get Go: both hosts' commands in one line, so that no platform check is needed to choose one. setup never - * runs it (the Windows installer and snap each ask for administrator rights of their own, and setup asks once). - */ -export const GO_INSTALL_HINT = "install it: on Windows `winget install -e --id GoLang.Go`, on Linux `sudo snap install go --classic`, or see https://go.dev/dl"; - -/** Go downloads its toolchain when go.mod asks for a newer one, and compiles the daemon from a cold cache. */ -const AGENT_TIMEOUT_MS = 15 * 60_000; -const WEB_TIMEOUT_MS = 30 * 60_000; -const STDERR_LINES = 10; - -function why(answer: RunResult, limit: number): string { - if (answer.timedOut) return `it did not finish within ${limit / 60_000} minutes`; - return answer.startError?.message ?? `exit code ${answer.code ?? answer.signal}`; -} - -/** - * Builds dot-agentd. Always run, never skipped: Go keeps its own build cache, so an unchanged daemon takes seconds, - * and the image builder turns the same bytes into "already built". - */ -export async function buildAgent(deps: BuildDeps): Promise { - const build = agentdBuildCommand(deps.repoRoot); - const answer = await deps.run(build.command, build.args, { env: { ...deps.env, ...build.env }, timeoutMs: AGENT_TIMEOUT_MS }); - if (answer.startError?.code === "ENOENT") { - return { - ok: false, - lines: [ - "Go is not installed, or is not on PATH: the guest daemon dot-agentd is written in Go", - GO_INSTALL_HINT, - "then open a new terminal (the installer changes PATH) and run: invisible-dots setup --all", - ], - }; - } - if (answer.code !== 0 || answer.timedOut) { - const tail = answer.stderr.trim().split(/\r?\n/).slice(-STDERR_LINES); - return { ok: false, lines: [`building dot-agentd failed (${why(answer, AGENT_TIMEOUT_MS)}):`, ...tail.filter((line) => line.trim() !== "").map((line) => ` ${line}`)] }; - } - return { ok: true, lines: [`built ${defaultRuntimeInputs(deps.repoRoot).agentdBinary}`] }; -} - -/** Builds the web client unless it is complete already (a rebuild after an update is the command this prints). */ -export async function buildWeb(deps: BuildDeps): Promise { - if (await deps.webBuilt()) return { ok: true, lines: [`already built; after an update, build it again with: ${WEB_BUILD_COMMAND}`] }; - // The output is the person's to see: a Next build takes minutes and prints its progress. - const answer = await deps.run(deps.node, [webBuildScript(deps.repoRoot)], { - env: deps.env, - inheritStdio: true, - timeoutMs: WEB_TIMEOUT_MS, - }); - if (answer.code !== 0 || answer.timedOut) { - return { ok: false, lines: [`building the web client failed (${why(answer, WEB_TIMEOUT_MS)}; its output is above)`, `after fixing it, run: invisible-dots setup --all`] }; - } - if (!(await deps.webBuilt())) { - return { ok: false, lines: [`the web client build finished but its server files are not where invisible-dots looks (apps/web/.next/standalone); build it with: ${WEB_BUILD_COMMAND}`] }; - } - return { ok: true, lines: ["built the web client"] }; -} diff --git a/apps/cli/src/setup/install.ts b/apps/cli/src/setup/install.ts deleted file mode 100644 index bc757569..00000000 --- a/apps/cli/src/setup/install.ts +++ /dev/null @@ -1,257 +0,0 @@ -/** - * The CLI's platform split (architecture section 1.1), and the only place in - * apps/cli that asks which operating system it runs on: how QEMU and the - * accelerator are installed, the distribution's package with sudo on Linux, - * one UAC prompt on Windows that enables the HypervisorPlatform feature and - * runs the official QEMU installer silently. How the accelerator is READ - * (/dev/kvm, the feature state) is the vm-manager's `checkAcceleratorAccess`, - * which the API's doctor route runs too. - * - * Everything else `setup` and `doctor` do is the same code on both hosts. - */ -// The Windows branch builds Windows paths whatever host the tests run on. -import { win32 } from "node:path"; -import { ENV } from "@invisible-dots/shared"; -import { - checkKvmDevice, - firstLine, - HYPERVISOR_PLATFORM_FEATURE, - MIN_QEMU_VERSION_TEXT, - officialQemuDir, - powershellArgs, - powershellPath, - UNSUPPORTED_FIX, - type AccessDeps, -} from "@invisible-dots/vm-manager"; -import { - ELEVATED_RESULT_FILE, - elevatedSetupScript, - type ElevatedScriptOptions, - elevationLauncherScript, - parseElevatedResult, - RESTART_REQUIRED_EXIT_CODE, - UNSAFE_WORK_DIR_EXIT_CODE, -} from "./powershell.js"; -import { installerFileName, type WindowsQemuPin } from "./qemu-pin.js"; - -export const APT_PACKAGES = ["qemu-system-x86", "qemu-utils"] as const; - -/** The UAC prompt waits for a person, and the installer copies a few hundred MiB. */ -const ELEVATED_TIMEOUT_MS = 60 * 60 * 1000; - -export interface InstallDeps extends AccessDeps { - log(line: string): void; - readText(path: string): Promise; - /** Linux: whether setup runs as root (see `currentUserIsRoot`), which it refuses. */ - isRoot: boolean; - /** Windows: the pinned installer, read only when QEMU has to be installed. */ - windowsQemuPin(): WindowsQemuPin; - makeTempDir(): Promise; - removeDir(path: string): Promise; - /** Downloads `pin.url` to `dest` as the normal user and rejects unless it hashes to `pin.sha256`. */ - download(pin: WindowsQemuPin, dest: string): Promise; - /** Windows: the security identifier of the user running setup. */ - userSid(): Promise; - /** A name no one can guess, for the elevated session's own directory. */ - uniqueName(): string; -} - -/** - * Whether this process runs as root. getuid exists only on POSIX hosts, so - * on Windows the answer is false: there setup never runs as an - * administrator, it asks for one prompt instead. - */ -export function currentUserIsRoot(): boolean { - return process.getuid?.() === 0; -} - -/** - * Why setup must not go on, or null. As root on Linux it would check - * /dev/kvm for root instead of for the person who runs the server, report - * the accelerator ready, and that person's first Dot would then fail; - * setup calls sudo itself for the one step that needs it, as Windows asks - * for one prompt. - */ -export function setupRefusal(deps: Pick): string | null { - if (deps.platform === "linux" && deps.isRoot) { - return "run invisible-dots setup as the user who runs the server, not as root: it calls sudo itself for the step that needs it"; - } - return null; -} - -export interface InstallRequest { - /** qemu-system-x86_64 or qemu-img is missing or too old. */ - installQemu: boolean; - /** The accelerator check answered `missing`. */ - enableAccelerator: boolean; -} - -export type InstallOutcome = - /** Installed; doctor should now see QEMU and the accelerator. */ - | { kind: "done"; lines: string[] } - /** Installed, but Windows must restart before the accelerator works. */ - | { kind: "restart"; lines: string[] } - /** The person has to run something setup does not run itself. */ - | { kind: "manual"; lines: string[] } - | { kind: "failed"; lines: string[] }; - -/** Installs what doctor found missing, the way this host installs things. */ -export async function installHostPrerequisites(request: InstallRequest, deps: InstallDeps): Promise { - if (deps.platform === "linux") return installOnLinux(request, deps); - if (deps.platform === "win32") return installOnWindows(request, deps); - return { kind: "failed", lines: [`${deps.platform} hosts are not supported: ${UNSUPPORTED_FIX}`] }; -} - -// Linux - -export interface OsRelease { - id: string; - idLike: string[]; - prettyName?: string; -} - -/** /etc/os-release, the fields that say which package manager the distribution uses. */ -export function parseOsRelease(text: string): OsRelease { - const fields: Record = {}; - for (const line of text.split(/\r?\n/)) { - const match = /^([A-Z_]+)=(.*)$/.exec(line.trim()); - if (match) fields[match[1]!] = match[2]!.replace(/^(["'])(.*)\1$/, "$2"); - } - return { - id: (fields.ID ?? "").toLowerCase(), - idLike: (fields.ID_LIKE ?? "").toLowerCase().split(/\s+/).filter(Boolean), - ...(fields.PRETTY_NAME ? { prettyName: fields.PRETTY_NAME } : {}), - }; -} - -export type PackageManager = { kind: "apt" } | { kind: "print"; line: string }; - -/** - * apt is run; other package managers only get their line printed, because a - * wrong package name on a distribution nobody tested would install the wrong - * thing with root rights. - */ -export function packageManagerFor(os: OsRelease): PackageManager { - const family = [os.id, ...os.idLike]; - if (family.some((id) => id === "debian" || id === "ubuntu")) return { kind: "apt" }; - if (family.some((id) => id === "fedora" || id === "rhel" || id === "centos")) { - return { kind: "print", line: "sudo dnf install -y qemu-system-x86 qemu-img" }; - } - if (family.includes("arch")) return { kind: "print", line: "sudo pacman -S --needed qemu-system-x86 qemu-img" }; - return { kind: "print", line: `install qemu-system-x86_64 and qemu-img (QEMU ${MIN_QEMU_VERSION_TEXT} or newer) with your distribution's package manager` }; -} - -async function installOnLinux(request: InstallRequest, deps: InstallDeps): Promise { - const manual: string[] = []; - if (request.installQemu) { - const os = parseOsRelease(await deps.readText("/etc/os-release").catch(() => "")); - const manager = packageManagerFor(os); - if (manager.kind === "apt") { - // Never run as root (setupRefusal), so sudo is always the way to the one root step. - const argv = ["sudo", "apt-get", "install", "-y", ...APT_PACKAGES]; - deps.log(`$ ${argv.join(" ")}`); - const answer = await deps.run(argv[0]!, argv.slice(1), { inheritStdio: true }); - if (answer.code !== 0) { - const why = answer.startError?.message ?? `exit code ${answer.code ?? answer.signal}`; - return { kind: "failed", lines: [`${argv.join(" ")} failed (${why})`] }; - } - } else { - manual.push(`QEMU is not installed by setup on ${os.prettyName ?? (os.id || "this distribution")}; run:`, ` ${manager.line}`); - } - } - - const access = await checkKvmDevice(deps); - if (access.status !== "ok") manual.push(`${access.detail}; fix:`, ` ${access.fix}`); - return manual.length > 0 ? { kind: "manual", lines: manual } : { kind: "done", lines: [] }; -} - -// Windows - -async function installOnWindows(request: InstallRequest, deps: InstallDeps): Promise { - if (!request.installQemu && !request.enableAccelerator) return { kind: "done", lines: [] }; - const work = await deps.makeTempDir(); - // The elevated session's own directory; it checks this is directly under the real ProgramData. - const programData = deps.env.ProgramData ?? deps.env.PROGRAMDATA ?? "C:\\ProgramData"; - const elevatedDir = win32.join(programData, `invisible-dots-setup-${deps.uniqueName()}`); - try { - let installer: ElevatedScriptOptions["installer"]; - if (request.installQemu) { - const pin = deps.windowsQemuPin(); - const path = win32.join(work, installerFileName(pin)); - deps.log(`downloading the official QEMU ${pin.version} installer: ${pin.url}`); - await deps.download(pin, path); - deps.log(`verified its SHA-256 (${pin.sha256})`); - installer = { path, sha256: pin.sha256, silentArgs: pin.silentArgs }; - } - - const userSid = await deps.userSid(); - const steps = [ - ...(request.enableAccelerator ? [`enable the ${HYPERVISOR_PLATFORM_FEATURE} feature (dism)`] : []), - ...(installer ? ["run the QEMU installer silently"] : []), - ]; - deps.log(`asking for administrator rights once, to ${steps.join(" and ")}`); - const elevated = elevatedSetupScript({ - enableHypervisorPlatform: request.enableAccelerator, - ...(installer ? { installer } : {}), - workDir: elevatedDir, - userSid, - }); - const powershell = powershellPath(deps.env); - const launch = await deps.run(powershell, powershellArgs(elevationLauncherScript(powershell, elevated)), { timeoutMs: ELEVATED_TIMEOUT_MS }); - - const text = await deps.readText(win32.join(elevatedDir, ELEVATED_RESULT_FILE)).catch(() => undefined); - if (text === undefined) { - if (launch.code === UNSAFE_WORK_DIR_EXIT_CODE) { - return { - kind: "failed", - lines: [`the elevated step could not create its own directory ${elevatedDir} for administrators only, so it ran nothing; run setup again`], - }; - } - const why = firstLine(launch.stderr) || (launch.timedOut ? "it did not finish within an hour" : `exit code ${launch.code}`); - const declined = /cancel/i.test(launch.stderr); - return { - kind: "failed", - lines: [ - declined - ? "administrator rights were not granted (the prompt was declined); nothing was changed" - : `the elevated step did not report back: ${why}`, - ], - }; - } - return interpretElevatedResult(text, deps.env); - } finally { - await deps.removeDir(elevatedDir).catch(() => undefined); - await deps.removeDir(work).catch(() => undefined); - } -} - -/** What the elevated session's result file means for the person. */ -export function interpretElevatedResult(text: string, env: Record = {}): InstallOutcome { - let result; - try { - result = parseElevatedResult(text); - } catch (error) { - return { kind: "failed", lines: [`the elevated step wrote an unreadable result: ${(error as Error).message}`] }; - } - if (result.error) return { kind: "failed", lines: [`the elevated step failed: ${result.error}`] }; - const lines: string[] = []; - if (result.installer_exit_code === 0) { - lines.push("QEMU installed"); - // The installer reuses the directory of an earlier QEMU, which may be one invisible-dots does not search. - const official = officialQemuDir(env); - // Windows paths compared as Windows does: either separator, any case, no trailing one. - const trim = (dir: string) => dir.replace(/[\\/]+/g, "\\").replace(/\\$/, "").toLowerCase(); - if (result.install_dir && (!official || trim(result.install_dir) !== trim(official))) { - lines.push(`it went to ${result.install_dir}, where an earlier QEMU was installed: set ${ENV.QEMU_DIR}=${result.install_dir} so invisible-dots uses it`); - } - } - if (result.dism_exit_code === RESTART_REQUIRED_EXIT_CODE) { - lines.push( - `the ${HYPERVISOR_PLATFORM_FEATURE} feature is enabled; Windows must restart before it works`, - "restart Windows, then run: invisible-dots doctor", - ); - return { kind: "restart", lines }; - } - if (result.dism_exit_code === 0) lines.push(`the ${HYPERVISOR_PLATFORM_FEATURE} feature is enabled`); - return { kind: "done", lines }; -} diff --git a/apps/cli/src/setup/powershell.ts b/apps/cli/src/setup/powershell.ts deleted file mode 100644 index 96f58c84..00000000 --- a/apps/cli/src/setup/powershell.ts +++ /dev/null @@ -1,160 +0,0 @@ -/** - * The PowerShell that `invisible-dots setup` runs on Windows (architecture - * section 11.2). Pure text generation, so the tests pin every line that will - * run with administrator rights. - * - * Scripts travel as -EncodedCommand (base64 of UTF-16LE), never as a file: - * nothing on disk can be swapped between generating the elevated script and - * the UAC prompt, and no argument needs quoting rules beyond PowerShell's own. - */ - -import { encodeCommand, HYPERVISOR_PLATFORM_FEATURE, powershellArgs } from "@invisible-dots/vm-manager"; - -/** dism's "succeeded, restart required". */ -export const RESTART_REQUIRED_EXIT_CODE = 3010; - -/** - * A PowerShell single-quoted string literal. PowerShell also treats the - * typographic quotes U+2018 to U+201B as single quotes, so they are doubled too. - */ -export function psQuote(value: string): string { - return `'${value.replace(/['\u2018\u2019\u201A\u201B]/g, "$&$&")}'`; -} - -export interface ElevatedScriptOptions { - enableHypervisorPlatform: boolean; - /** The downloaded and verified QEMU installer; omitted when QEMU is already installed. */ - installer?: { path: string; sha256: string; silentArgs: readonly string[] }; - /** - * A directory that does not exist yet, directly under the system's - * ProgramData. The elevated session creates it for administrators and - * SYSTEM alone (the person running setup may read it) and keeps in it - * everything it runs or writes: the installer's copy and the result. - */ - workDir: string; - /** The security identifier of the person running setup, who reads the result and then removes `workDir`. */ - userSid: string; -} - -/** The file in `workDir` the elevated session writes what happened to, as JSON. */ -export const ELEVATED_RESULT_FILE = "result.json"; - -/** The elevated session's exit code when it could not create its private directory, so nothing ran. */ -export const UNSAFE_WORK_DIR_EXIT_CODE = 2; - -/** - * The elevated session. It records each exit code in a JSON file the - * non-elevated setup reads back, because the exit code of an elevated - * process is all Start-Process hands back and it cannot carry both steps. - * - * Nothing it runs or writes is in a place the normal user can change: a - * program running as that user could swap an installer between its hash - * check and its start, plant a DLL next to it, or turn the result file into - * a link to anywhere. So the installer is copied into a new directory under - * ProgramData that only administrators and SYSTEM can write, created with - * that ACL in one step and checked to be empty; the COPY is hashed and run, - * from that directory; the result is written there. dism is found through - * the system directory Windows reports, never an environment variable the - * user can set. - */ -export function elevatedSetupScript(options: ElevatedScriptOptions): string { - const unsafe = UNSAFE_WORK_DIR_EXIT_CODE; - const lines = [ - "# invisible-dots setup: the one step that needs administrator rights (architecture section 11.2).", - "$ErrorActionPreference = 'Stop'", - "$result = [ordered]@{ dism_exit_code = $null; installer_exit_code = $null; install_dir = $null; error = $null }", - `$work = ${psQuote(options.workDir)}`, - `$user = New-Object System.Security.Principal.SecurityIdentifier ${psQuote(options.userSid)}`, - "function New-Rule($sid, $rights) { New-Object System.Security.AccessControl.FileSystemAccessRule($sid, $rights, 'ContainerInherit,ObjectInherit', 'None', 'Allow') }", - `if ((Split-Path -Parent $work) -ne [Environment]::GetFolderPath('CommonApplicationData')) { exit ${unsafe} }`, - "$acl = New-Object System.Security.AccessControl.DirectorySecurity", - "$acl.SetAccessRuleProtection($true, $false)", - "$acl.AddAccessRule((New-Rule (New-Object System.Security.Principal.SecurityIdentifier 'S-1-5-32-544') 'FullControl'))", - "$acl.AddAccessRule((New-Rule (New-Object System.Security.Principal.SecurityIdentifier 'S-1-5-18') 'FullControl'))", - "$acl.AddAccessRule((New-Rule $user 'ReadAndExecute'))", - `if (Test-Path -LiteralPath $work) { exit ${unsafe} }`, - "[void][System.IO.Directory]::CreateDirectory($work, $acl)", - `if (-not ((Get-Acl -LiteralPath $work).AreAccessRulesProtected) -or (@(Get-ChildItem -LiteralPath $work -Force).Count -ne 0)) { exit ${unsafe} }`, - "try {", - ]; - if (options.enableHypervisorPlatform) { - lines.push( - " $dism = Join-Path ([Environment]::SystemDirectory) 'dism.exe'", - ` & $dism /online /enable-feature /featurename:${HYPERVISOR_PLATFORM_FEATURE} /all /norestart`, - " $result.dism_exit_code = $LASTEXITCODE", - ` if ($LASTEXITCODE -ne 0 -and $LASTEXITCODE -ne ${RESTART_REQUIRED_EXIT_CODE}) { throw "dism exited with code $LASTEXITCODE" }`, - ); - } - if (options.installer) { - lines.push( - " $installer = Join-Path $work 'qemu-setup.exe'", - ` Copy-Item -LiteralPath ${psQuote(options.installer.path)} -Destination $installer`, - " $hash = (Get-FileHash -LiteralPath $installer -Algorithm SHA256).Hash.ToLowerInvariant()", - ` if ($hash -ne ${psQuote(options.installer.sha256.toLowerCase())}) { throw "the QEMU installer is not the one that was verified (its SHA-256 is $hash)" }`, - ` $setup = Start-Process -FilePath $installer -ArgumentList ${options.installer.silentArgs.map(psQuote).join(",")} -WorkingDirectory $work -Wait -PassThru`, - " $result.installer_exit_code = $setup.ExitCode", - ' if ($setup.ExitCode -ne 0) { throw "the QEMU installer exited with code $($setup.ExitCode)" }', - " $result.install_dir = (Get-ItemProperty -LiteralPath 'HKLM:\\Software\\qemu64' -ErrorAction SilentlyContinue).Install_Dir", - ); - } - lines.push( - "} catch {", - " $result.error = $_.Exception.Message", - "}", - "Remove-Item -LiteralPath (Join-Path $work 'qemu-setup.exe') -Force -ErrorAction SilentlyContinue", - `[System.IO.File]::WriteAllText((Join-Path $work ${psQuote(ELEVATED_RESULT_FILE)}), ($result | ConvertTo-Json -Compress))`, - "# Nothing in it runs any more: setup, as the normal user, reads the result and removes the directory.", - "$done = Get-Acl -LiteralPath $work", - "$done.AddAccessRule((New-Rule $user 'FullControl'))", - "Set-Acl -LiteralPath $work -AclObject $done", - "", - ); - return lines.join("\r\n"); -} - -/** - * The non-elevated launcher: ONE Start-Process -Verb RunAs -Wait, so the - * person sees exactly one UAC prompt. Declining it makes Start-Process throw, - * which is reported on stderr and as exit code 1. - */ -export function elevationLauncherScript(powershell: string, elevatedScript: string): string { - const args = powershellArgs(elevatedScript).map(psQuote).join(","); - return [ - "$ErrorActionPreference = 'Stop'", - "try {", - ` $elevated = Start-Process -FilePath ${psQuote(powershell)} -Verb RunAs -Wait -PassThru -ArgumentList ${args}`, - " exit $elevated.ExitCode", - "} catch {", - " [Console]::Error.WriteLine($_.Exception.Message)", - " exit 1", - "}", - "", - ].join("\r\n"); -} - -export interface ElevatedResult { - dism_exit_code: number | null; - installer_exit_code: number | null; - /** Where the installer put QEMU, as its own registry value says. */ - install_dir: string | null; - error: string | null; -} - -/** The result file, written by Windows PowerShell; tolerates a byte order mark. */ -export function parseElevatedResult(text: string): ElevatedResult { - const value = JSON.parse(text.replace(/^\uFEFF/, "")) as Record; - const code = (key: string): number | null => { - const v = value[key]; - if (v === null || v === undefined) return null; - if (typeof v !== "number" || !Number.isInteger(v)) throw new Error(`${key} is not an exit code: ${JSON.stringify(v)}`); - return v; - }; - const error = value.error; - const installDir = value.install_dir; - return { - dism_exit_code: code("dism_exit_code"), - installer_exit_code: code("installer_exit_code"), - install_dir: typeof installDir === "string" && installDir.trim() !== "" ? installDir.trim() : null, - error: typeof error === "string" && error.length > 0 ? error : null, - }; -} diff --git a/apps/cli/src/setup/qemu-pin.ts b/apps/cli/src/setup/qemu-pin.ts deleted file mode 100644 index 9a885148..00000000 --- a/apps/cli/src/setup/qemu-pin.ts +++ /dev/null @@ -1,63 +0,0 @@ -/** - * The QEMU version `setup` installs on Windows, pinned in - * virtualization/qemu/windows.json (architecture sections 2 and 11.2): the - * official installer's URL, its SHA-256 and the arguments that run it - * silently. The project never redistributes QEMU; the pin only says which - * official bytes are acceptable and how to run them. Fields this file does - * not read (size, SHA-512, the provenance notes) are for the person who - * updates the pin. - */ - -export interface WindowsQemuPin { - version: string; - url: string; - sha256: string; - /** Passed to the installer in the elevated session, e.g. ["/S"] for NSIS. */ - silentArgs: string[]; -} - -export const WINDOWS_QEMU_PIN_FILE = "virtualization/qemu/windows.json"; - -/** Validates the pin file, so a typo stops setup before anything is downloaded or elevated. */ -export function parseWindowsQemuPin(value: unknown, origin = WINDOWS_QEMU_PIN_FILE): WindowsQemuPin { - if (typeof value !== "object" || value === null) throw new Error(`${origin}: expected a JSON object`); - const record = value as Record; - const text = (key: string): string => { - const v = record[key]; - if (typeof v !== "string" || v.trim() === "") throw new Error(`${origin}: "${key}" must be a non-empty string`); - return v.trim(); - }; - const version = text("version"); - const url = text("url"); - const sha256 = text("sha256").toLowerCase(); - if (!/^\d+\.\d+/.test(version)) throw new Error(`${origin}: "version" must start with major.minor, got "${version}"`); - let parsed: URL; - try { - parsed = new URL(url); - } catch { - throw new Error(`${origin}: "url" is not a URL: "${url}"`); - } - if (parsed.protocol !== "https:") throw new Error(`${origin}: "url" must use https, got "${url}"`); - if (!/^[0-9a-f]{64}$/.test(sha256)) throw new Error(`${origin}: "sha256" must be 64 hexadecimal characters`); - const silent = record.silent_args; - // Without its silent switch the installer opens a wizard in the elevated session and setup waits on it forever. - if (!Array.isArray(silent) || silent.length === 0 || !silent.every((a) => typeof a === "string" && a.trim() !== "")) { - throw new Error(`${origin}: "silent_args" must be a non-empty list of strings, e.g. ["/S"]`); - } - return { version, url, sha256, silentArgs: silent.map((a: string) => a.trim()) }; -} - -/** - * The file name the installer is saved under: the URL's own name when it is - * a plain .exe name, else a fixed one. It ends up inside a PowerShell string - * and a Windows path, so nothing but letters, digits, ".", "_" and "-" is kept. - */ -export function installerFileName(pin: WindowsQemuPin): string { - let last = ""; - try { - last = decodeURIComponent(new URL(pin.url).pathname.split("/").pop() ?? ""); - } catch { - // A malformed escape in the URL: fall back to the fixed name below. - } - return /^[A-Za-z0-9._-]+\.exe$/i.test(last) ? last : `qemu-${pin.version.replace(/[^A-Za-z0-9.-]/g, "")}-setup.exe`; -} diff --git a/apps/cli/src/setup/setup.ts b/apps/cli/src/setup/setup.ts deleted file mode 100644 index c396a896..00000000 --- a/apps/cli/src/setup/setup.ts +++ /dev/null @@ -1,130 +0,0 @@ -/** - * `invisible-dots setup` (architecture section 11.2): run doctor, install - * only what is missing for QEMU and its accelerator, then check again. The - * rest of doctor's list (images, the web client, the OpenRouter key) has its own commands, - * which setup names at the end instead of running. - */ -import type { DoctorCheck, DoctorCheckId } from "@invisible-dots/shared"; -import { allOk, runDoctor, STORE_OPENROUTER_KEY, WEB_BUILD_COMMAND, type DoctorDeps } from "@invisible-dots/vm-manager"; -import { renderReport } from "../doctor/render.js"; -import { EXIT } from "../exit.js"; -import { installHostPrerequisites, setupRefusal, type InstallDeps, type InstallOutcome, type InstallRequest } from "./install.js"; - -/** The checks setup is responsible for; it succeeds when these are ok. */ -export const SETUP_CHECKS: readonly DoctorCheckId[] = ["qemu", "qemu-img", "accelerator", "accelerator-probe"]; - -export interface SetupDeps { - doctor: DoctorDeps; - install: InstallDeps; - out(text: string): void; - /** Default: the platform module's installHostPrerequisites; tests pass a fake. */ - installPrerequisites?: (request: InstallRequest, deps: InstallDeps) => Promise; -} - -function byId(results: readonly DoctorCheck[]): Map { - return new Map(results.map((r) => [r.id, r])); -} - -function setupReady(results: readonly DoctorCheck[]): boolean { - const checks = byId(results); - return SETUP_CHECKS.every((id) => checks.get(id)?.status === "ok"); -} - -/** What is left after QEMU and the accelerator, as the commands that do it. */ -export function nextSteps(results: readonly DoctorCheck[]): string[] { - const checks = byId(results); - const steps: string[] = []; - if (checks.get("golden-image")?.status !== "ok" || checks.get("runtime-image")?.status !== "ok") steps.push("invisible-dots image build"); - // The row is optional (`DoctorDeps.webBuild`): a report without it has no client to build. - const web = checks.get("web"); - if (web && web.status !== "ok") steps.push(WEB_BUILD_COMMAND); - if (checks.get("openrouter")?.status !== "ok") { - steps.push("invisible-dots server (keep it running, then in another terminal:)"); - steps.push(STORE_OPENROUTER_KEY); - } else { - steps.push("invisible-dots server"); - } - return steps; -} - -function printOutcome(outcome: InstallOutcome, out: (text: string) => void): void { - for (const line of outcome.lines) out(`${line}\n`); -} - -/** What `prepareHost` ends with: the doctor report once QEMU and the accelerator are ready, or the exit code of the run that could not get them ready. */ -export type HostOutcome = { ready: true; results: readonly DoctorCheck[] } | { ready: false; code: number }; - -/** - * The work of `setup`, up to QEMU and its accelerator being ready: the one place that does it, run by `setup` itself - * and as a step of `setup --all`. It prints what it does; what to do next is the caller's to say. - */ -export async function prepareHost(deps: SetupDeps): Promise { - const { out } = deps; - const refusal = setupRefusal(deps.install); - if (refusal) { - out(`${refusal}\n`); - return { ready: false, code: EXIT.usage }; - } - out("checking this host first (invisible-dots doctor):\n\n"); - const before = await runDoctor(deps.doctor); - out(`${renderReport(before)}\n`); - - const checks = byId(before); - const status = (id: DoctorCheckId) => checks.get(id)?.status; - const request = { - installQemu: status("qemu") !== "ok" || status("qemu-img") !== "ok", - enableAccelerator: status("accelerator") === "missing", - }; - - if (setupReady(before)) { - out("QEMU and its accelerator are ready; nothing to install.\n"); - return { ready: true, results: before }; - } - if (!request.installQemu && !request.enableAccelerator) { - out("setup cannot fix this by installing something:\n"); - for (const id of SETUP_CHECKS) { - const check = checks.get(id); - if (check && check.status !== "ok") out(` ${check.label}: ${check.detail}${check.fix ? `\n fix: ${check.fix}` : ""}\n`); - } - return { ready: false, code: EXIT.failed }; - } - let outcome: InstallOutcome; - try { - outcome = await (deps.installPrerequisites ?? installHostPrerequisites)(request, deps.install); - } catch (error) { - outcome = { kind: "failed", lines: [(error as Error).message] }; - } - printOutcome(outcome, out); - if (outcome.kind === "restart") return { ready: false, code: EXIT.restart }; - if (outcome.kind === "failed") { - out("setup stopped; nothing after the failed step was run.\n"); - return { ready: false, code: EXIT.failed }; - } - if (outcome.kind === "manual") { - out("then run: invisible-dots doctor\n"); - return { ready: false, code: EXIT.failed }; - } - out("\nchecking again:\n\n"); - const after = await runDoctor(deps.doctor); - out(`${renderReport(after)}\n`); - if (!setupReady(after)) { - out("QEMU or its accelerator is still not usable; see the fix lines above.\n"); - return { ready: false, code: EXIT.failed }; - } - return { ready: true, results: after }; -} - -export async function runSetup(deps: SetupDeps): Promise { - const outcome = await prepareHost(deps); - return outcome.ready ? finish(outcome.results, deps.out) : outcome.code; -} - -function finish(results: readonly DoctorCheck[], out: (text: string) => void): number { - if (allOk(results)) { - out("this host is ready: invisible-dots server\n"); - } else { - out("QEMU is ready. Next:\n"); - for (const step of nextSteps(results)) out(` ${step}\n`); - } - return EXIT.ok; -} diff --git a/apps/cli/src/web.ts b/apps/cli/src/web.ts deleted file mode 100644 index 21752e98..00000000 --- a/apps/cli/src/web.ts +++ /dev/null @@ -1,164 +0,0 @@ -/** - * The web client as a child of `invisible-dots server` (architecture section - * 9.7): the built Next standalone server, started on its own loopback port and - * stopped with the control plane. The web client is a companion, never a - * dependency: when it cannot start, the control plane and the command line go - * on without it, and the server says why. - */ -import { stat } from "node:fs/promises"; -import { dirname, join } from "node:path"; -import { abortableSleep, allowlistedEnvironment, BASE_CHILD_ENV_VARS, ENV } from "@invisible-dots/shared"; -import { startProcess, WEB_BUILD_COMMAND, type StartedProcess, type StartProcessOptions, type WebBuild } from "@invisible-dots/vm-manager"; - -/** - * Where `npm run build --workspace @invisible-dots/web` leaves the server - * (apps/web/scripts/standalone.mjs writes it there: Next puts the standalone - * tree under .next/standalone and the app at its own path inside it, because - * the workspace root is the tracing root) and the static files it serves. - */ -const ENTRY = ["apps", "web", ".next", "standalone", "apps", "web", "server.js"] as const; -const STATIC = ["apps", "web", ".next", "standalone", "apps", "web", ".next", "static"] as const; - -/** The script `npm run build --workspace @invisible-dots/web` runs, which `setup --all` runs with node itself. */ -export function webBuildScript(repoRoot: string): string { - return join(repoRoot, "apps", "web", "scripts", "build.mjs"); -} - -export async function locateWebBuild(repoRoot: string): Promise { - const entry = join(repoRoot, ...ENTRY); - for (const path of [entry, join(repoRoot, ...STATIC)]) { - if (!(await stat(path).catch(() => undefined))) return { entry, missing: path }; - } - return { entry, missing: undefined }; -} - -export class WebStartError extends Error {} - -export interface WebServer { - /** The address the person opens, e.g. http://127.0.0.1:3000. */ - url: string; - /** Stop the child; resolves once it has exited. Idempotent. */ - stop(): Promise; -} - -export interface WebServerOptions { - build: WebBuild; - /** A fixed port: the person has to know where to go (parseListen refuses port 0 for this setting). */ - listen: { host: string; port: number }; - /** The control plane's address, where the web server's proxy sends `/api`. */ - apiUrl: string; - /** The server's own environment: only what the web server needs is passed on. */ - env: Record; - /** Called once if the child exits while the server runs on. */ - onUnexpectedExit: (message: string) => void; - /** Ends the wait for the child to answer, e.g. on Ctrl+C during startup. */ - signal?: AbortSignal; - start?: (command: string, args: readonly string[], options: StartProcessOptions) => StartedProcess; - /** How long the child gets to answer its first request. */ - readyTimeoutMs?: number; - /** How long the child gets to exit after SIGTERM before it is killed. */ - stopTimeoutMs?: number; -} - -const READY_TIMEOUT_MS = 30_000; -const STOP_TIMEOUT_MS = 10_000; -const READY_POLL_MS = 100; - -/** - * The environment of the web server. Not the server's own: it may hold - * DATABASE_URL with its password. The web server gets what it reads: where - * to listen (Next's PORT and HOSTNAME), where the API is, which data - * directory holds `api.token` (read on every request, so a rotated token - * takes effect), the API token itself only when the server was given it that - * way, and the extra host names its Host check allows. `parentPid` is the - * server's own pid: the web server exits when that process is gone, so a - * `kill -9` of the server does not leave it holding the port. - */ -export function webEnvironment(options: Pick & { parentPid: number }): Record { - const { env, listen, apiUrl, parentPid } = options; - return { - ...allowlistedEnvironment(env, [...BASE_CHILD_ENV_VARS, ENV.HOME, ENV.TOKEN, ENV.WEB_ALLOWED_HOSTS]), - [ENV.URL]: apiUrl, - [ENV.WEB_PARENT_PID]: String(parentPid), - PORT: String(listen.port), - HOSTNAME: listen.host, - }; -} - -function urlOf(listen: { host: string; port: number }): string { - return `http://${listen.host.includes(":") ? `[${listen.host}]` : listen.host}:${listen.port}`; -} - -/** Whether anything answers HTTP at `url`. */ -function answers(url: string): Promise { - return fetch(url, { redirect: "manual", signal: AbortSignal.timeout(2000) }).then( - (response) => response.body?.cancel().then(() => true, () => true) ?? true, - () => false, - ); -} - -/** - * Start the web server and resolve once it answers a request. Throws - * WebStartError, with the child already gone, when the build is incomplete, - * the child exits first (a port in use ends here), it does not answer in time - * or `signal` aborts. - */ -export async function startWebServer(options: WebServerOptions): Promise { - const { build, listen } = options; - if (build.missing) { - throw new WebStartError(`the web client is not built (${build.missing} does not exist); build it with: ${WEB_BUILD_COMMAND}`); - } - if (options.signal?.aborted) throw new WebStartError("stopped before the web client started"); - const url = urlOf(listen); - // Something else answering there would make the wait below succeed for the wrong server. - if (await answers(url)) { - throw new WebStartError(`${url} is already in use; free the port or set ${ENV.WEB_LISTEN} to another host:port`); - } - // The server script chdirs into its own directory, so that is where it starts: never the server's directory. - const env = webEnvironment({ ...options, parentPid: process.pid }); - const child = (options.start ?? startProcess)(process.execPath, [build.entry], { cwd: dirname(build.entry), env }); - - let exited = false; - let started = false; - let stopping = false; - let exitText = ""; - const exit = new Promise((resolve) => { - child.onExit((code, signal, error) => { - exited = true; - const why = error?.message ?? (code === null ? `signal ${signal}` : `exit code ${code}`); - exitText = `the web server exited (${why})${child.stderrTail().trim() ? `: ${child.stderrTail().trim().split(/\r?\n/).slice(-3).join(" | ")}` : ""}`; - if (started && !stopping) options.onUnexpectedExit(exitText); - resolve(); - }); - }); - // The server's own hard exit (a second Ctrl+C) must not leave the child holding the port. - const killAtExit = () => child.kill("SIGKILL"); - process.on("exit", killAtExit); - - const stop = async () => { - if (stopping) return exit; - stopping = true; - process.off("exit", killAtExit); - child.kill("SIGTERM"); - const grace = setTimeout(() => child.kill("SIGKILL"), options.stopTimeoutMs ?? STOP_TIMEOUT_MS); - await exit; - clearTimeout(grace); - }; - - const deadline = Date.now() + (options.readyTimeoutMs ?? READY_TIMEOUT_MS); - try { - for (;;) { - if (options.signal?.aborted) throw new WebStartError("stopped while the web client was starting"); - if (exited) throw new WebStartError(exitText); - if (Date.now() > deadline) throw new WebStartError(`the web server did not answer at ${url} within ${(options.readyTimeoutMs ?? READY_TIMEOUT_MS) / 1000} s`); - if ((await answers(url)) && !exited) { - started = true; - return { url, stop }; - } - await abortableSleep(READY_POLL_MS, options.signal); - } - } catch (error) { - await stop(); - throw error; - } -} diff --git a/apps/cli/test/build.test.ts b/apps/cli/test/build.test.ts deleted file mode 100644 index 3dc7541d..00000000 --- a/apps/cli/test/build.test.ts +++ /dev/null @@ -1,98 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { agentdBuildCommand, defaultRuntimeInputs } from "@invisible-dots/image-builder"; -import { WEB_BUILD_COMMAND } from "@invisible-dots/vm-manager"; -import { fakeRunner, type Answer } from "../../vm-manager/test/doctor-fakes.js"; -import { buildAgent, buildWeb, GO_INSTALL_HINT, type BuildDeps } from "../src/setup/build.js"; -import { webBuildScript } from "../src/web.js"; - -const REPO = "/repo"; - -function deps(respond: (command: string, args: readonly string[]) => Answer, options: { built?: boolean[] } = {}) { - const runner = fakeRunner(respond); - const built = [...(options.built ?? [false, true])]; - const result: BuildDeps = { - run: runner.run, - env: { PATH: "/usr/bin", HOME: "/home/someone" }, - repoRoot: REPO, - node: "/usr/bin/node", - webBuilt: async () => built.length > 1 ? built.shift()! : built[0]!, - }; - return { deps: result, calls: runner.calls }; -} - -describe("the guest daemon build", () => { - it("runs the image builder's go build command with its linux/amd64 environment added to the caller's", async () => { - const { deps: d, calls } = deps(() => ({})); - const step = await buildAgent(d); - const build = agentdBuildCommand(REPO); - expect(calls).toHaveLength(1); - expect(calls[0]!.command).toBe("go"); - expect(calls[0]!.args).toEqual(build.args); - expect(calls[0]!.options.env).toEqual({ PATH: "/usr/bin", HOME: "/home/someone", CGO_ENABLED: "0", GOOS: "linux", GOARCH: "amd64" }); - expect(step).toEqual({ ok: true, lines: [`built ${defaultRuntimeInputs(REPO).agentdBinary}`] }); - }); - - it("asks for Go, with both hosts' install commands, when it is not installed, and to open a new terminal", async () => { - const missing = Object.assign(new Error("spawn go ENOENT"), { code: "ENOENT" }); - const { deps: d } = deps(() => ({ code: null, startError: missing })); - const step = await buildAgent(d); - expect(step.ok).toBe(false); - expect(step.lines).toContain(GO_INSTALL_HINT); - expect(GO_INSTALL_HINT).toContain("winget install -e --id GoLang.Go"); - expect(GO_INSTALL_HINT).toContain("sudo snap install go --classic"); - expect(step.lines.join("\n")).toContain("open a new terminal"); - expect(step.lines.join("\n")).toContain("invisible-dots setup --all"); - }); - - it("shows the end of the compiler's complaint when the build fails", async () => { - const stderr = Array.from({ length: 30 }, (_, i) => `error line ${i}`).join("\n"); - const { deps: d } = deps(() => ({ code: 1, stderr })); - const step = await buildAgent(d); - expect(step.ok).toBe(false); - expect(step.lines[0]).toBe("building dot-agentd failed (exit code 1):"); - expect(step.lines).toHaveLength(11); - expect(step.lines.at(-1)).toBe(" error line 29"); - }); - - it("reports a build that ran out of time", async () => { - const { deps: d } = deps(() => ({ code: null, timedOut: true })); - expect((await buildAgent(d)).lines[0]).toBe("building dot-agentd failed (it did not finish within 15 minutes):"); - }); -}); - -describe("the web client build", () => { - it("skips a client that is built, and says how to build it again", async () => { - const { deps: d, calls } = deps(() => ({}), { built: [true] }); - const step = await buildWeb(d); - expect(calls).toEqual([]); - expect(step.ok).toBe(true); - expect(step.lines[0]).toContain(WEB_BUILD_COMMAND); - }); - - it("runs the web build script with node, no shell, Next's telemetry off and the person's terminal for its output", async () => { - const { deps: d, calls } = deps(() => ({})); - const step = await buildWeb(d); - expect(step).toEqual({ ok: true, lines: ["built the web client"] }); - expect(calls).toHaveLength(1); - expect(calls[0]!.command).toBe("/usr/bin/node"); - expect(calls[0]!.args).toEqual([webBuildScript(REPO)]); - expect(calls[0]!.options.env).toMatchObject({ PATH: "/usr/bin" }); - expect(calls[0]!.options.inheritStdio).toBe(true); - }); - - it("stops with the command to run again when the build fails", async () => { - const { deps: d } = deps(() => ({ code: 1 })); - const step = await buildWeb(d); - expect(step.ok).toBe(false); - expect(step.lines[0]).toBe("building the web client failed (exit code 1; its output is above)"); - expect(step.lines[1]).toContain("invisible-dots setup --all"); - }); - - it("does not call a build done whose server files are not there", async () => { - const { deps: d } = deps(() => ({}), { built: [false] }); - const step = await buildWeb(d); - expect(step.ok).toBe(false); - expect(step.lines[0]).toContain("not where invisible-dots looks"); - expect(step.lines[0]).toContain(WEB_BUILD_COMMAND); - }); -}); diff --git a/apps/cli/test/cli.test.ts b/apps/cli/test/cli.test.ts deleted file mode 100644 index 6bd3605c..00000000 --- a/apps/cli/test/cli.test.ts +++ /dev/null @@ -1,813 +0,0 @@ -import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; -import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; -import type { AddressInfo } from "node:net"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { parseDotConfig } from "@invisible-dots/shared"; -import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; -import cliPackage from "../package.json" with { type: "json" }; -import { commandOf, EXIT, interruptIsAsked, run, SAMPLE_DOT, type CliIo, type HostCommands } from "../src/index.js"; - -const TOKEN = "cli-test-token-0123456789"; - -interface Recorded { - method: string; - path: string; - query: URLSearchParams; - body: unknown; -} - -const now = "2026-10-02T08:00:00.000Z"; -const dot = { - id: "dot_01abc", - name: "fare-watch", - config: { - name: "fare-watch", - model: { provider: "openrouter", id: "test/model" }, - computer: { cpu: 2, memory: "4gb", disk: "40gb", idle_timeout: "15m" }, - }, - status: "READY", - error: null, - config_version: 1, - created_at: now, - updated_at: now, - computer_state: "RUNNING", -}; -const computer = { - dot_id: dot.id, - vm_name: "invisible-dot-dot_01abc", - guest_port: 40123, - pid: 4242, - state: "RUNNING", - golden_image: null, - runtime_image: null, - event_cursor: 3, - last_active_at: now, - next_automation_at: "2030-01-01T09:00:00.000Z", - stop_reason: null, - last_error: null, - updated_at: now, - ready: true, -}; -const task = { - id: "task_01", - dot_id: dot.id, - description: "find the cheapest day", - priority: 0, - status: "COMPLETED", - created_at: now, - scheduled_at: null, - started_at: now, - finished_at: now, - summary: "tuesday", - error: null, -}; -const channel = { - kind: "telegram", - enabled: true, - status: "connected", - status_detail: null, - account: "dot_helper_bot", - settings: { approvals: true, notify_tasks: true, show_arguments: true }, - peers: [{ peer_id: "10", role: "owner", label: "Ann (@ann)", created_at: now }], - created_at: now, -}; -let events = Array.from({ length: 5 }, (_, i) => ({ - id: i + 1, - dot_id: dot.id, - type: i % 2 ? "agent.state" : "automation.next_run", - data: i % 2 ? { state: "IDLE", guest_event_id: "x", guest_ts: now } : { next_run_at_ms: null }, - source: "guest", - guest_seq: i + 1, - created_at: now, -})); - -let server: Server; -let base: string; -const requests: Recorded[] = []; -let stopped = false; -let stoppedByPerson = false; -/** The Dot has no automation due: the engine reported none. */ -let noAutomationDue = false; -/** What the fake server answers about WhatsApp. */ -let whatsappLinked = false; -let whatsappOff = false; -let linkFrames: { state: string; [key: string]: unknown }[] = []; -/** The config_version the fake server holds: an update that expects another one is the 409 of a concurrent change. */ -let configVersion = 1; -/** Another client saves the Dot's config between a read and the PATCH that follows it. */ -let savedMeanwhile = false; - -function send(res: ServerResponse, status: number, body?: unknown) { - res.writeHead(status, { "content-type": "application/json" }); - res.end(body === undefined ? undefined : JSON.stringify(body)); -} - -async function handle(req: IncomingMessage, res: ServerResponse) { - const url = new URL(req.url ?? "/", "http://x"); - const chunks: Buffer[] = []; - for await (const chunk of req) chunks.push(chunk as Buffer); - const text = Buffer.concat(chunks).toString("utf8"); - const body = text ? JSON.parse(text) : undefined; - requests.push({ method: req.method ?? "", path: url.pathname, query: url.searchParams, body }); - if (req.headers.authorization !== `Bearer ${TOKEN}`) return send(res, 401, { error: "unauthorized", message: "bad token" }); - const route = `${req.method} ${url.pathname}`; - const byName = (p: string) => p.replace("/fare-watch", `/${dot.id}`); - switch (byName(route)) { - case "GET /api/health": - return send(res, 200, { status: "ok", database: "ok", version: "9.9.9" }); - case "POST /api/dots": - if (typeof body?.config === "string" && body.config.includes("Bad Name")) { - return send(res, 400, { - error: "invalid_config", - message: "invalid Dot configuration: name: must be lowercase", - details: [{ path: "name", message: "must be lowercase" }], - }); - } - return send(res, 201, { ...dot, status: "CREATING" }); - case "GET /api/dots": - return send(res, 200, { dots: [dot] }); - case `GET /api/dots/${dot.id}`: - return send(res, 200, { ...dot, config_version: configVersion }); - case `PATCH /api/dots/${dot.id}`: - if (savedMeanwhile) configVersion++; - if (body?.expected_config_version !== configVersion) return send(res, 409, { error: "dot_changed", message: "the Dot's config changed since it was read" }); - return send(res, 200, { ...dot, config_version: configVersion + 1 }); - case `DELETE /api/dots/${dot.id}`: - return send(res, 202, { accepted: true }); - case "POST /api/tasks/task_02/cancel": - return send(res, 200, { ...task, id: "task_02", status: "CANCELLED" }); - case "POST /api/tasks/task_01/cancel": - return send(res, 409, { error: "task_finished", message: "task task_01 is already COMPLETED" }); - case `GET /api/dots/${dot.id}/computer`: - return send(res, 200, { ...(stoppedByPerson ? { ...computer, state: "STOPPED", stop_reason: "user", ready: false } : computer), ...(noAutomationDue ? { next_automation_at: null } : {}) }); - case `GET /api/dots/${dot.id}/tasks`: - return send(res, 200, { tasks: [task] }); - case `POST /api/dots/${dot.id}/tasks`: - return send(res, 201, { ...task, id: "task_02", status: "PENDING", description: body.description }); - case `POST /api/dots/${dot.id}/messages`: - return send(res, 202, { message_id: "msg_1", event_id: 9, delivery: "queued" }); - case `POST /api/dots/${dot.id}/computer/start`: - case `POST /api/dots/${dot.id}/computer/stop`: - case `POST /api/dots/${dot.id}/computer/reboot`: - return send(res, 202, { accepted: true }); - case `GET /api/dots/${dot.id}/browser-identities`: - if (stopped) return send(res, 409, { error: "computer_stopped", message: "the computer of Dot fare-watch is STOPPED; start it first" }); - return send(res, 200, { - identities: [ - { id: "shop-abc123", name: "Shop", createdAt: now, lastUsedAt: null, status: "available", profilePath: "/p", hasProxy: false }, - { id: "work-def456", name: "Work", createdAt: now, lastUsedAt: null, status: "available", profilePath: "/q", hasProxy: true }, - ], - }); - case "GET /api/approvals": - return send(res, 200, { - approvals: [{ id: "apr_1", dot_id: dot.id, task_id: null, tool: "browser_identity_delete", permission: "browser.identity.delete", arguments: {}, reason: "cleanup", status: "pending", note: null, created_at: now, resolved_at: null }], - }); - case "POST /api/approvals/apr_1/approve": - return send(res, 200, { id: "apr_1", status: "approved", note: body?.note ?? null }); - case "POST /api/approvals/apr_1/reject": - return send(res, 200, { id: "apr_1", status: "rejected", note: body?.note ?? null }); - case "PUT /api/secrets/openrouter": - return send(res, 200, { pushed: 2 }); - case `GET /api/dots/${dot.id}/mcp-secrets`: - return send(res, 200, { dot_id: dot.id, secrets: [{ server: "web", name: "Authorization", set: false }] }); - case `PUT /api/dots/${dot.id}/mcp-secrets/web/Authorization`: - return send(res, 200, { dot_id: dot.id, secrets: [{ server: "web", name: "Authorization", set: true }] }); - case `DELETE /api/dots/${dot.id}/mcp-secrets/web/Authorization`: - return send(res, 200, { dot_id: dot.id, secrets: [{ server: "web", name: "Authorization", set: false }] }); - case `GET /api/dots/${dot.id}/tools`: - if (stopped) return send(res, 409, { error: "computer_stopped", message: "the computer of Dot fare-watch is STOPPED; start it first" }); - return send(res, 200, { - tools: [], - mcp_servers: [ - { name: "time", state: "connected", error: null, tools: 2 }, - { name: "web", state: "failed", error: "its secret Authorization is not set", tools: 0 }, - ], - }); - case `GET /api/dots/${dot.id}/channels`: - return send(res, 200, { channels: whatsappLinked ? [channel, { ...channel, kind: "whatsapp", account: "15550001111", peers: [] }] : [channel] }); - case `PUT /api/dots/${dot.id}/channels/telegram`: - if (body?.token === "9:REFUSED-TOKEN-VALUE") { - return send(res, 400, { error: "invalid_credentials", message: "Telegram refused the bot token: it is wrong, or it was revoked in @BotFather. Paste a current token." }); - } - return send(res, 201, channel); - case `POST /api/dots/${dot.id}/channels/telegram/pairing`: - return send(res, 201, { code: "ABCD2345", deep_link: "https://t.me/dot_helper_bot?start=ABCD2345", message: "/start ABCD2345", expires_at: "2026-10-02T08:10:00.000Z" }); - case `POST /api/dots/${dot.id}/channels/whatsapp/pairing`: - if (whatsappLinked) { - return send(res, 201, { code: "WXYZ6789", deep_link: "https://wa.me/15550001111?text=pair%20WXYZ6789", message: "pair WXYZ6789", expires_at: "2026-10-02T08:10:00.000Z" }); - } - return send(res, 201, { code: "WXYZ6789", deep_link: null, message: "pair WXYZ6789", expires_at: "2026-10-02T08:10:00.000Z" }); - case `POST /api/dots/${dot.id}/channels/whatsapp/link`: - if (whatsappOff) return send(res, 400, { error: "invalid_request", message: 'no "whatsapp" channel: it is off: set INVISIBLE_DOTS_WHATSAPP=1 and restart the server to turn it on' }); - return send(res, 202, { ...channel, kind: "whatsapp", status: "connecting", account: null, peers: [] }); - case `GET /api/dots/${dot.id}/channels/whatsapp/qr`: - res.writeHead(200, { "content-type": "text/event-stream" }); - res.write(": connected\n\n"); - for (const frame of linkFrames) res.write(`data: ${JSON.stringify(frame)}\n\n`); - if (linkFrames.at(-1)?.state === "linked" || linkFrames.at(-1)?.state === "failed") return res.end(); - return; // left open, like the real stream - case `DELETE /api/dots/${dot.id}/channels/telegram`: - case `DELETE /api/dots/${dot.id}/channels/whatsapp`: - return send(res, 204); - case `GET /api/dots/${dot.id}/events`: { - // The route's paging: after and before bound the ids, order=desc gives the newest first, limit keeps that many. - const after = Number(url.searchParams.get("after") ?? 0); - const before = Number(url.searchParams.get("before") ?? Infinity); - const kept = events.filter((e) => e.id > after && e.id < before); - const ordered = url.searchParams.get("order") === "desc" ? [...kept].reverse() : kept; - return send(res, 200, { events: ordered.slice(0, Number(url.searchParams.get("limit") ?? 1000)) }); - } - case "GET /api/stream": { - res.writeHead(200, { "content-type": "text/event-stream" }); - const after = Number(url.searchParams.get("after") ?? 0); - res.write(": connected\n\n"); - const live = { ...events[0]!, id: after + 1, type: "task.completed", data: { task_id: "task_02", summary: "done" } }; - res.write(`id: ${live.id}\ndata: ${JSON.stringify(live)}\n\n`); - return; // left open, like the real stream - } - default: - return send(res, 404, { error: "not_found", message: `no route ${route}` }); - } -} - -let configDir: string; - -beforeAll(async () => { - server = createServer((req, res) => void handle(req, res)); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; - configDir = await mkdtemp(join(tmpdir(), "idots-cli-")); -}); - -afterAll(async () => { - server.closeAllConnections(); - await new Promise((resolve) => server.close(() => resolve())); - await rm(configDir, { recursive: true, force: true }); -}); - -beforeEach(() => { - requests.length = 0; - stopped = false; - stoppedByPerson = false; - noAutomationDue = false; - whatsappLinked = false; - whatsappOff = false; - linkFrames = []; - configVersion = 1; - savedMeanwhile = false; -}); - -async function cli( - argv: string[], - options: { env?: Record; stdin?: string; line?: string; tty?: boolean; signal?: AbortSignal; host?: HostCommands } = {}, -) { - let stdout = ""; - let stderr = ""; - const io: CliIo = { - stdout: (t) => (stdout += t), - stderr: (t) => (stderr += t), - readStdin: async () => options.stdin ?? "", - readSecret: async () => options.line ?? "", - stdinIsTTY: options.tty ?? false, - env: { INVISIBLE_DOTS_URL: base, INVISIBLE_DOTS_TOKEN: TOKEN, INVISIBLE_DOTS_HOME: configDir, ...options.env }, - cwd: configDir, - ...(options.signal ? { signal: options.signal } : {}), - ...(options.host ? { host: options.host } : {}), - }; - const code = await run(argv, io); - return { code, stdout, stderr }; -} - -describe("usage", () => { - it("prints help with exit 0, and exit 2 with no command, an unknown command or an unknown flag", async () => { - expect((await cli(["--help"])).code).toBe(EXIT.ok); - const none = await cli([]); - expect(none.code).toBe(EXIT.usage); - expect(none.stdout).toContain("invisible-dots setup"); - expect((await cli(["launch"])).stderr).toMatch(/unknown command "launch"/); - expect((await cli(["list", "--bogus"])).code).toBe(EXIT.usage); - expect((await cli(["status"])).stderr).toMatch(/missing /); - expect((await cli(["--version"])).stdout).toBe(`${cliPackage.version}\n`); - }); -}); - -describe("server and token", () => { - it("reads the token from /config/api.token when INVISIBLE_DOTS_TOKEN is unset", async () => { - await mkdir(join(configDir, "config"), { recursive: true }); - await writeFile(join(configDir, "config", "api.token"), `${TOKEN}\n`); - const result = await cli(["list"], { env: { INVISIBLE_DOTS_TOKEN: "" } }); - expect(result.code).toBe(EXIT.ok); - await rm(join(configDir, "config"), { recursive: true }); - }); - - it("exit 4 without a token or with a refused one, exit 3 when the server is unreachable", async () => { - const missing = await cli(["list"], { env: { INVISIBLE_DOTS_TOKEN: "", INVISIBLE_DOTS_HOME: join(configDir, "none") } }); - expect(missing.code).toBe(EXIT.auth); - expect(missing.stderr).toMatch(/no API token: .*api\.token does not exist yet; start the server once/); - const refused = await cli(["list"], { env: { INVISIBLE_DOTS_TOKEN: "wrong-token-123456789" } }); - expect(refused.code).toBe(EXIT.auth); - expect(refused.stderr).toMatch(/refused the API token/); - const down = await cli(["list"], { env: { INVISIBLE_DOTS_URL: "http://127.0.0.1:1" } }); - expect(down.code).toBe(EXIT.unreachable); - expect(down.stderr).toMatch(/cannot reach the invisible_dots API/); - }); -}); - -describe("commands", () => { - it("init writes the sample, checks the server and refuses to overwrite without --force", async () => { - const first = await cli(["init", "sample.yaml"]); - expect(first.code).toBe(EXIT.ok); - expect(first.stdout).toMatch(/is reachable \(version 9\.9\.9/); - expect(await readFile(join(configDir, "sample.yaml"), "utf8")).toBe(SAMPLE_DOT); - const again = await cli(["init", "sample.yaml"]); - expect(again.code).toBe(EXIT.failed); - expect(again.stderr).toMatch(/already exists/); - expect((await cli(["init", "sample.yaml", "--force"])).code).toBe(EXIT.ok); - }); - - it("writes a sample the host accepts as it is: every key of it is one the schema knows", () => { - expect(parseDotConfig(SAMPLE_DOT).name).toBe("my-first-dot"); - }); - - it("create sends the YAML text and prints validation details on 400", async () => { - await writeFile(join(configDir, "good.yaml"), SAMPLE_DOT); - const ok = await cli(["create", "good.yaml"]); - expect(ok.code).toBe(EXIT.ok); - expect(ok.stdout).toMatch(/created Dot fare-watch/); - expect(requests.find((r) => r.method === "POST")?.body).toEqual({ config: SAMPLE_DOT }); - - await writeFile(join(configDir, "bad.yaml"), "name: Bad Name\n"); - const bad = await cli(["create", "bad.yaml"]); - expect(bad.code).toBe(EXIT.failed); - expect(bad.stderr).toMatch(/\[400 invalid_config\]/); - expect(bad.stderr).toMatch(/name: must be lowercase/); - expect((await cli(["create", "missing.yaml"])).code).toBe(EXIT.usage); - }); - - it("list, status and tasks print tables, and --json prints the data", async () => { - const list = await cli(["list"]); - expect(list.stdout).toMatch(/^NAME\s+STATUS\s+COMPUTER/); - expect(list.stdout).toContain("fare-watch READY"); - const json = await cli(["list", "--json"]); - expect(JSON.parse(json.stdout)[0].id).toBe(dot.id); - const status = await cli(["status", "fare-watch"]); - expect(status.code).toBe(EXIT.ok); - expect(status.stdout).toMatch(/computer\s+RUNNING, ready/); - expect(status.stdout).toMatch(/next automation\s+2030-01-01T09:00:00.000Z/); - expect(status.stdout).toContain("task_01"); - // It shows the newest ten tasks, and asks for ten. - expect(requests.find((r) => r.path === "/api/dots/fare-watch/tasks")?.query.get("limit")).toBe("10"); - expect((await cli(["tasks", "fare-watch"])).stdout).toContain("COMPLETED"); - }); - - it("status does not say paused for a Dot the person stopped that has no automation due", async () => { - stoppedByPerson = true; - noAutomationDue = true; - const status = await cli(["status", "fare-watch"]); - expect(status.stdout).toMatch(/next automation\s+none due/); - expect(status.stdout).not.toMatch(/paused/); - }); - - it("status says the automations are paused while the person's stop lasts, and stop says so when it is asked for", async () => { - stoppedByPerson = true; - const status = await cli(["status", "fare-watch"]); - expect(status.stdout).toMatch(/computer\s+STOPPED/); - expect(status.stdout).toMatch(/next automation\s+paused: the computer was stopped by you \(start it to resume\)/); - expect(status.stdout).not.toContain("2030-01-01T09:00:00.000Z"); - - const stop = await cli(["computer", "fare-watch", "stop"]); - expect(stop.stdout).toMatch(/automations are paused while the computer is stopped; resume them with: invisible-dots computer fare-watch start/); - expect((await cli(["computer", "fare-watch", "start"])).stdout).not.toMatch(/paused/); - }); - - it("message and task send what was typed", async () => { - const message = await cli(["message", "fare-watch", "how", "is", "it", "going?"]); - expect(message.stdout).toMatch(/message queued/); - expect(requests.at(-1)?.body).toEqual({ text: "how is it going?" }); - const queued = await cli(["task", "fare-watch", "check", "Lisbon", "--priority", "3", "--at", "2030-01-01T08:00:00Z"]); - expect(queued.code).toBe(EXIT.ok); - expect(requests.at(-1)?.body).toEqual({ description: "check Lisbon", priority: 3, scheduled_at: "2030-01-01T08:00:00.000Z" }); - expect((await cli(["task", "fare-watch", "x", "--priority", "high"])).code).toBe(EXIT.usage); - expect((await cli(["message", "fare-watch"])).code).toBe(EXIT.usage); - }); - - it("computer actions, and browser identities with the server's 409 shown", async () => { - for (const action of ["start", "stop", "reboot"]) { - expect((await cli(["computer", "fare-watch", action])).code).toBe(EXIT.ok); - expect(requests.at(-1)?.path).toBe(`/api/dots/fare-watch/computer/${action}`); - } - expect((await cli(["computer", "fare-watch", "pause"])).code).toBe(EXIT.usage); - const listed = (await cli(["browser", "fare-watch", "identities"])).stdout; - expect(listed).toContain("shop-abc123"); - // The identity has no proxy of its own, the normal case: the column says none instead of asking for one; one that has a - // proxy says only that. - expect(listed).toMatch(/shop-abc123\s+Shop\s+available\s+never\s+-\s*$/m); - expect(listed).toMatch(/work-def456\s+Work\s+available\s+never\s+yes\s*$/m); - stopped = true; - const conflict = await cli(["browser", "fare-watch", "identities"]); - expect(conflict.code).toBe(EXIT.failed); - expect(conflict.stderr).toMatch(/start it first \[409 computer_stopped\]/); - }); - - it("approvals, approve and reject", async () => { - const pending = await cli(["approvals"]); - expect(pending.stdout).toContain("apr_1"); - const listed = requests.find((r) => r.path === "/api/approvals"); - expect(listed?.query.get("status")).toBe("pending"); - expect(listed?.query.get("order")).toBeNull(); - // The Dot by its name, and the question and the permission in the web client's words. - expect(pending.stdout.split("\n")[0]).toMatch(/^ID\s+STATUS\s+DOT\s+WANTS TO\s+PERMISSION\s+REASON$/); - expect(pending.stdout).toMatch(/apr_1\s+pending\s+fare-watch\s+delete a browser identity\s+Delete browser identities \(high risk\)\s+cleanup/); - requests.length = 0; - await cli(["approvals", "--all"]); - const all = requests.find((r) => r.path === "/api/approvals"); - expect(all?.query.get("status")).toBeNull(); - // The newest first, so the cap of the list drops the oldest and never the newest. - expect(all?.query.get("order")).toBe("desc"); - const approved = await cli(["approve", "apr_1", "--note", "fine"]); - expect(approved.stdout).toBe("approval apr_1 approved\n"); - expect(requests.at(-1)?.body).toEqual({ note: "fine" }); - expect((await cli(["reject", "apr_1"])).stdout).toBe("approval apr_1 rejected\n"); - expect(requests.at(-1)?.body).toEqual({}); - await cli(["approve", "apr_1", "--always"]); - expect(requests.at(-1)?.body).toEqual({ always: true }); - await cli(["approve", "apr_1", "--always", "--note", "ok"]); - expect(requests.at(-1)?.body).toEqual({ note: "ok", always: true }); - const refused = await cli(["reject", "apr_1", "--always"]); - expect(refused.code).toBe(EXIT.usage); - expect(refused.stderr).toContain("--always applies to approve"); - }); - - it("secret openrouter reads the key from stdin only", async () => { - const stored = await cli(["secret", "openrouter"], { stdin: "sk-or-v1-abc\n" }); - expect(stored.code).toBe(EXIT.ok); - expect(stored.stdout).toMatch(/pushed to 2 running Dots/); - expect(requests.at(-1)?.body).toEqual({ value: "sk-or-v1-abc" }); - await cli(["secret", "openrouter", "--dot", "fare-watch"], { stdin: "sk-or-v1-abc" }); - expect(requests.at(-1)?.body).toEqual({ value: "sk-or-v1-abc", dot_id: "fare-watch" }); - const inArgs = await cli(["secret", "openrouter", "sk-or-v1-abc"]); - expect(inArgs.code).toBe(EXIT.usage); - expect(inArgs.stderr).toMatch(/never from arguments/); - expect((await cli(["secret", "openrouter"], { stdin: " " })).code).toBe(EXIT.usage); - }); - - it("secret mcp reads the value from stdin only, keeps a space a header needs, and --clear removes it", async () => { - const stored = await cli(["secret", "mcp", "--dot", "fare-watch", "web", "Authorization"], { stdin: "Bearer tok-1\n" }); - expect(stored.code).toBe(EXIT.ok); - expect(stored.stdout).toContain("Authorization of the MCP server web stored"); - expect(requests.at(-1)).toMatchObject({ method: "PUT", body: { value: "Bearer tok-1" } }); - expect(stored.stdout).not.toContain("tok-1"); - const cleared = await cli(["secret", "mcp", "--dot", "fare-watch", "web", "Authorization", "--clear"]); - expect(cleared.stdout).toContain("cleared"); - expect(requests.at(-1)?.method).toBe("DELETE"); - const inArgs = await cli(["secret", "mcp", "--dot", "fare-watch", "web", "Authorization", "tok-1"]); - expect(inArgs.code).toBe(EXIT.usage); - expect(inArgs.stderr).toMatch(/never from arguments/); - expect((await cli(["secret", "mcp", "web", "Authorization"], { stdin: "x" })).stderr).toContain("missing --dot"); - }); - - it("mcp lists the declared servers with where each is and which secrets are set, and still the secrets of a stopped Dot", async () => { - const listed = await cli(["mcp", "fare-watch"]); - expect(listed.code).toBe(EXIT.ok); - expect(listed.stdout).toMatch(/time\s+connected, 2 tools\s+-/); - expect(listed.stdout).toMatch(/web\s+failed: its secret Authorization is not set\s+Authorization \(not set\)/); - stopped = true; - try { - const off = await cli(["mcp", "fare-watch"]); - expect(off.code).toBe(EXIT.ok); - expect(off.stdout).toMatch(/web\s+computer stopped\s+Authorization \(not set\)/); - } finally { - stopped = false; - } - }); - - it("secret openrouter in a terminal reads one line, which Enter ends on every host", async () => { - // A terminal never reaches end of input by itself, and the key that ends it differs by host: - // only the line is read, so the command never waits for it. - const typed = await cli(["secret", "openrouter"], { - tty: true, - line: "sk-or-v1-typed", - stdin: "never read: a terminal does not end", - }); - expect(typed.code).toBe(EXIT.ok); - expect(typed.stderr).toBe("paste the OpenRouter API key, then press Enter:\n"); - expect(typed.stderr).not.toMatch(/Ctrl/); - expect(requests.at(-1)?.body).toEqual({ value: "sk-or-v1-typed" }); - const empty = await cli(["secret", "openrouter"], { tty: true, line: "" }); - expect(empty.code).toBe(EXIT.usage); - // The hint is a command that runs as printed in PowerShell too: no "<" redirection. - expect(empty.stderr).toContain('run "invisible-dots secret openrouter" in a terminal'); - expect(empty.stderr).not.toContain("<"); - }); - - it("channel add reads the token from stdin or one line of a terminal, never from arguments, and never prints it", async () => { - const piped = await cli(["channel", "add", "telegram", "--dot", "fare-watch"], { stdin: "123456:SECRET-TOKEN-VALUE\n" }); - expect(piped.code).toBe(EXIT.ok); - expect(requests.at(-1)).toMatchObject({ method: "PUT", path: `/api/dots/fare-watch/channels/telegram`, body: { token: "123456:SECRET-TOKEN-VALUE" } }); - expect(piped.stdout).toContain("linked Telegram bot @dot_helper_bot to Dot fare-watch\n"); - expect(piped.stdout).toContain("next: invisible-dots channel pair telegram --dot fare-watch\n"); - expect(piped.stdout).toMatch(/not end-to-end encrypted/); - expect(piped.stdout + piped.stderr).not.toContain("SECRET-TOKEN"); - - const typed = await cli(["channel", "add", "telegram", "--dot", "fare-watch"], { tty: true, line: "123456:TYPED-TOKEN-VALUE", stdin: "never read" }); - expect(typed.code).toBe(EXIT.ok); - expect(typed.stderr).toBe("paste the Telegram bot token from @BotFather, then press Enter:\n"); - expect(requests.at(-1)?.body).toEqual({ token: "123456:TYPED-TOKEN-VALUE" }); - - const inArgs = await cli(["channel", "add", "telegram", "123456:SECRET-TOKEN-VALUE", "--dot", "fare-watch"]); - expect(inArgs.code).toBe(EXIT.usage); - expect(inArgs.stderr).toMatch(/never from arguments/); - expect(inArgs.stderr).not.toContain("SECRET-TOKEN"); - const empty = await cli(["channel", "add", "telegram", "--dot", "fare-watch"], { tty: true, line: "" }); - expect(empty.code).toBe(EXIT.usage); - expect(empty.stderr).toContain('run "invisible-dots channel add telegram --dot fare-watch" in a terminal'); - expect((await cli(["channel", "add", "telegram"], { stdin: "123456:SECRET-TOKEN-VALUE" })).stderr).toMatch(/missing --dot /); - expect((await cli(["channel", "add", "whatsapp", "--dot", "fare-watch"])).stderr).toMatch(/only telegram is supported/); - const json = await cli(["channel", "add", "telegram", "--dot", "fare-watch", "--json"], { stdin: "123456:SECRET-TOKEN-VALUE" }); - expect(JSON.parse(json.stdout)).toMatchObject({ kind: "telegram", account: "dot_helper_bot" }); - }); - - it("channel add shows what the server said about a refused token, without the token", async () => { - const refused = await cli(["channel", "add", "telegram", "--dot", "fare-watch"], { stdin: "9:REFUSED-TOKEN-VALUE" }); - expect(refused.code).toBe(EXIT.failed); - expect(refused.stderr).toMatch(/refused the bot token.*\[400 invalid_credentials\]/); - expect(refused.stderr).not.toContain("REFUSED-TOKEN-VALUE"); - }); - - it("channel list prints a table of every Dot's channels, or one Dot's, and --json the data", async () => { - const all = await cli(["channel", "list"]); - expect(all.code).toBe(EXIT.ok); - expect(all.stdout.split("\n")[0]).toMatch(/^DOT\s+CHANNEL\s+STATUS\s+ACCOUNT\s+PEOPLE\s+NOTE/); - expect(all.stdout).toMatch(/fare-watch\s+telegram\s+connected\s+@dot_helper_bot\s+Ann \(@ann\)/); - const one = await cli(["channel", "list", "--dot", "fare-watch"]); - expect(one.stdout).toBe(all.stdout); - expect(JSON.parse((await cli(["channel", "list", "--json"])).stdout)).toEqual([{ dot: "fare-watch", channel }]); - expect((await cli(["channel", "list", "extra"])).code).toBe(EXIT.usage); - }); - - it("channel pair prints the link and the code, or only the code where a channel has no link", async () => { - const paired = await cli(["channel", "pair", "telegram", "--dot", "fare-watch"]); - expect(paired.code).toBe(EXIT.ok); - expect(paired.stdout).toBe( - "open this link on the device where you use telegram, then press Start (valid until 2026-10-02T08:10:00.000Z):\n https://t.me/dot_helper_bot?start=ABCD2345\nor send the bot: /start ABCD2345\n", - ); - expect((await cli(["channel", "pair", "whatsapp", "--dot", "fare-watch"])).stdout).toBe("pairing code WXYZ6789, valid until 2026-10-02T08:10:00.000Z\n"); - expect((await cli(["channel", "pair", "carrier-pigeon", "--dot", "fare-watch"])).stderr).toMatch(/unknown channel "carrier-pigeon": use telegram or whatsapp/); - expect((await cli(["channel", "pair", "telegram"])).stderr).toMatch(/missing --dot /); - }); - - it("channel remove unlinks, and an unknown subcommand is a usage error", async () => { - const removed = await cli(["channel", "remove", "telegram", "--dot", "fare-watch"]); - expect(removed.code).toBe(EXIT.ok); - expect(removed.stdout).toBe("unlinked telegram from Dot fare-watch: its token and paired people are deleted\n"); - expect(requests.at(-1)).toMatchObject({ method: "DELETE", path: "/api/dots/fare-watch/channels/telegram" }); - expect((await cli(["channel", "remove", "whatsapp", "--dot", "fare-watch"])).stdout).toBe("unlinked whatsapp from Dot fare-watch: the linked device's keys and paired people are deleted\n"); - expect((await cli(["channel", "remove", "telegram"])).code).toBe(EXIT.usage); - expect((await cli(["channel", "mute"])).stderr).toMatch(/unknown channel subcommand "mute": use add, link, list, pair or remove/); - expect((await cli(["channel"])).stderr).toMatch(/missing add\|link\|list\|pair\|remove/); - }); - - it("channel link whatsapp warns of the risk, shows each code as a QR for the terminal and ends when the number is linked", async () => { - linkFrames = [{ state: "waiting" }, { state: "code", code: "2@first-code" }, { state: "code", code: "2@second-code" }, { state: "linked", account: "15550001111" }]; - const linked = await cli(["channel", "link", "whatsapp", "--dot", "fare-watch"]); - expect(linked.code).toBe(EXIT.ok); - expect(requests.map((r) => `${r.method} ${r.path}`).slice(-2)).toEqual(["POST /api/dots/fare-watch/channels/whatsapp/link", "GET /api/dots/fare-watch/channels/whatsapp/qr"]); - expect(linked.stderr).toContain("can answer by banning the account"); - expect(linked.stderr).toContain("Linked devices"); - // Two codes, drawn with the block characters of a terminal QR, and never printed as text. - expect(linked.stdout.match(/█|▀|▄/g)?.length ?? 0).toBeGreaterThan(100); - expect(linked.stdout).not.toContain("2@first-code"); - expect(linked.stdout).toContain("linked WhatsApp number +15550001111 to Dot fare-watch\nnext: invisible-dots channel pair whatsapp --dot fare-watch\n"); - }); - - it("channel link whatsapp exits with the reason when the link fails, and says what the server said when WhatsApp is off", async () => { - linkFrames = [{ state: "waiting" }, { state: "failed", detail: "The link was not completed: the code expired. Start linking again." }]; - const failed = await cli(["channel", "link", "whatsapp", "--dot", "fare-watch"]); - expect(failed.code).toBe(EXIT.failed); - expect(failed.stderr).toContain("invisible-dots: The link was not completed: the code expired. Start linking again.\n"); - - whatsappOff = true; - const off = await cli(["channel", "link", "whatsapp", "--dot", "fare-watch"]); - expect(off.code).toBe(EXIT.failed); - expect(off.stderr).toContain("INVISIBLE_DOTS_WHATSAPP=1"); - }); - - it("channel link whatsapp --json prints the frames, code included, as lines of JSON", async () => { - linkFrames = [{ state: "code", code: "2@first-code" }, { state: "linked", account: "15550001111" }]; - const json = await cli(["channel", "link", "whatsapp", "--dot", "fare-watch", "--json"]); - expect(json.code).toBe(EXIT.ok); - expect(json.stdout.trim().split("\n").map((line) => JSON.parse(line))).toEqual(linkFrames); - }); - - it("channel link takes only whatsapp and a Dot, and channel add points WhatsApp to link", async () => { - expect((await cli(["channel", "link", "telegram", "--dot", "fare-watch"])).stderr).toMatch(/only whatsapp is linked that way/); - expect((await cli(["channel", "link", "whatsapp"])).stderr).toMatch(/missing --dot /); - expect((await cli(["channel", "link", "whatsapp", "extra", "--dot", "fare-watch"])).code).toBe(EXIT.usage); - expect((await cli(["channel", "add", "whatsapp", "--dot", "fare-watch"])).stderr).toContain('invisible-dots channel link whatsapp --dot '); - }); - - it("channel pair gives WhatsApp's link and words once the number is known, and list shows the number with a plus", async () => { - whatsappLinked = true; - const paired = await cli(["channel", "pair", "whatsapp", "--dot", "fare-watch"]); - expect(paired.stdout).toBe( - "open this link on the device where you use whatsapp, then press Send (valid until 2026-10-02T08:10:00.000Z):\n https://wa.me/15550001111?text=pair%20WXYZ6789\nor send the number: pair WXYZ6789\n", - ); - const list = await cli(["channel", "list", "--dot", "fare-watch"]); - expect(list.stdout).toMatch(/fare-watch\s+telegram\s+connected\s+@dot_helper_bot/); - expect(list.stdout).toMatch(/fare-watch\s+whatsapp\s+connected\s+\+15550001111/); - }); - - it("logs prints the tail without follow, and follows the stream until interrupted", async () => { - const tail = await cli(["logs", "fare-watch", "--tail", "2", "--no-follow"]); - expect(tail.code).toBe(EXIT.ok); - const lines = tail.stdout.trim().split("\n"); - expect(lines).toHaveLength(2); - expect(lines[0]).toMatch(/#4 agent\.state \{"state":"IDLE"\}$/); - // The log is read from its end, the newest two in one request, not paged from its start. - const reads = requests.filter((r) => r.path === `/api/dots/${dot.id}/events`); - expect(reads.map((r) => [r.query.get("order"), r.query.get("limit"), r.query.get("after")])).toEqual([["desc", "2", null]]); - - const controller = new AbortController(); - let output = ""; - const io: CliIo = { - stdout: (t) => { - output += t; - if (t.includes("task.completed")) controller.abort(); - }, - stderr: () => {}, - readStdin: async () => "", - readSecret: async () => "", - stdinIsTTY: false, - env: { INVISIBLE_DOTS_URL: base, INVISIBLE_DOTS_TOKEN: TOKEN }, - cwd: configDir, - signal: controller.signal, - }; - expect(await run(["logs", "fare-watch", "--tail", "1"], io)).toBe(EXIT.ok); - expect(output).toMatch(/#5 automation\.next_run/); - expect(output).toMatch(/#6 task\.completed/); - const stream = requests.find((r) => r.path === "/api/stream"); - expect(stream?.query.get("after")).toBe("5"); - expect(stream?.query.get("dot_id")).toBe(dot.id); - }); - - it("logs --tail 0 prints nothing old and follows from the newest event, which it reads alone", async () => { - const quiet = await cli(["logs", "fare-watch", "--tail", "0", "--no-follow"]); - expect(quiet.code).toBe(EXIT.ok); - expect(quiet.stdout).toBe(""); - expect(requests.filter((r) => r.path === `/api/dots/${dot.id}/events`).map((r) => r.query.get("limit"))).toEqual(["1"]); - }); - - it("logs --tail beyond a page reads the newest pages from the end, older one by one, and stops at the start of the log", async () => { - const saved = events; - events = Array.from({ length: 2500 }, (_, i) => ({ ...saved[0]!, id: i + 1 })); - try { - const tail = await cli(["logs", "fare-watch", "--tail", "2200", "--no-follow"]); - const lines = tail.stdout.trim().split("\n"); - expect(lines).toHaveLength(2200); - expect(lines[0]).toContain("#301 "); - expect(lines.at(-1)).toContain("#2500 "); - const reads = requests.filter((r) => r.path === `/api/dots/${dot.id}/events`); - expect(reads.map((r) => [r.query.get("limit"), r.query.get("before")])).toEqual([ - ["1000", null], - ["1000", "1501"], - ["200", "501"], - ]); - requests.length = 0; - expect((await cli(["logs", "fare-watch", "--tail", "9000", "--no-follow"])).stdout.trim().split("\n")).toHaveLength(2500); - expect(requests.filter((r) => r.path === `/api/dots/${dot.id}/events`)).toHaveLength(3); - } finally { - events = saved; - } - }); - - it("cancel cancels a task, and shows the server's 409 for one that has finished", async () => { - const cancelled = await cli(["cancel", "task_02"]); - expect(cancelled.code).toBe(EXIT.ok); - expect(cancelled.stdout).toBe("task task_02 CANCELLED\n"); - expect(requests.at(-1)).toMatchObject({ method: "POST", path: "/api/tasks/task_02/cancel" }); - const finished = await cli(["cancel", "task_01"]); - expect(finished.code).toBe(EXIT.failed); - expect(finished.stderr).toMatch(/already COMPLETED \[409 task_finished\]/); - expect((await cli(["cancel"])).stderr).toMatch(/missing /); - }); - - it("delete refuses without --yes and says what it destroys, and deletes with it", async () => { - const refused = await cli(["delete", "fare-watch"]); - expect(refused.code).toBe(EXIT.usage); - expect(refused.stderr).toContain("cannot be undone"); - expect(refused.stderr).toContain("invisible-dots delete fare-watch --yes"); - expect(requests).toHaveLength(0); - const deleted = await cli(["delete", "fare-watch", "--yes"]); - expect(deleted.code).toBe(EXIT.ok); - expect(requests.at(-1)).toMatchObject({ method: "DELETE", path: `/api/dots/${dot.id}` }); - expect(deleted.stdout).toBe(`deleting Dot fare-watch; what it did stays in its log: invisible-dots logs ${dot.id}\n`); - }); - - it("update sends the YAML with the config version it read, so a change made meanwhile is a 409 and nothing is saved", async () => { - await writeFile(join(configDir, "changed.yaml"), SAMPLE_DOT); - configVersion = 4; - const updated = await cli(["update", "fare-watch", "changed.yaml"]); - expect(updated.code).toBe(EXIT.ok); - expect(updated.stdout).toBe("updated Dot fare-watch (config version 5)\n"); - expect(requests.at(-1)).toMatchObject({ method: "PATCH", path: `/api/dots/${dot.id}`, body: { config: SAMPLE_DOT, expected_config_version: 4 } }); - - // The web client saved between this command's read and its write. - savedMeanwhile = true; - const raced = await cli(["update", "fare-watch", "changed.yaml"]); - expect(raced.code).toBe(EXIT.failed); - expect(raced.stderr).toMatch(/\[409 dot_changed\]/); - expect((await cli(["update", "fare-watch", "missing.yaml"])).code).toBe(EXIT.usage); - expect((await cli(["update", "fare-watch"])).stderr).toMatch(/missing /); - }); -}); - -describe("host commands", () => { - function fakeHost() { - const calls: string[] = []; - const host: HostCommands = { - doctor: async (options, io) => { - calls.push(`doctor json=${options.json}`); - io.stdout("all 10 checks ok\n"); - return EXIT.ok; - }, - setup: async () => { - calls.push("setup"); - return EXIT.restart; - }, - setupAll: async () => { - calls.push("setup --all"); - return EXIT.ok; - }, - imageBuild: async () => { - calls.push("image build"); - return EXIT.failed; - }, - server: async (_io, options) => { - calls.push(`server web=${options.web}`); - return EXIT.ok; - }, - }; - return { host, calls }; - } - - it("routes setup, doctor, image build and server, and returns their exit codes", async () => { - const { host, calls } = fakeHost(); - expect((await cli(["doctor"], { host })).stdout).toBe("all 10 checks ok\n"); - expect((await cli(["doctor", "--json"], { host })).code).toBe(EXIT.ok); - expect((await cli(["setup"], { host })).code).toBe(EXIT.restart); - expect((await cli(["setup", "--all"], { host })).code).toBe(EXIT.ok); - expect((await cli(["image", "build"], { host })).code).toBe(EXIT.failed); - expect((await cli(["server"], { host })).code).toBe(EXIT.ok); - expect((await cli(["server", "--no-web"], { host })).code).toBe(EXIT.ok); - expect(calls).toEqual(["doctor json=false", "doctor json=true", "setup", "setup --all", "image build", "server web=true", "server web=false"]); - // None of them needs the API or its token. - expect(requests).toHaveLength(0); - }); - - it("refuses stray arguments with exit 2, and reports a host command that throws with exit 1", async () => { - const { host, calls } = fakeHost(); - expect((await cli(["image"], { host })).stderr).toMatch(/missing build/); - expect((await cli(["image", "pull"], { host })).stderr).toMatch(/unknown image subcommand "pull"/); - expect((await cli(["image", "build", "now"], { host })).code).toBe(EXIT.usage); - expect((await cli(["setup", "x"], { host })).code).toBe(EXIT.usage); - expect((await cli(["setup", "--all", "x"], { host })).code).toBe(EXIT.usage); - expect((await cli(["doctor", "all"], { host })).stderr).toMatch(/doctor takes no arguments/); - expect(calls).toEqual([]); - - host.server = async () => { - throw new Error("another invisible-dots server (pid 42) is already running"); - }; - const failed = await cli(["server"], { host }); - expect(failed.code).toBe(EXIT.failed); - expect(failed.stderr).toBe("invisible-dots: another invisible-dots server (pid 42) is already running\n"); - }); - - it("lists both channels where a command takes either", async () => { - const help = (await cli(["--help"])).stdout; - expect(help).toContain("invisible-dots channel pair telegram|whatsapp --dot "); - expect(help).toContain("invisible-dots channel remove telegram|whatsapp --dot "); - }); - - it("lists setup --all in the help, first among the host commands", async () => { - const help = (await cli(["--help"])).stdout; - expect(help).toMatch(/invisible-dots setup --all .*everything in one run/); - expect(help.indexOf("setup --all")).toBeLessThan(help.indexOf("invisible-dots setup ")); - }); - - it("asks the process to stop on Ctrl-C only for logs, image build and setup --all", () => { - expect(interruptIsAsked(["logs", "fare-watch"])).toBe(true); - expect(interruptIsAsked(["image", "build"])).toBe(true); - expect(interruptIsAsked(["setup", "--all"])).toBe(true); - expect(interruptIsAsked(["--all", "setup"])).toBe(true); - expect(interruptIsAsked(["setup"])).toBe(false); - expect(interruptIsAsked(["server"])).toBe(false); - expect(interruptIsAsked(["approvals", "--all"])).toBe(false); - }); - - it("finds the command word for main.ts wherever the flags are", () => { - expect(commandOf(["--json", "doctor"])).toBe("doctor"); - expect(commandOf(["logs", "fare-watch", "--tail", "3"])).toBe("logs"); - expect(commandOf(["--help"])).toBeUndefined(); - }); -}); diff --git a/apps/cli/test/doctor.test.ts b/apps/cli/test/doctor.test.ts deleted file mode 100644 index a35a8159..00000000 --- a/apps/cli/test/doctor.test.ts +++ /dev/null @@ -1,48 +0,0 @@ -import type { DoctorCheck } from "@invisible-dots/shared"; -import { describe, expect, it } from "vitest"; -import { doctorAnswer } from "@invisible-dots/vm-manager"; -import { FOUND, healthyDoctor, QEMU_IMG } from "../../vm-manager/test/doctor-fakes.js"; -import { doctorCommand } from "../src/doctor/command.js"; -import { renderReport } from "../src/doctor/render.js"; -import { EXIT } from "../src/exit.js"; - -const find = (results: DoctorCheck[], id: DoctorCheck["id"]) => results.find((r) => r.id === id)!; - -describe("doctor report", () => { - const sample: DoctorCheck[] = [ - { id: "node", label: "Node", status: "ok", detail: "24.19.0" }, - { id: "qemu", label: "QEMU", status: "missing", detail: "qemu-system-x86_64 not found", fix: "invisible-dots setup" }, - { id: "disk", label: "data directory", status: "failed", detail: "1.0 GiB free at /data", fix: "free some space" }, - { id: "openrouter", label: "OpenRouter key", status: "ok", detail: "stored" }, - ]; - - it("prints one aligned line per check, a fix line under each that is not ok, and a count", () => { - expect(renderReport(sample)).toBe( - [ - "ok Node 24.19.0", - "missing QEMU qemu-system-x86_64 not found", - " fix: invisible-dots setup", - "failed data directory 1.0 GiB free at /data", - " fix: free some space", - "ok OpenRouter key stored", - "4 checks: 2 ok, 1 missing, 1 failed", - "", - ].join("\n"), - ); - expect(renderReport([sample[0]!, sample[3]!]).endsWith("all 2 checks ok\n")).toBe(true); - }); - - it("exits 0 only when every check is ok, and prints JSON with --json", async () => { - let text = ""; - expect(await doctorCommand(healthyDoctor().deps, { json: false }, (t) => (text += t))).toBe(EXIT.ok); - expect(text.endsWith("all 9 checks ok\n")).toBe(true); - - let json = ""; - const missing = healthyDoctor({ findQemu: async () => ({ ...FOUND, img: undefined }) }).deps; - expect(await doctorCommand(missing, { json: true }, (t) => (json += t))).toBe(EXIT.failed); - const parsed = JSON.parse(json) as { ok: boolean; checks: DoctorCheck[] }; - expect(parsed.ok).toBe(false); - expect(find(parsed.checks, "qemu-img")).toMatchObject({ status: "missing", fix: "invisible-dots setup" }); - expect(QEMU_IMG).toBe(FOUND.img); - }); -}); diff --git a/apps/cli/test/install.test.ts b/apps/cli/test/install.test.ts deleted file mode 100644 index ec2a0034..00000000 --- a/apps/cli/test/install.test.ts +++ /dev/null @@ -1,234 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { - installHostPrerequisites, - interpretElevatedResult, - packageManagerFor, - parseOsRelease, - setupRefusal, - type InstallDeps, -} from "../src/setup/install.js"; -import { encodeCommand, powershellArgs } from "@invisible-dots/vm-manager"; -import { elevatedSetupScript, elevationLauncherScript } from "../src/setup/powershell.js"; -import type { WindowsQemuPin } from "../src/setup/qemu-pin.js"; -import { fakeRunner, type Answer } from "../../vm-manager/test/doctor-fakes.js"; - -const PS = "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe"; -const PIN: WindowsQemuPin = { - version: "10.0.0", - url: "https://qemu.weilnetz.de/w64/2025/qemu-w64-setup-20250422.exe", - sha256: "b".repeat(64), - silentArgs: ["/S"], -}; -const UBUNTU = 'PRETTY_NAME="Ubuntu 24.04.3 LTS"\nNAME="Ubuntu"\nID=ubuntu\nID_LIKE=debian\n'; - -function errno(code: string): NodeJS.ErrnoException { - return Object.assign(new Error(`${code}: /dev/kvm`), { code }); -} - -interface HostOptions { - platform: NodeJS.Platform; - respond?: (command: string, args: readonly string[]) => Answer; - kvm?: string; - osRelease?: string; - isRoot?: boolean; - /** What the elevated session leaves in its result file; undefined writes nothing. */ - elevatedResult?: string; -} - -function fakeHost(options: HostOptions) { - const logs: string[] = []; - const downloads: { pin: WindowsQemuPin; dest: string }[] = []; - const removed: string[] = []; - const work = "C:\\Users\\someone\\AppData\\Local\\Temp\\invisible-dots-setup-x1"; - const elevatedDir = "C:\\ProgramData\\invisible-dots-setup-0123abcd"; - const runner = fakeRunner((command, args) => options.respond?.(command, args) ?? {}); - const deps: InstallDeps = { - platform: options.platform, - env: { SystemRoot: "C:\\Windows", ProgramData: "C:\\ProgramData", ProgramW6432: "C:\\Program Files" }, - run: runner.run, - openReadWrite: async () => { - if (options.kvm) throw errno(options.kvm); - }, - log: (line) => logs.push(line), - readText: async (path) => { - if (path === "/etc/os-release" && options.osRelease !== undefined) return options.osRelease; - if (path === `${elevatedDir}\\result.json` && options.elevatedResult !== undefined) return options.elevatedResult; - throw errno("ENOENT"); - }, - isRoot: options.isRoot ?? false, - windowsQemuPin: () => PIN, - makeTempDir: async () => work, - removeDir: async (path) => { - removed.push(path); - }, - download: async (pin, dest) => { - downloads.push({ pin, dest }); - }, - userSid: async () => "S-1-5-21-1-2-3-1001", - uniqueName: () => "0123abcd", - }; - return { deps, calls: runner.calls, logs, downloads, removed, work, elevatedDir }; -} - -describe("an unsupported host", () => { - it("is refused by setup's install step", async () => { - expect((await installHostPrerequisites({ installQemu: true, enableAccelerator: false }, fakeHost({ platform: "darwin" }).deps)).kind).toBe("failed"); - }); -}); - -describe("Linux install", () => { - it("runs sudo apt-get install on apt systems with the terminal handed over, then checks /dev/kvm", async () => { - const host = fakeHost({ platform: "linux", osRelease: UBUNTU }); - const outcome = await installHostPrerequisites({ installQemu: true, enableAccelerator: false }, host.deps); - expect(outcome).toEqual({ kind: "done", lines: [] }); - expect(host.calls).toEqual([{ command: "sudo", args: ["apt-get", "install", "-y", "qemu-system-x86", "qemu-utils"], options: { inheritStdio: true } }]); - expect(host.logs).toEqual(["$ sudo apt-get install -y qemu-system-x86 qemu-utils"]); - }); - - it("reports a failed apt-get", async () => { - const host = fakeHost({ platform: "linux", osRelease: "ID=debian\n", respond: () => ({ code: 100 }) }); - const outcome = await installHostPrerequisites({ installQemu: true, enableAccelerator: false }, host.deps); - expect(host.calls[0]).toMatchObject({ command: "sudo", args: ["apt-get", "install", "-y", "qemu-system-x86", "qemu-utils"] }); - expect(outcome).toEqual({ kind: "failed", lines: ["sudo apt-get install -y qemu-system-x86 qemu-utils failed (exit code 100)"] }); - }); - - it("refuses root on Linux only: Windows setup never runs as an administrator", () => { - expect(setupRefusal({ platform: "linux", isRoot: true })).toMatch(/not as root/); - expect(setupRefusal({ platform: "linux", isRoot: false })).toBeNull(); - expect(setupRefusal({ platform: "win32", isRoot: false })).toBeNull(); - }); - - it("only prints the dnf or pacman line elsewhere, and still checks /dev/kvm", async () => { - const host = fakeHost({ platform: "linux", osRelease: 'ID=fedora\nPRETTY_NAME="Fedora Linux 42"\n', kvm: "EACCES" }); - const outcome = await installHostPrerequisites({ installQemu: true, enableAccelerator: false }, host.deps); - expect(host.calls).toEqual([]); - expect(outcome).toEqual({ - kind: "manual", - lines: [ - "QEMU is not installed by setup on Fedora Linux 42; run:", - " sudo dnf install -y qemu-system-x86 qemu-img", - "/dev/kvm exists but this user cannot open it read-write; fix:", - " sudo usermod -aG kvm $USER, then log out and in again (a new login is needed for the group to apply)", - ], - }); - }); - - it("prints usermod and the new login when only /dev/kvm is not accessible", async () => { - const outcome = await installHostPrerequisites({ installQemu: false, enableAccelerator: true }, fakeHost({ platform: "linux", kvm: "EACCES" }).deps); - expect(outcome.kind).toBe("manual"); - expect(outcome.lines.join("\n")).toContain("sudo usermod -aG kvm $USER"); - expect(outcome.lines.join("\n")).toContain("a new login is needed"); - }); - - it("reads the package manager from ID and ID_LIKE", () => { - expect(parseOsRelease(UBUNTU)).toEqual({ id: "ubuntu", idLike: ["debian"], prettyName: "Ubuntu 24.04.3 LTS" }); - expect(packageManagerFor(parseOsRelease("ID=linuxmint\nID_LIKE=\"ubuntu debian\"\n"))).toEqual({ kind: "apt" }); - expect(packageManagerFor(parseOsRelease("ID=rocky\nID_LIKE=\"rhel centos fedora\"\n"))).toMatchObject({ line: "sudo dnf install -y qemu-system-x86 qemu-img" }); - expect(packageManagerFor(parseOsRelease("ID=endeavouros\nID_LIKE=arch\n"))).toMatchObject({ line: "sudo pacman -S --needed qemu-system-x86 qemu-img" }); - expect(packageManagerFor(parseOsRelease("ID=gentoo\n"))).toMatchObject({ kind: "print" }); - expect(packageManagerFor(parseOsRelease(""))).toMatchObject({ kind: "print" }); - }); -}); - -describe("Windows install", () => { - it("downloads and verifies the pinned installer, then runs dism and the installer in ONE elevated session; 3010 means restart", async () => { - const host = fakeHost({ platform: "win32", elevatedResult: '\uFEFF{"dism_exit_code":3010,"installer_exit_code":0,"error":null}' }); - const outcome = await installHostPrerequisites({ installQemu: true, enableAccelerator: true }, host.deps); - - const installer = `${host.work}\\qemu-w64-setup-20250422.exe`; - expect(host.downloads).toEqual([{ pin: PIN, dest: installer }]); - const elevated = elevatedSetupScript({ - enableHypervisorPlatform: true, - installer: { path: installer, sha256: PIN.sha256, silentArgs: ["/S"] }, - workDir: host.elevatedDir, - userSid: "S-1-5-21-1-2-3-1001", - }); - expect(host.calls).toEqual([{ command: PS, args: powershellArgs(elevationLauncherScript(PS, elevated)), options: { timeoutMs: 3_600_000 } }]); - const launcher = Buffer.from(host.calls[0]!.args.at(-1)!, "base64").toString("utf16le"); - expect(launcher).toContain(`-Verb RunAs -Wait -PassThru`); - expect(launcher).toContain(encodeCommand(elevated)); - - expect(outcome).toEqual({ - kind: "restart", - lines: [ - "QEMU installed", - "the HypervisorPlatform feature is enabled; Windows must restart before it works", - "restart Windows, then run: invisible-dots doctor", - ], - }); - // The elevated session's own directory goes too, once its result was read. - expect(host.removed).toEqual([host.elevatedDir, host.work]); - expect(host.logs).toEqual([ - `downloading the official QEMU 10.0.0 installer: ${PIN.url}`, - `verified its SHA-256 (${PIN.sha256})`, - "asking for administrator rights once, to enable the HypervisorPlatform feature (dism) and run the QEMU installer silently", - ]); - }); - - it("enables only the feature without downloading anything when QEMU is already installed", async () => { - const host = fakeHost({ platform: "win32", elevatedResult: '{"dism_exit_code":0,"installer_exit_code":null,"error":null}' }); - const outcome = await installHostPrerequisites({ installQemu: false, enableAccelerator: true }, host.deps); - expect(host.downloads).toEqual([]); - const launcher = Buffer.from(host.calls[0]!.args.at(-1)!, "base64").toString("utf16le"); - const elevated = Buffer.from(/'-EncodedCommand','([^']+)'/.exec(launcher)![1]!, "base64").toString("utf16le"); - expect(elevated).toContain("/featurename:HypervisorPlatform"); - expect(elevated).not.toContain("$installer"); - expect(outcome).toEqual({ kind: "done", lines: ["the HypervisorPlatform feature is enabled"] }); - }); - - it("installs only QEMU when the feature is already on", async () => { - const host = fakeHost({ platform: "win32", elevatedResult: '{"dism_exit_code":null,"installer_exit_code":0,"error":null}' }); - expect(await installHostPrerequisites({ installQemu: true, enableAccelerator: false }, host.deps)).toEqual({ kind: "done", lines: ["QEMU installed"] }); - expect(host.downloads).toHaveLength(1); - }); - - it("reports a declined UAC prompt, a failure inside the elevated session, and a download that does not verify", async () => { - const declined = fakeHost({ platform: "win32", respond: () => ({ code: 1, stderr: "This operation was canceled by the user.\r\n" }) }); - expect(await installHostPrerequisites({ installQemu: false, enableAccelerator: true }, declined.deps)).toEqual({ - kind: "failed", - lines: ["administrator rights were not granted (the prompt was declined); nothing was changed"], - }); - expect(declined.removed).toEqual([declined.elevatedDir, declined.work]); - - const inside = fakeHost({ platform: "win32", elevatedResult: '{"dism_exit_code":50,"installer_exit_code":null,"error":"dism exited with code 50"}' }); - expect(await installHostPrerequisites({ installQemu: false, enableAccelerator: true }, inside.deps)).toEqual({ - kind: "failed", - lines: ["the elevated step failed: dism exited with code 50"], - }); - - const tampered = fakeHost({ platform: "win32" }); - tampered.deps.download = async () => { - throw new Error("https://example.invalid/q.exe hashes to 00, but the pin is bb"); - }; - await expect(installHostPrerequisites({ installQemu: true, enableAccelerator: true }, tampered.deps)).rejects.toThrow(/but the pin is/); - expect(tampered.calls).toEqual([]); - expect(tampered.removed).toEqual([tampered.elevatedDir, tampered.work]); - }); - - it("interprets the result file", () => { - expect(interpretElevatedResult("not json").kind).toBe("failed"); - expect(interpretElevatedResult('{"dism_exit_code":3010,"installer_exit_code":null,"error":null}').kind).toBe("restart"); - }); - - it("names the directory to set when the installer reused an earlier one invisible-dots does not search", () => { - const env = { ProgramW6432: "C:\\Program Files" }; - const installedIn = (dir: string) => JSON.stringify({ dism_exit_code: null, installer_exit_code: 0, install_dir: dir, error: null }); - // The official directory (with the trailing separator the registry value has) needs nothing more. - expect(interpretElevatedResult(installedIn("C:\\Program Files\\qemu\\"), env)).toEqual({ - kind: "done", - lines: ["QEMU installed"], - }); - expect(interpretElevatedResult(installedIn("D:\\Tools\\qemu"), env)).toEqual({ - kind: "done", - lines: ["QEMU installed", "it went to D:\\Tools\\qemu, where an earlier QEMU was installed: set INVISIBLE_DOTS_QEMU_DIR=D:\\Tools\\qemu so invisible-dots uses it"], - }); - }); - - it("says the elevated step ran nothing when it could not make its own directory safely", async () => { - const host = fakeHost({ platform: "win32", respond: () => ({ code: 2 }) }); - expect(await installHostPrerequisites({ installQemu: false, enableAccelerator: true }, host.deps)).toEqual({ - kind: "failed", - lines: [`the elevated step could not create its own directory ${host.elevatedDir} for administrators only, so it ran nothing; run setup again`], - }); - }); -}); diff --git a/apps/cli/test/powershell.test.ts b/apps/cli/test/powershell.test.ts deleted file mode 100644 index f0d3abb0..00000000 --- a/apps/cli/test/powershell.test.ts +++ /dev/null @@ -1,162 +0,0 @@ -import { existsSync } from "node:fs"; -import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { encodeCommand, HYPERVISOR_PLATFORM_STATE_SCRIPT, powershellArgs, powershellPath, runProcess } from "@invisible-dots/vm-manager"; -import { elevatedSetupScript, elevationLauncherScript, parseElevatedResult, psQuote } from "../src/setup/powershell.js"; - -const SHA = "a".repeat(64); - -const WORK = "C:\\ProgramData\\invisible-dots-setup-0123abcd"; -const SID = "S-1-5-21-1-2-3-1001"; - -describe("generated PowerShell", () => { - it("runs dism and a verified COPY of the installer in one elevated session, from a directory only administrators can change", () => { - const script = elevatedSetupScript({ - enableHypervisorPlatform: true, - installer: { path: "C:\\Users\\O'Brien\\AppData\\Local\\Temp\\idots-1\\qemu-w64-setup-20250422.exe", sha256: SHA.toUpperCase(), silentArgs: ["/S"] }, - workDir: WORK, - userSid: SID, - }); - expect(script.split("\r\n")).toEqual([ - "# invisible-dots setup: the one step that needs administrator rights (architecture section 11.2).", - "$ErrorActionPreference = 'Stop'", - "$result = [ordered]@{ dism_exit_code = $null; installer_exit_code = $null; install_dir = $null; error = $null }", - "$work = 'C:\\ProgramData\\invisible-dots-setup-0123abcd'", - "$user = New-Object System.Security.Principal.SecurityIdentifier 'S-1-5-21-1-2-3-1001'", - "function New-Rule($sid, $rights) { New-Object System.Security.AccessControl.FileSystemAccessRule($sid, $rights, 'ContainerInherit,ObjectInherit', 'None', 'Allow') }", - // Under the real ProgramData (not one an environment variable names), new, protected and empty. - "if ((Split-Path -Parent $work) -ne [Environment]::GetFolderPath('CommonApplicationData')) { exit 2 }", - "$acl = New-Object System.Security.AccessControl.DirectorySecurity", - "$acl.SetAccessRuleProtection($true, $false)", - "$acl.AddAccessRule((New-Rule (New-Object System.Security.Principal.SecurityIdentifier 'S-1-5-32-544') 'FullControl'))", - "$acl.AddAccessRule((New-Rule (New-Object System.Security.Principal.SecurityIdentifier 'S-1-5-18') 'FullControl'))", - "$acl.AddAccessRule((New-Rule $user 'ReadAndExecute'))", - "if (Test-Path -LiteralPath $work) { exit 2 }", - "[void][System.IO.Directory]::CreateDirectory($work, $acl)", - "if (-not ((Get-Acl -LiteralPath $work).AreAccessRulesProtected) -or (@(Get-ChildItem -LiteralPath $work -Force).Count -ne 0)) { exit 2 }", - "try {", - // dism through the system directory Windows reports, never $env:SystemRoot. - " $dism = Join-Path ([Environment]::SystemDirectory) 'dism.exe'", - " & $dism /online /enable-feature /featurename:HypervisorPlatform /all /norestart", - " $result.dism_exit_code = $LASTEXITCODE", - ' if ($LASTEXITCODE -ne 0 -and $LASTEXITCODE -ne 3010) { throw "dism exited with code $LASTEXITCODE" }', - // The copy is what is hashed and what runs, in the directory nobody else can write. - " $installer = Join-Path $work 'qemu-setup.exe'", - " Copy-Item -LiteralPath 'C:\\Users\\O''Brien\\AppData\\Local\\Temp\\idots-1\\qemu-w64-setup-20250422.exe' -Destination $installer", - " $hash = (Get-FileHash -LiteralPath $installer -Algorithm SHA256).Hash.ToLowerInvariant()", - ` if ($hash -ne '${SHA}') { throw "the QEMU installer is not the one that was verified (its SHA-256 is $hash)" }`, - " $setup = Start-Process -FilePath $installer -ArgumentList '/S' -WorkingDirectory $work -Wait -PassThru", - " $result.installer_exit_code = $setup.ExitCode", - ' if ($setup.ExitCode -ne 0) { throw "the QEMU installer exited with code $($setup.ExitCode)" }', - " $result.install_dir = (Get-ItemProperty -LiteralPath 'HKLM:\\Software\\qemu64' -ErrorAction SilentlyContinue).Install_Dir", - "} catch {", - " $result.error = $_.Exception.Message", - "}", - "Remove-Item -LiteralPath (Join-Path $work 'qemu-setup.exe') -Force -ErrorAction SilentlyContinue", - // The result goes into the same private directory, never into one the user can redirect. - "[System.IO.File]::WriteAllText((Join-Path $work 'result.json'), ($result | ConvertTo-Json -Compress))", - "# Nothing in it runs any more: setup, as the normal user, reads the result and removes the directory.", - "$done = Get-Acl -LiteralPath $work", - "$done.AddAccessRule((New-Rule $user 'FullControl'))", - "Set-Acl -LiteralPath $work -AclObject $done", - "", - ]); - expect(script).not.toContain("$env:"); - }); - - it("leaves out the step that is not needed", () => { - const onlyFeature = elevatedSetupScript({ enableHypervisorPlatform: true, workDir: WORK, userSid: SID }); - expect(onlyFeature).toContain("dism.exe"); - expect(onlyFeature).not.toContain("$installer ="); - const onlyQemu = elevatedSetupScript({ enableHypervisorPlatform: false, installer: { path: "C:\\t\\q.exe", sha256: SHA, silentArgs: ["/S"] }, workDir: WORK, userSid: SID }); - expect(onlyQemu).not.toContain("dism.exe"); - expect(onlyQemu).toContain("-ArgumentList '/S'"); - }); - - it("asks for elevation with one Start-Process -Verb RunAs -Wait and passes the script encoded", () => { - const elevated = "Write-Output 'hi'"; - const launcher = elevationLauncherScript("C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe", elevated); - expect(launcher.split("\r\n")).toEqual([ - "$ErrorActionPreference = 'Stop'", - "try {", - ` $elevated = Start-Process -FilePath 'C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe' -Verb RunAs -Wait -PassThru -ArgumentList '-NoProfile','-NonInteractive','-ExecutionPolicy','Bypass','-EncodedCommand','${encodeCommand(elevated)}'`, - " exit $elevated.ExitCode", - "} catch {", - " [Console]::Error.WriteLine($_.Exception.Message)", - " exit 1", - "}", - "", - ]); - expect(launcher.match(/Start-Process/g)).toHaveLength(1); - expect(Buffer.from(encodeCommand(elevated), "base64").toString("utf16le")).toBe(elevated); - }); - - it("quotes every kind of single quote PowerShell recognises", () => { - expect(psQuote("plain")).toBe("'plain'"); - expect(psQuote("it's")).toBe("'it''s'"); - expect(psQuote("a\u2019b\u2018c")).toBe("'a\u2019\u2019b\u2018\u2018c'"); - }); - - it("reads the result file, with or without a byte order mark", () => { - expect(parseElevatedResult('\uFEFF{"dism_exit_code":3010,"installer_exit_code":0,"error":null}')).toEqual({ - dism_exit_code: 3010, - installer_exit_code: 0, - install_dir: null, - error: null, - }); - expect(parseElevatedResult('{"dism_exit_code":null,"installer_exit_code":0,"install_dir":"D:\\\\q","error":null}').install_dir).toBe("D:\\q"); - expect(parseElevatedResult('{"dism_exit_code":null,"installer_exit_code":null,"error":"boom"}').error).toBe("boom"); - expect(() => parseElevatedResult('{"dism_exit_code":"x"}')).toThrow(/not an exit code/); - }); -}); - -/* - * The same scripts run through the real Windows PowerShell where it exists: - * the parser must accept every generated script, and the non-elevated parts - * (quoting, the hash check, the result file) must behave as the code above - * assumes. Nothing here elevates or changes the system. - */ -const windowsPowerShell = powershellPath(process.env); -const hasPowerShell = existsSync(windowsPowerShell); - -// Each test starts Windows PowerShell a few times; under a parallel suite one start can take seconds. -describe.skipIf(!hasPowerShell)("generated PowerShell on a real Windows PowerShell", { timeout: 120_000 }, () => { - let dir: string; - beforeAll(async () => { - // A quote and a space in the path: the cases quoting has to survive. - dir = await mkdtemp(join(tmpdir(), "idots ps'q-")); - }); - afterAll(async () => { - await rm(dir, { recursive: true, force: true }); - }); - - const ps = (script: string) => runProcess(windowsPowerShell, powershellArgs(script), { timeoutMs: 60_000 }); - - it("parses every generated script without errors", async () => { - const elevated = elevatedSetupScript({ enableHypervisorPlatform: true, installer: { path: join(dir, "q.exe"), sha256: SHA, silentArgs: ["/S"] }, workDir: WORK, userSid: SID }); - for (const script of [elevated, elevationLauncherScript(windowsPowerShell, elevated), HYPERVISOR_PLATFORM_STATE_SCRIPT]) { - const check = `$errors = $null; [void][System.Management.Automation.Language.Parser]::ParseInput(${psQuote(script)}, [ref]$null, [ref]$errors); $errors.Count`; - const result = await ps(check); - expect(result.stderr).toBe(""); - expect(result.stdout.trim()).toBe("0"); - } - }); - - it("runs nothing when its directory would not be directly under the real ProgramData", async () => { - // A directory the normal user chose (here a temporary one): the session stops before it creates or runs anything. - const installer = join(dir, "qemu-setup.exe"); - await writeFile(installer, "not the verified bytes"); - const workDir = join(dir, "elevated"); - const run = await ps(elevatedSetupScript({ enableHypervisorPlatform: false, installer: { path: installer, sha256: SHA, silentArgs: ["/S"] }, workDir, userSid: SID })); - expect(run.code).toBe(2); - expect(existsSync(workDir)).toBe(false); - }); - - it("reads the HypervisorPlatform state without administrator rights", async () => { - const result = await ps(HYPERVISOR_PLATFORM_STATE_SCRIPT); - expect(result.code).toBe(0); - expect(result.stdout.trim()).toMatch(/^[1-4]?$/); - }); -}); diff --git a/apps/cli/test/qemu-pin.test.ts b/apps/cli/test/qemu-pin.test.ts deleted file mode 100644 index 88da4579..00000000 --- a/apps/cli/test/qemu-pin.test.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { readFile } from "node:fs/promises"; -import { describe, expect, it } from "vitest"; -import { installerFileName, parseWindowsQemuPin } from "../src/setup/qemu-pin.js"; - -describe("Windows QEMU pin", () => { - const good = { version: "10.0.0", url: "https://qemu.weilnetz.de/w64/2025/qemu-w64-setup-20250422.exe", sha256: "AB".repeat(32), silent_args: ["/S"] }; - - it("accepts a version, an https URL and a SHA-256, and normalises the hash", () => { - expect(parseWindowsQemuPin(good)).toEqual({ version: good.version, url: good.url, sha256: "ab".repeat(32), silentArgs: ["/S"] }); - }); - - it("accepts the pin file in this repository", async () => { - const pin = parseWindowsQemuPin(JSON.parse(await readFile(new URL("../../../virtualization/qemu/windows.json", import.meta.url), "utf8"))); - expect(pin.url.startsWith("https://")).toBe(true); - expect(installerFileName(pin)).toMatch(/\.exe$/); - }); - - it("refuses a pin without its silent switch", () => { - expect(() => parseWindowsQemuPin({ ...good, silent_args: [] })).toThrow(/"silent_args" must be a non-empty list/); - expect(() => parseWindowsQemuPin({ ...good, silent_args: undefined })).toThrow(/silent_args/); - }); - - it("refuses anything that would make setup download the wrong bytes", () => { - expect(() => parseWindowsQemuPin(null)).toThrow(/expected a JSON object/); - expect(() => parseWindowsQemuPin({ ...good, url: "http://qemu.weilnetz.de/x.exe" })).toThrow(/must use https/); - expect(() => parseWindowsQemuPin({ ...good, sha256: "abc" })).toThrow(/64 hexadecimal/); - expect(() => parseWindowsQemuPin({ ...good, version: "" })).toThrow(/"version" must be a non-empty string/); - expect(() => parseWindowsQemuPin({ ...good, version: "latest" })).toThrow(/major\.minor/); - expect(() => parseWindowsQemuPin({ ...good, url: "not a url" })).toThrow(/not a URL/); - }); - - it("saves the installer under its own plain name, or a fixed one", () => { - expect(installerFileName(parseWindowsQemuPin(good))).toBe("qemu-w64-setup-20250422.exe"); - expect(installerFileName(parseWindowsQemuPin({ ...good, url: "https://example.org/download?id=7" }))).toBe("qemu-10.0.0-setup.exe"); - expect(installerFileName(parseWindowsQemuPin({ ...good, url: "https://example.org/a%27b.exe" }))).toBe("qemu-10.0.0-setup.exe"); - }); -}); diff --git a/apps/cli/test/secret-input.test.ts b/apps/cli/test/secret-input.test.ts deleted file mode 100644 index 235e1385..00000000 --- a/apps/cli/test/secret-input.test.ts +++ /dev/null @@ -1,85 +0,0 @@ -/** - * The line a person pastes at a prompt (an OpenRouter key, a bot token) is read with the echo off. The tests play the - * terminal: a stream that says it is a TTY, takes raw mode and records what is written to the screen. - */ -import { createInterface } from "node:readline"; -import { PassThrough, Writable } from "node:stream"; -import { describe, expect, it } from "vitest"; -import { readSecretLine, SecretInputCancelled, type TerminalInput } from "../src/secret-input.js"; - -function terminal() { - const input = new PassThrough() as PassThrough & TerminalInput & { raw: boolean[] }; - input.isTTY = true; - input.raw = []; - input.setRawMode = (mode: boolean) => void input.raw.push(mode); - const screen: string[] = []; - const output = { write: (text: string) => (screen.push(String(text)), true) }; - return { input, output, screen: () => screen.join("") }; -} - -const KEY = "sk-or-v1-0123456789abcdef0123456789abcdef"; - -describe("a secret typed at a prompt", () => { - it("would show on the screen if the line were read by a readline that writes there (so the checks below can fail)", async () => { - const t = terminal(); - const echoed: string[] = []; - const screen = new Writable({ - write(chunk, _encoding, done) { - echoed.push(String(chunk)); - done(); - }, - }); - const lines = createInterface({ input: t.input, output: screen, terminal: true }); - const line = new Promise((resolve) => lines.once("line", resolve)); - t.input.write(`${KEY}\r`); - await line; - lines.close(); - expect(echoed.join("")).toContain(KEY); - }); - - it("is read as typed, and nothing of it is written to the screen, not as it is typed and not after", async () => { - const t = terminal(); - const read = readSecretLine(t.input, t.output); - t.input.write(KEY); - t.input.write("\r"); - expect(await read).toBe(KEY); - // Only the newline that Enter would have echoed: the next line of output does not follow the prompt. - expect(t.screen()).toBe("\n"); - expect(t.screen()).not.toContain("sk-or"); - }); - - it("is read when it arrives in pieces, as a paste does, and keeps what backspace did not take", async () => { - const t = terminal(); - const read = readSecretLine(t.input, t.output); - t.input.write("123456:AA"); - t.input.write("BBxx\u007f\u007fCC"); - t.input.write("\n"); - expect(await read).toBe("123456:AABBCC"); - expect(t.screen()).toBe("\n"); - }); - - it("is given up at Ctrl-C, with nothing of what was typed shown", async () => { - const t = terminal(); - const read = readSecretLine(t.input, t.output); - t.input.write("half of a key"); - t.input.write("\u0003"); - await expect(read).rejects.toBeInstanceOf(SecretInputCancelled); - expect(t.screen()).not.toContain("half"); - }); - - it("returns what was typed when the input ends before Enter", async () => { - const t = terminal(); - const read = readSecretLine(t.input, t.output); - t.input.write("abc"); - t.input.end(); - expect(await read).toBe("abc"); - }); - - it("puts the terminal back as it found it", async () => { - const t = terminal(); - const read = readSecretLine(t.input, t.output); - t.input.write("x\r"); - await read; - expect(t.input.raw.at(-1)).toBe(false); - }); -}); diff --git a/apps/cli/test/serve.test.ts b/apps/cli/test/serve.test.ts deleted file mode 100644 index 25a3c7fa..00000000 --- a/apps/cli/test/serve.test.ts +++ /dev/null @@ -1,178 +0,0 @@ -import { EventEmitter } from "node:events"; -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { parseListen } from "@invisible-dots/api"; -import type { HostPaths } from "@invisible-dots/shared"; -import { serve, type ServeDeps } from "../src/serve.js"; -import type { WebServer, WebServerOptions } from "../src/web.js"; - -let repoRoot: string; - -beforeEach(async () => { - repoRoot = await mkdtemp(join(tmpdir(), "idots-serve-")); -}); - -afterEach(() => rm(repoRoot, { recursive: true, force: true })); - -async function buildWeb() { - const app = join(repoRoot, "apps", "web", ".next", "standalone", "apps", "web"); - await mkdir(join(app, ".next", "static"), { recursive: true }); - await writeFile(join(app, "server.js"), ""); -} - -/** The control plane and the web server as fakes that record what happened to them, in order. */ -function harness(over: { webStart?: (options: WebServerOptions) => Promise } = {}) { - const log: string[] = []; - const lines: string[] = []; - const signals = new EventEmitter(); - const webOptions: WebServerOptions[] = []; - const logger = { - debug() {}, - info: (m: string) => lines.push(`info ${m}`), - warn: (m: string) => lines.push(`warn ${m}`), - error: (m: string) => lines.push(`error ${m}`), - }; - let stopped!: () => void; - const deps: ServeDeps = { - startServer: async (options) => { - log.push(`server start env=${options.env ? "given" : "none"}`); - return { - url: "http://127.0.0.1:8787", - paths: { apiTokenPath: "/home/x/.invisible-dots/config/api.token" } as HostPaths, - close: async () => { - log.push("server close"); - }, - }; - }, - untilStopSignal: (close, _logger, emitter) => - new Promise((resolve) => { - stopped = resolve; - emitter?.once("SIGINT", () => void close().then(resolve)); - }), - parseListen, - startWebServer: - over.webStart ?? - (async (options) => { - webOptions.push(options); - log.push("web start"); - return { - url: `http://${options.listen.host}:${options.listen.port}`, - stop: async () => { - log.push("web stop"); - }, - }; - }), - }; - const run = (options: { web: boolean; env?: Record }) => - serve({ env: options.env ?? {}, logger, web: options.web, repoRoot, signals }, deps); - return { run, log, lines, signals, webOptions, release: () => stopped?.() }; -} - -/** Waits until the check holds: serve starts its parts asynchronously, and a fixed wait loses that race on a loaded machine. */ -const until = (check: () => void) => vi.waitFor(check, { timeout: 5_000, interval: 5 }); - -describe("serve", () => { - it("starts the control plane, then the web client on the default address pointed at it, and stops them in the other order", async () => { - await buildWeb(); - const h = harness(); - const running = h.run({ web: true }); - await until(() => expect(h.log).toEqual(["server start env=given", "web start"])); - expect(h.webOptions[0]).toMatchObject({ listen: { host: "127.0.0.2", port: 3000 }, apiUrl: "http://127.0.0.1:8787" }); - expect(h.webOptions[0]!.build.missing).toBeUndefined(); - expect(h.lines).toContain("info API token in /home/x/.invisible-dots/config/api.token; stop with Ctrl+C (Dots keep running)"); - expect(h.lines).toContain("info web client at http://127.0.0.2:3000"); - h.signals.emit("SIGINT"); - await running; - expect(h.log).toEqual(["server start env=given", "web start", "web stop", "server close"]); - }); - - it("listens where INVISIBLE_DOTS_WEB_LISTEN says", async () => { - await buildWeb(); - const h = harness(); - const running = h.run({ web: true, env: { INVISIBLE_DOTS_WEB_LISTEN: "127.0.0.1:3100" } }); - await until(() => expect(h.webOptions[0]?.listen).toEqual({ host: "127.0.0.1", port: 3100 })); - h.signals.emit("SIGINT"); - await running; - }); - - it("fails before starting anything when INVISIBLE_DOTS_WEB_LISTEN is malformed", async () => { - const h = harness(); - await expect(h.run({ web: true, env: { INVISIBLE_DOTS_WEB_LISTEN: "three-thousand" } })).rejects.toThrow(/INVISIBLE_DOTS_WEB_LISTEN must look like/); - expect(h.log).toEqual([]); - }); - - it("names the web client's own default, not the API's, when INVISIBLE_DOTS_WEB_LISTEN is malformed", async () => { - const h = harness(); - await expect(h.run({ web: true, env: { INVISIBLE_DOTS_WEB_LISTEN: "three-thousand" } })).rejects.toThrow( - 'INVISIBLE_DOTS_WEB_LISTEN must look like "127.0.0.2:3000", got "three-thousand"', - ); - }); - - it("fails before starting anything when INVISIBLE_DOTS_WEB_LISTEN names port 0", async () => { - const h = harness(); - await expect(h.run({ web: true, env: { INVISIBLE_DOTS_WEB_LISTEN: "127.0.0.1:0" } })).rejects.toThrow( - 'INVISIBLE_DOTS_WEB_LISTEN needs a fixed port, got "127.0.0.1:0"', - ); - expect(h.log).toEqual([]); - }); - - it("does not start the web client with --no-web, and does not read INVISIBLE_DOTS_WEB_LISTEN", async () => { - await buildWeb(); - const h = harness(); - const running = h.run({ web: false, env: { INVISIBLE_DOTS_WEB_LISTEN: "garbage" } }); - // Running: it waits for the stop signal. - await until(() => expect(h.signals.listenerCount("SIGINT")).toBe(1)); - h.signals.emit("SIGINT"); - await running; - expect(h.log).toEqual(["server start env=given", "server close"]); - }); - - it("keeps the control plane running, and says why, when the web client cannot start", async () => { - const h = harness({ - webStart: async () => { - throw new Error("http://127.0.0.1:3000 is already in use"); - }, - }); - const running = h.run({ web: true }); - await until(() => expect(h.lines).toContain("warn web client not started: http://127.0.0.1:3000 is already in use; the control plane keeps running")); - expect(h.log).toEqual(["server start env=given"]); - h.signals.emit("SIGINT"); - await running; - expect(h.log).toEqual(["server start env=given", "server close"]); - }); - - it("closes the control plane on Ctrl+C while the web client is still starting, and ends that start", async () => { - let aborted = false; - let starting = false; - const h = harness({ - webStart: (options) => - new Promise((_resolve, reject) => { - starting = true; - options.signal!.addEventListener("abort", () => { - aborted = true; - reject(new Error("stopped while the web client was starting")); - }); - }), - }); - await buildWeb(); - const running = h.run({ web: true }); - await until(() => expect(starting).toBe(true)); - h.signals.emit("SIGINT"); - await running; - expect(aborted).toBe(true); - expect(h.log).toEqual(["server start env=given", "server close"]); - // Ending the start is what the person asked for, not a failure to report. - expect(h.lines.filter((line) => line.startsWith("warn"))).toEqual([]); - }); - - it("names the missing build in the web start it hands over, so the warning carries the fix", async () => { - const h = harness(); - const running = h.run({ web: true }); - await until(() => expect(h.webOptions).toHaveLength(1)); - expect(h.webOptions[0]!.build.missing).toBe(h.webOptions[0]!.build.entry); - h.signals.emit("SIGINT"); - await running; - }); -}); diff --git a/apps/cli/test/setup-all.test.ts b/apps/cli/test/setup-all.test.ts deleted file mode 100644 index 029c95ca..00000000 --- a/apps/cli/test/setup-all.test.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { DEFAULT_WEB_LISTEN, ENV } from "@invisible-dots/shared"; -import { EXIT } from "../src/exit.js"; -import { runSetupAll, SETUP_ALL_COMMAND, webAddress, type SetupAllDeps } from "../src/setup/all.js"; -import type { StepResult } from "../src/setup/build.js"; -import type { HostOutcome } from "../src/setup/setup.js"; - -const DONE: StepResult = { ok: true, lines: ["done"] }; - -/** A run of `setup --all` against fake steps; `calls` is the order they ran in. */ -async function all(overrides: Partial = {}) { - let text = ""; - const calls: string[] = []; - const deps: SetupAllDeps = { - out: (t) => (text += t), - agent: async () => (calls.push("agent"), { ok: true, lines: ["built /repo/guest/dot-agentd/bin/dot-agentd"] }), - host: async () => (calls.push("host"), { ready: true, results: [] } satisfies HostOutcome), - web: async () => (calls.push("web"), DONE), - images: async () => (calls.push("images"), EXIT.ok), - env: {}, - ...overrides, - }; - const code = await runSetupAll(deps); - return { code, text, calls }; -} - -describe("setup --all", () => { - it("runs the guest daemon, QEMU, the web client and the images in that order, then says how to open the web client", async () => { - const result = await all(); - expect(result.code).toBe(EXIT.ok); - expect(result.calls).toEqual(["agent", "host", "web", "images"]); - expect(result.text).toMatch(/\[1\/4\] the guest daemon[\s\S]*\[2\/4\] QEMU[\s\S]*\[3\/4\] the web client[\s\S]*\[4\/4\] the guest images/); - expect(result.text).toContain("built /repo/guest/dot-agentd/bin/dot-agentd"); - expect(result.text).toContain("1. start the server and leave it running: invisible-dots server"); - expect(result.text).toContain(`2. open http://${DEFAULT_WEB_LISTEN} in your browser\n`); - expect(result.text).not.toMatch(/sign in|api\.token/); - expect(result.text).toContain("OpenRouter key"); - }); - - it("names the address the web client is configured to listen on", async () => { - expect(webAddress({})).toBe("http://127.0.0.2:3000"); - expect(webAddress({ [ENV.WEB_LISTEN]: " 127.0.0.1:4100 " })).toBe("http://127.0.0.1:4100"); - expect(webAddress({ [ENV.WEB_LISTEN]: "" })).toBe("http://127.0.0.2:3000"); - const result = await all({ env: { [ENV.WEB_LISTEN]: "127.0.0.1:4100" } }); - expect(result.text).toContain("open http://127.0.0.1:4100 in your browser"); - }); - - it("stops at a missing Go before it changes anything on the host, and says to run the same command again", async () => { - const result = await all({ agent: async () => ({ ok: false, lines: ["Go is not installed", "install it with: winget install -e --id GoLang.Go"] }) }); - expect(result.code).toBe(EXIT.failed); - expect(result.calls).toEqual([]); - expect(result.text).toContain(" install it with: winget install -e --id GoLang.Go\n"); - expect(result.text).toContain(`run ${SETUP_ALL_COMMAND} again: it skips what is done`); - expect(result.text).not.toContain("This computer is ready"); - }); - - it("stops with exit code 5 where Windows must restart, runs no later step, and says the same command carries on", async () => { - const result = await all({ host: async () => ({ ready: false, code: EXIT.restart }) }); - expect(result.code).toBe(EXIT.restart); - expect(result.text).toContain(`Restart the computer now, then run ${SETUP_ALL_COMMAND} again from this folder: it carries on with step 3.`); - expect(result.text).not.toContain("[3/4]"); - expect(result.text).not.toContain("This computer is ready"); - }); - - it("stops where QEMU or the accelerator cannot be made ready (a new login on Linux), keeping setup's own exit code", async () => { - const result = await all({ host: async () => ({ ready: false, code: EXIT.failed }) }); - expect(result.code).toBe(EXIT.failed); - expect(result.calls).toEqual(["agent"]); - expect(result.text).toContain("log out and in again if it says so"); - expect(result.text).toContain(`run ${SETUP_ALL_COMMAND} again`); - }); - - it("adds nothing to setup's refusal to run as root", async () => { - const result = await all({ host: async () => ({ ready: false, code: EXIT.usage }) }); - expect(result.code).toBe(EXIT.usage); - expect(result.text).not.toContain("Restart"); - expect(result.text).not.toContain("Stopped"); - }); - - it("stops at a web build that failed, before the long image build", async () => { - const calls: string[] = []; - const result = await all({ - web: async () => ({ ok: false, lines: ["building the web client failed (exit code 1; its output is above)"] }), - images: async () => (calls.push("images"), EXIT.ok), - }); - expect(result.code).toBe(EXIT.failed); - expect(calls).toEqual([]); - expect(result.text).toContain("building the web client failed"); - }); - - it("reports an image build that failed or threw, and how to continue", async () => { - const failed = await all({ images: async () => EXIT.failed }); - expect(failed.code).toBe(EXIT.failed); - expect(failed.text).toContain("image build ended with exit code 1"); - const threw = await all({ - images: async () => { - throw new Error("the download of the Ubuntu image failed"); - }, - }); - expect(threw.code).toBe(EXIT.failed); - expect(threw.text).toContain("image build stopped: the download of the Ubuntu image failed"); - expect(threw.text).toContain(`run ${SETUP_ALL_COMMAND} again`); - expect(threw.text).not.toContain("This computer is ready"); - }); - - it("stops between steps once Ctrl-C was pressed", async () => { - const controller = new AbortController(); - const result = await all({ - signal: controller.signal, - host: async () => { - controller.abort(); - return { ready: true, results: [] }; - }, - }); - expect(result.code).toBe(EXIT.failed); - expect(result.calls).toEqual(["agent"]); - expect(result.text).toContain("interrupted"); - }); -}); diff --git a/apps/cli/test/setup.test.ts b/apps/cli/test/setup.test.ts deleted file mode 100644 index c619c0b3..00000000 --- a/apps/cli/test/setup.test.ts +++ /dev/null @@ -1,191 +0,0 @@ -import { describe, expect, it } from "vitest"; -import type { DoctorCheck } from "@invisible-dots/shared"; -import { WEB_BUILD_COMMAND, type DoctorDeps } from "@invisible-dots/vm-manager"; -import { EXIT } from "../src/exit.js"; -import type { InstallDeps, InstallOutcome, InstallRequest } from "../src/setup/install.js"; -import { nextSteps, prepareHost, runSetup } from "../src/setup/setup.js"; -import { FOUND, healthyDoctor } from "../../vm-manager/test/doctor-fakes.js"; - -const MISSING_QEMU = async () => ({ searched: ["PATH"] }); -const FEATURE_OFF: DoctorCheck = { id: "accelerator", label: "accelerator", status: "missing", detail: "the HypervisorPlatform feature is disabled", fix: "invisible-dots setup" }; - -/** - * A setup run against fakes. `install` stands in for the platform module: - * it records the request and may change what the second doctor run sees, - * the way a real installation would. - */ -async function setup(options: { - doctor: Partial; - outcome?: InstallOutcome; - afterInstall?: Partial; -}) { - let text = ""; - const requests: InstallRequest[] = []; - const { deps: doctor } = healthyDoctor(options.doctor); - const install = { - platform: "win32", - } as unknown as InstallDeps; - const code = await runSetup({ - doctor, - install, - out: (t) => (text += t), - installPrerequisites: async (request) => { - requests.push(request); - Object.assign(doctor, options.afterInstall ?? {}); - return options.outcome ?? { kind: "done", lines: [] }; - }, - }); - return { code, text, requests }; -} - -describe("setup", () => { - it("refuses to run as root on Linux, before it checks or installs anything", async () => { - let text = ""; - let asked = false; - const { deps: doctor, calls } = healthyDoctor(); - const code = await runSetup({ - doctor, - install: { platform: "linux", isRoot: true } as unknown as InstallDeps, - out: (t) => (text += t), - installPrerequisites: async () => { - asked = true; - return { kind: "done", lines: [] }; - }, - }); - expect(code).toBe(EXIT.usage); - expect(text).toMatch(/as the user who runs the server, not as root: it calls sudo itself/); - expect(asked).toBe(false); - expect(calls).toEqual([]); - }); - - it("installs nothing when QEMU and the accelerator are ready, and names what is left", async () => { - const result = await setup({ - doctor: { images: async () => [{ id: "golden-image", label: "golden image", status: "missing", detail: "none", fix: "invisible-dots image build" }, { id: "runtime-image", label: "runtime ISO", status: "ok", detail: "ok" }] }, - }); - expect(result.code).toBe(EXIT.ok); - expect(result.requests).toEqual([]); - expect(result.text).toContain("QEMU and its accelerator are ready; nothing to install."); - expect(result.text).toContain(" invisible-dots image build\n"); - }); - - it("asks for QEMU and the feature together, then checks again and succeeds", async () => { - const result = await setup({ - doctor: { findQemu: MISSING_QEMU, acceleratorAccess: async () => FEATURE_OFF }, - outcome: { kind: "done", lines: ["QEMU installed", "the HypervisorPlatform feature is enabled"] }, - afterInstall: { findQemu: async () => FOUND, acceleratorAccess: async () => ({ id: "accelerator", label: "accelerator", status: "ok", detail: "enabled" }) }, - }); - expect(result.requests).toEqual([{ installQemu: true, enableAccelerator: true }]); - expect(result.code).toBe(EXIT.ok); - expect(result.text).toContain("QEMU installed\n"); - expect(result.text).toContain("checking again:"); - expect(result.text).toContain("this host is ready: invisible-dots server\n"); - }); - - it("leaves the feature alone when the probe shows WHPX already works", async () => { - const result = await setup({ doctor: { acceleratorAccess: async () => FEATURE_OFF } }); - expect(result.requests).toEqual([]); - expect(result.code).toBe(EXIT.ok); - }); - - it("stops with exit 5 when Windows must restart", async () => { - const { deps } = healthyDoctor(); - const versions = deps.run; - const result = await setup({ - doctor: { - acceleratorAccess: async () => FEATURE_OFF, - run: async (command, args, options) => - args.includes("-accel") ? { code: 1, signal: null, stdout: "", stderr: "WHPX: No accelerator found, hr=80370102\n", timedOut: false } : versions(command, args, options), - }, - outcome: { kind: "restart", lines: ["restart Windows, then run: invisible-dots doctor"] }, - }); - expect(result.requests).toEqual([{ installQemu: false, enableAccelerator: true }]); - expect(result.code).toBe(EXIT.restart); - expect(result.text).not.toContain("checking again"); - expect(result.text.trimEnd().endsWith("restart Windows, then run: invisible-dots doctor")).toBe(true); - }); - - it("exits 1 on a failed or manual step, and when the second check still fails", async () => { - const failed = await setup({ doctor: { findQemu: MISSING_QEMU }, outcome: { kind: "failed", lines: ["the elevated step failed: boom"] } }); - expect(failed.code).toBe(EXIT.failed); - expect(failed.text).toContain("the elevated step failed: boom\nsetup stopped"); - - const manual = await setup({ doctor: { findQemu: MISSING_QEMU }, outcome: { kind: "manual", lines: [" sudo dnf install -y qemu-system-x86 qemu-img"] } }); - expect(manual.code).toBe(EXIT.failed); - expect(manual.text).toContain("then run: invisible-dots doctor"); - - const still = await setup({ doctor: { findQemu: MISSING_QEMU } }); - expect(still.code).toBe(EXIT.failed); - expect(still.text).toContain("QEMU or its accelerator is still not usable"); - }); - - it("does not try to install anything when only the probe fails", async () => { - const { deps } = healthyDoctor(); - const versions = deps.run; - const result = await setup({ - doctor: { - run: async (command, args, options) => - args.includes("-accel") ? { code: 1, signal: null, stdout: "", stderr: "WHPX: No accelerator found, hr=00000000\n", timedOut: false } : versions(command, args, options), - }, - }); - expect(result.code).toBe(EXIT.failed); - expect(result.requests).toEqual([]); - expect(result.text).toContain("setup cannot fix this by installing something:\n accelerator probe: qemu-system-x86_64 -accel kvm -cpu host,-vmx,-svm -machine q35 failed: WHPX: No accelerator found"); - }); - - it("names the server and the key as the next steps while they are missing", () => { - const missingKey: DoctorCheck[] = [ - { id: "golden-image", label: "golden image", status: "ok", detail: "" }, - { id: "runtime-image", label: "runtime ISO", status: "ok", detail: "" }, - { id: "web", label: "web client", status: "ok", detail: "" }, - { id: "openrouter", label: "OpenRouter key", status: "failed", detail: "" }, - ]; - expect(nextSteps(missingKey)).toEqual([ - "invisible-dots server (keep it running, then in another terminal:)", - // One command for both hosts: "<" is a parser error in Windows PowerShell. - "invisible-dots secret openrouter", - ]); - }); - - it("names the web build as a next step while the web client is not built", () => { - const notBuilt: DoctorCheck[] = [ - { id: "golden-image", label: "golden image", status: "ok", detail: "" }, - { id: "runtime-image", label: "runtime ISO", status: "ok", detail: "" }, - { id: "web", label: "web client", status: "missing", detail: "", fix: WEB_BUILD_COMMAND }, - { id: "openrouter", label: "OpenRouter key", status: "ok", detail: "" }, - ]; - expect(nextSteps(notBuilt)).toEqual(["npm run build --workspace @invisible-dots/web", "invisible-dots server"]); - }); - - it("names no web build when the report has no web row: the host is not asked to check a client", () => { - const noWebRow: DoctorCheck[] = [ - { id: "golden-image", label: "golden image", status: "ok", detail: "" }, - { id: "runtime-image", label: "runtime ISO", status: "ok", detail: "" }, - { id: "openrouter", label: "OpenRouter key", status: "ok", detail: "" }, - ]; - expect(nextSteps(noWebRow)).toEqual(["invisible-dots server"]); - }); - describe("prepareHost, the part of setup that setup --all runs as its second step", () => { - const install = { platform: "win32" } as unknown as InstallDeps; - - it("hands back the report once QEMU is ready and leaves what to do next to its caller", async () => { - let text = ""; - const { deps: doctor } = healthyDoctor(); - const outcome = await prepareHost({ doctor, install, out: (t) => (text += t) }); - expect(outcome.ready).toBe(true); - expect(outcome.ready && outcome.results.map((r) => r.id)).toContain("accelerator"); - expect(text).toContain("QEMU and its accelerator are ready; nothing to install."); - expect(text).not.toContain("this host is ready"); - expect(text).not.toContain("Next:"); - }); - - it("hands back setup's exit code when it cannot get the host ready", async () => { - const { deps: doctor } = healthyDoctor({ findQemu: MISSING_QEMU, acceleratorAccess: async () => FEATURE_OFF }); - const restart = await prepareHost({ doctor, install, out: () => undefined, installPrerequisites: async () => ({ kind: "restart", lines: [] }) }); - expect(restart).toEqual({ ready: false, code: EXIT.restart }); - const failed = await prepareHost({ doctor, install, out: () => undefined, installPrerequisites: async () => ({ kind: "failed", lines: [] }) }); - expect(failed).toEqual({ ready: false, code: EXIT.failed }); - const root = await prepareHost({ doctor, install: { platform: "linux", isRoot: true } as unknown as InstallDeps, out: () => undefined }); - expect(root).toEqual({ ready: false, code: EXIT.usage }); - }); - }); -}); diff --git a/apps/cli/test/web.test.ts b/apps/cli/test/web.test.ts deleted file mode 100644 index 32e5bcb3..00000000 --- a/apps/cli/test/web.test.ts +++ /dev/null @@ -1,234 +0,0 @@ -import { spawn } from "node:child_process"; -import { realpathSync } from "node:fs"; -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; -import { createServer } from "node:net"; -import { tmpdir } from "node:os"; -import { dirname, join } from "node:path"; -import { fileURLToPath, pathToFileURL } from "node:url"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { locateWebBuild, startWebServer, webEnvironment, WebStartError, type WebServer, type WebServerOptions } from "../src/web.js"; - -let root: string; -let started: WebServer[] = []; - -beforeEach(async () => { - root = await mkdtemp(join(tmpdir(), "idots-web-")); -}); - -afterEach(async () => { - await Promise.all(started.map((web) => web.stop())); - started = []; - await rm(root, { recursive: true, force: true }); -}); - -/** A server script that answers every request with the environment it was started with. */ -const ECHO_SERVER = ` -import { createServer } from "node:http"; -createServer((req, res) => { - res.setHeader("content-type", "application/json"); - res.end(JSON.stringify({ env: process.env, cwd: process.cwd() })); -}).listen(Number(process.env.PORT), process.env.HOSTNAME); -`; - -async function script(source: string): Promise { - const entry = join(root, "server.mjs"); - await writeFile(entry, source); - return entry; -} - -async function freePort(): Promise { - return new Promise((resolve) => { - const probe = createServer(); - probe.listen(0, "127.0.0.1", () => { - const port = (probe.address() as { port: number }).port; - probe.close(() => resolve(port)); - }); - }); -} - -async function start(entry: string, over: Partial = {}) { - const port = await freePort(); - const exits: string[] = []; - const web = await startWebServer({ - build: { entry, missing: undefined }, - listen: { host: "127.0.0.1", port }, - apiUrl: "http://127.0.0.1:8787", - env: { PATH: process.env.PATH }, - onUnexpectedExit: (message) => exits.push(message), - ...over, - }); - started.push(web); - return { web, port, exits }; -} - -describe("locateWebBuild", () => { - it("names the server script and the first file of the build that is missing", async () => { - const empty = await locateWebBuild(root); - expect(empty.entry).toBe(join(root, "apps", "web", ".next", "standalone", "apps", "web", "server.js")); - expect(empty.missing).toBe(empty.entry); - - const app = join(root, "apps", "web", ".next", "standalone", "apps", "web"); - await mkdir(app, { recursive: true }); - await writeFile(join(app, "server.js"), ""); - expect((await locateWebBuild(root)).missing).toBe(join(app, ".next", "static")); - - await mkdir(join(app, ".next", "static"), { recursive: true }); - expect((await locateWebBuild(root)).missing).toBeUndefined(); - }); -}); - -describe("webEnvironment", () => { - it("passes on where to listen and the API, the data directory and what the Host check allows, and nothing else", () => { - const env = webEnvironment({ - env: { - PATH: "/bin", - INVISIBLE_DOTS_HOME: "/data", - INVISIBLE_DOTS_WEB_ALLOWED_HOSTS: "dots.example", - DATABASE_URL: "postgres://user:secret@db/dots", - OPENROUTER_API_KEY: "sk-secret", - INVISIBLE_DOTS_URL: "http://elsewhere:1", - }, - listen: { host: "127.0.0.1", port: 3000 }, - apiUrl: "http://127.0.0.1:8787", - parentPid: 4242, - }); - expect(env).toEqual({ - PATH: "/bin", - INVISIBLE_DOTS_HOME: "/data", - INVISIBLE_DOTS_WEB_ALLOWED_HOSTS: "dots.example", - INVISIBLE_DOTS_URL: "http://127.0.0.1:8787", - INVISIBLE_DOTS_WEB_PARENT_PID: "4242", - PORT: "3000", - HOSTNAME: "127.0.0.1", - }); - }); - - it("passes the API token only when the server was given it in its environment", () => { - const env = webEnvironment({ - env: { INVISIBLE_DOTS_TOKEN: "token-from-the-environment" }, - listen: { host: "127.0.0.1", port: 3000 }, - apiUrl: "http://127.0.0.1:8787", - parentPid: 4242, - }); - expect(env.INVISIBLE_DOTS_TOKEN).toBe("token-from-the-environment"); - }); -}); - -describe("startWebServer", { timeout: 30_000 }, () => { - it("starts the script on the listen address with the web environment and answers at its url", async () => { - const entry = await script(ECHO_SERVER); - const { web, port } = await start(entry, { env: { PATH: process.env.PATH, DATABASE_URL: "postgres://user:secret@db/dots" } }); - expect(web.url).toBe(`http://127.0.0.1:${port}`); - const body = (await (await fetch(web.url)).json()) as { env: Record; cwd: string }; - expect(body.env.PORT).toBe(String(port)); - expect(body.env.HOSTNAME).toBe("127.0.0.1"); - expect(body.env.INVISIBLE_DOTS_URL).toBe("http://127.0.0.1:8787"); - expect(body.env.INVISIBLE_DOTS_WEB_PARENT_PID).toBe(String(process.pid)); - expect(body.env.DATABASE_URL).toBeUndefined(); - // It runs in its own directory, never the server's. - expect(realpathSync.native(body.cwd)).toBe(realpathSync.native(root)); - }); - - it("stops the child: nothing answers afterwards, and a second stop is harmless", async () => { - const entry = await script(ECHO_SERVER); - const { web, exits } = await start(entry); - await web.stop(); - await web.stop(); - await expect(fetch(web.url, { signal: AbortSignal.timeout(1000) })).rejects.toThrow(); - expect(exits).toEqual([]); - }); - - it("reports a child that exits later, once", async () => { - const entry = await script(` - import { createServer } from "node:http"; - createServer((req, res) => res.end("ok")).listen(Number(process.env.PORT), process.env.HOSTNAME); - setTimeout(() => { console.error("out of memory"); process.exit(7); }, 1500); - `); - const { exits } = await start(entry); - await expect.poll(() => exits.length, { timeout: 10_000 }).toBe(1); - expect(exits[0]).toBe("the web server exited (exit code 7): out of memory"); - }); - - it("fails with the child's own words when it exits before it answers", async () => { - const entry = await script(`console.error("Error: listen EADDRINUSE"); process.exit(1);`); - await expect(start(entry)).rejects.toThrow(new WebStartError("the web server exited (exit code 1): Error: listen EADDRINUSE")); - }); - - it("refuses a port something already answers on, instead of waiting on the wrong server", async () => { - const other = await script(ECHO_SERVER); - const { web, port } = await start(other); - await expect(start(other, { listen: { host: "127.0.0.1", port } })).rejects.toThrow( - `${web.url} is already in use; free the port or set INVISIBLE_DOTS_WEB_LISTEN to another host:port`, - ); - }); - - it("fails and kills the child when it does not answer in time", async () => { - const entry = await script(`setInterval(() => {}, 1000);`); - await expect(start(entry, { readyTimeoutMs: 600, stopTimeoutMs: 2000 })).rejects.toThrow(/did not answer at http:\/\/127\.0\.0\.1:\d+ within 0\.6 s/); - }); - - it("stops waiting when the signal aborts, and leaves no child", async () => { - const entry = await script(`setInterval(() => {}, 1000);`); - const controller = new AbortController(); - setTimeout(() => controller.abort(), 300); - await expect(start(entry, { signal: controller.signal })).rejects.toThrow("stopped while the web client was starting"); - }); - - it("does not start at all when the build is incomplete", async () => { - await expect(start("/nowhere/server.js", { build: { entry: "/nowhere/server.js", missing: "/nowhere/server.js" } })).rejects.toThrow( - "the web client is not built (/nowhere/server.js does not exist); build it with: npm run build --workspace @invisible-dots/web", - ); - }); -}); - -describe("a server that is killed outright", { timeout: 60_000 }, () => { - const here = dirname(fileURLToPath(import.meta.url)); - const watcher = pathToFileURL(join(here, "..", "..", "web", "src", "lib", "parent.ts")).href; - const webModule = pathToFileURL(join(here, "..", "src", "web.ts")).href; - - it("does not leave the web server holding its port: it exits once its parent is gone", async () => { - // Windows ends a Node child with its parent on its own (libuv puts children in a job object), so the - // orphan this guards against is a POSIX one; the web server's own exit is what keeps both the same. - // What the web build's instrumentation.ts does with the pid it is given, around a plain HTTP server. - const entry = await script(` - import { createServer } from "node:http"; - import { exitWhenParentGone, parentPidFrom } from ${JSON.stringify(watcher)}; - createServer((req, res) => res.end("ok")).listen(Number(process.env.PORT), process.env.HOSTNAME); - const exists = (pid) => { try { process.kill(pid, 0); return true; } catch (error) { return error.code === "EPERM"; } }; - const parentPid = parentPidFrom(process.env.INVISIBLE_DOTS_WEB_PARENT_PID); - if (parentPid !== undefined) exitWhenParentGone(parentPid, { exists, onGone: () => process.exit(0), intervalMs: 100 }); - `); - const port = await freePort(); - // The server stand-in: a process that starts the web server and then waits to be killed. - const parentScript = join(root, "parent.mjs"); - await writeFile( - parentScript, - ` - import { startWebServer } from ${JSON.stringify(webModule)}; - const web = await startWebServer({ - build: { entry: ${JSON.stringify(entry)}, missing: undefined }, - listen: { host: "127.0.0.1", port: ${port} }, - apiUrl: "http://127.0.0.1:8787", - env: { PATH: process.env.PATH }, - onUnexpectedExit: () => undefined, - }); - console.log(web.url); - setInterval(() => {}, 1000); - `, - ); - const parent = spawn(process.execPath, ["--import", "tsx", parentScript], { stdio: ["ignore", "pipe", "inherit"], cwd: join(here, "..") }); - try { - const url = await new Promise((resolve, reject) => { - parent.once("exit", () => reject(new Error("the stand-in server exited before the web server answered"))); - parent.stdout.once("data", (chunk: Buffer) => resolve(chunk.toString().trim())); - }); - expect((await fetch(url)).status).toBe(200); - parent.kill("SIGKILL"); - await expect - .poll(() => fetch(url, { signal: AbortSignal.timeout(500) }).then(() => "still answering", () => "gone"), { timeout: 15_000, interval: 200 }) - .toBe("gone"); - } finally { - parent.kill("SIGKILL"); - } - }); -}); diff --git a/apps/cli/tsconfig.json b/apps/cli/tsconfig.json deleted file mode 100644 index 31a061f4..00000000 --- a/apps/cli/tsconfig.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "include": [ - "src/**/*.ts", - "test/**/*.ts" - ] -} diff --git a/apps/scheduler/package.json b/apps/scheduler/package.json deleted file mode 100644 index c8df55a2..00000000 --- a/apps/scheduler/package.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "name": "@invisible-dots/scheduler", - "version": "0.0.0", - "private": true, - "description": "Task dispatcher: wakes Dots on work and puts them to sleep when idle.", - "license": "MIT", - "type": "module", - "exports": { - ".": "./src/index.ts", - "./testing": "./src/testing.ts" - }, - "scripts": { - "typecheck": "tsc -p tsconfig.json" - }, - "dependencies": { - "@invisible-dots/shared": "0.0.0", - "@invisible-dots/database": "0.0.0", - "@invisible-dots/events": "0.0.0" - } -} diff --git a/apps/scheduler/src/dispatcher.ts b/apps/scheduler/src/dispatcher.ts deleted file mode 100644 index a17e67ac..00000000 --- a/apps/scheduler/src/dispatcher.ts +++ /dev/null @@ -1,76 +0,0 @@ -/** - * Hands PENDING tasks to their Dots (section 9.2): claim one task per idle - * Dot with SKIP LOCKED. The claim stores the task's `task.created` in the - * inbound outbox in the same transaction; sending it, waking the Dot first - * when it sleeps, and failing the task when it cannot be delivered, is the - * inbound delivery's job (inbound.ts). - */ -import type { Database } from "@invisible-dots/database"; -import { errorMessage, type Clock, type Logger } from "@invisible-dots/shared"; - -// A claim skips a row another transaction holds (SKIP LOCKED); such holds last as long as one guest event's -// transaction. A pass that claimed nothing while a task could be claimed comes back this soon, this many times; -// past that the scheduler's timer finds the task. -const SKIPPED_RETRY_MS = 25; -const SKIPPED_RETRIES = 40; - -export class Dispatcher { - #running: Promise | null = null; - #again = false; - - constructor( - private readonly db: Database, - private readonly clock: Clock, - private readonly log: Logger, - /** Called with the Dot of every claimed task: its task.created is waiting in the outbox. */ - private readonly deliver: (dotId: string) => void, - ) {} - - /** - * Claim every task that can run now. Calls that arrive while a pass is - * running schedule one more pass instead of running concurrently, so one - * dispatcher never races itself. Resolves when the pass is over. - */ - dispatch(): Promise { - if (this.#running) { - this.#again = true; - return this.#running; - } - this.#running = (async () => { - try { - let skipped = 0; - do { - this.#again = false; - for (;;) { - const ts = this.clock.now().toISOString(); - const claimed = await this.db.transaction((tx) => tx.tasks.claimNext(ts)); - if (!claimed) break; - skipped = 0; - this.log.info("task claimed", { taskId: claimed.task.id, dotId: claimed.task.dot_id }); - this.deliver(claimed.task.dot_id); - } - if (!this.#again && skipped < SKIPPED_RETRIES && (await this.db.tasks.hasClaimable())) { - skipped++; - this.#again = true; - await new Promise((resolve) => setTimeout(resolve, SKIPPED_RETRY_MS)); - } - } while (this.#again); - } catch (error) { - this.log.error("dispatch pass failed", { error: errorMessage(error) }); - } finally { - this.#running = null; - } - })(); - return this.#running; - } - - /** Whether a pass is running. */ - get busy(): boolean { - return this.#running !== null; - } - - /** Wait for the current pass (tests and shutdown). */ - async settle(): Promise { - while (this.#running) await this.#running; - } -} diff --git a/apps/scheduler/src/driver.ts b/apps/scheduler/src/driver.ts deleted file mode 100644 index 4144689c..00000000 --- a/apps/scheduler/src/driver.ts +++ /dev/null @@ -1,164 +0,0 @@ -/** - * What the control plane needs from the VM layer. The real implementation - * is an adapter over @invisible-dots/vm-manager (apps/api/src/vm-driver.ts); - * tests use the in-process fakes of `./testing.js`. - */ -import type { - AgentStateAnswer, - BrowserIdentity, - BrowserIdentityListAnswer, - ComputerResources, - CreateBrowserIdentityRequest, - DotRuntimeConfig, - FileListAnswer, - HealthAnswer, - InboundEvent, - OutboundEvent, - RefusedEvent, - SecretsRequest, - SystemAnswer, - SkillListAnswer, - ToolListAnswer, - VmState, -} from "@invisible-dots/shared"; - -/** - * The guest routes the control plane calls (sections 5.2 and 5.3). The - * method names and shapes are those of vm-manager's GuestClient, which - * satisfies this interface as it is. Failed calls throw an error carrying a - * numeric `status` (0 when the guest could not be reached) and, when the - * guest sent one, its `{ error }` code as `code`. - */ -export interface GuestApi { - health(options?: { timeoutMs?: number; signal?: AbortSignal }): Promise; - system(): Promise; - pushSecrets(secrets: SecretsRequest): Promise; - putConfig(config: DotRuntimeConfig): Promise; - postEvent(event: InboundEvent): Promise; - state(): Promise; - listBrowserIdentities(): Promise; - createBrowserIdentity(body: CreateBrowserIdentityRequest): Promise; - getBrowserIdentity(id: string): Promise; - deleteBrowserIdentity(id: string): Promise; - /** The JPEG of an open identity's window; 409 `not_open` when it is closed, 503 `busy` while a call holds it. */ - getBrowserIdentityFrame(id: string): Promise; - /** End the identity's browser and keep its profile; closing a closed identity is not an error. */ - closeBrowserIdentity(id: string): Promise; - listTools(): Promise; - listSkills(): Promise; - prepareSleep(timeoutMs?: number): Promise; - screenshot(): Promise; - /** The bytes of a file; a file larger than `maxBytes` is refused with status 413 and code `FILE_TOO_LARGE`. */ - readFile(path: string, options?: { maxBytes?: number }): Promise; - listFiles(path: string): Promise; - /** - * The outbound event stream after `after`. It reconnects by itself on - * network errors and ends only when `signal` aborts or it hits an error - * that waiting cannot fix (a refused token). A message that is not an - * event the host knows is not yielded: `onRefused` is awaited for it, in - * order with the events, and the stream goes on past it. - */ - events(options: { after?: number; signal?: AbortSignal; onRefused?: (info: RefusedEvent) => void | Promise }): AsyncIterable; -} - -export interface ComputerSpecInput { - dotId: string; - /** The Dot token in clear; it goes into the seed (section 4.2). */ - token: string; - /** The Dot's VM proxy in clear, absent for a direct exit; it goes into the seed with the token. */ - proxy?: string; - resources: ComputerResources; -} - -export interface CreatedComputer { - /** The golden image the overlay is backed by; recorded, because it can never change for this disk. */ - goldenImage: string; - runtimeImage: string; -} - -export interface StartedComputer { - /** The host port on 127.0.0.1 that QEMU forwards to the guest's port 1024 (section 3.5). */ - guestPort: number; - /** The QEMU process. */ - pid: number; - runtimeImage: string; - /** The VM was already running; nothing was spawned and `guestPort` is the one it runs with. */ - alreadyRunning: boolean; -} - -/** What the VM layer sees of one Dot's VM: its disk, its pid file and its QEMU process (section 3.4). */ -export interface ComputerState { - /** Whether the Dot's disk exists, that is whether the VM was ever created and not destroyed. */ - exists: boolean; - /** - * RUNNING while the Dot's QEMU process runs, STOPPED when it does not, - * ERROR when a live pid cannot be proven to be that QEMU. Whether the guest - * inside is up is guest health, which the READY procedure checks. - */ - state: VmState; - /** The pid of the live QEMU process, null when there is none. */ - pid: number | null; - /** - * The host port forwarded to the guest, from the pid file written when - * QEMU was spawned, so it is right even when the database row is not; - * null when not running. - */ - guestPort: number | null; - /** Why the state is ERROR, for logs and error messages. */ - detail: string | null; -} - -/** Where a guest is reached: the forwarded port recorded in the `computers` row (section 5.1). */ -export interface GuestEndpoint { - dotId: string; - port: number; -} - -export interface WaitForHealthOptions { - timeoutMs: number; - intervalMs: number; - /** Timeout of each health request. */ - requestTimeoutMs: number; -} - -export interface ComputerDriver { - /** Overlay and seed (section 9.4 up to `write seed.iso`). Safe to retry. */ - create(spec: ComputerSpecInput): Promise; - /** Pick a guest port and spawn QEMU detached (sections 3.4 and 3.5). */ - start(spec: ComputerSpecInput & { goldenImage: string }): Promise; - /** - * Poll `GET /v1/health` until dot-agentd and the agent both report ok. - * Fails at once on a refused token (status 401) and when the VM's QEMU - * process exits, and after `timeoutMs` otherwise. - */ - waitForHealth(endpoint: GuestEndpoint, token: string, options: WaitForHealthOptions): Promise; - /** - * `POST /v1/system/poweroff` through the guest channel with the Dot's - * token, then the QEMU process is killed after the grace period (section 9.5). - */ - stop(dotId: string, token: string): Promise<{ forced: boolean }>; - /** - * A full stop and start, so the guest shuts down cleanly and new resources - * and a new runtime ISO apply. The guest port changes. - */ - reboot(spec: ComputerSpecInput & { goldenImage: string }): Promise; - /** Stop the VM if it runs and remove its directory with the disk. */ - destroy(dotId: string): Promise; - /** From the pid file and the process, never from the database: this is what reconciliation compares against. */ - state(dotId: string): Promise; - guest(endpoint: GuestEndpoint, token: string): GuestApi; - /** Release what this process holds. VMs keep running. */ - close(): Promise; -} - -/** The HTTP status a failed guest call carried, or 0 when the guest was not reached. */ -export function guestErrorStatus(error: unknown): number { - const status = (error as { status?: unknown })?.status; - return typeof status === "number" ? status : 0; -} - -/** The `{ error }` code of a failed guest call, when there was one. */ -export function guestErrorCode(error: unknown): string | undefined { - const code = (error as { code?: unknown })?.code; - return typeof code === "string" && !/^E[A-Z]+$/.test(code) ? code : undefined; -} diff --git a/apps/scheduler/src/guest-events.ts b/apps/scheduler/src/guest-events.ts deleted file mode 100644 index e830eaa0..00000000 --- a/apps/scheduler/src/guest-events.ts +++ /dev/null @@ -1,132 +0,0 @@ -/** - * What a guest event changes on the host besides the event log: task rows (state, summary, error, spend), - * approval rows, the Dot's status, the time of its next automation, and the MCP servers of its config. - */ -import type { Repositories } from "@invisible-dots/database"; -import { - isMcpServerName, - parseDotConfig, - parseMcpServerEntry, - resolvePermission, - TERMINAL_TASK_STATES, - type AgentState, - type DotRecord, - type DotState, - type HostEventDataMap, - type OutboundEvent, - type PermissionName, -} from "@invisible-dots/shared"; - -/** - * The Dot status for an agent state. READY means the computer is up and the - * agent is IDLE; DONE is the short step before IDLE (section 8.1) and still - * counts as working. - */ -export function dotStatusForAgent(state: AgentState): DotState { - switch (state) { - case "IDLE": - return "READY"; - case "WAITING_APPROVAL": - return "WAITING_APPROVAL"; - default: - return "RUNNING"; - } -} - -/** Dot states the agent's own reports never override: they are set by the control plane. */ -const STICKY_DOT_STATES: readonly DotState[] = ["CREATING", "ERROR", "IDLE"]; - -export interface AppliedGuestEvent { - /** A task reached a terminal state, so the Dot may take the next one. */ - taskSettled: boolean; - /** The Dot's config changed (an MCP server it added): it is pushed to the guest once the transaction commits. */ - reconfigured?: DotRecord; - /** What the host logs of it, in the same transaction: `dot.updated`, or why it refused (`mcp.server.refused`). */ - hostEvents?: HostEventOf[]; -} - -type HostEventOf = { [K in keyof HostEventDataMap]: { type: K; data: HostEventDataMap[K] } }[keyof HostEventDataMap]; - -/** The tool of the engine that asks to add an MCP server; its approvals are the person's consent to one. */ -export const ADD_MCP_SERVER_TOOL = "add_mcp_server"; -const ADD_PERMISSION: PermissionName = "mcp.servers.add"; - -/** - * A Dot added an MCP server to its own config. The config is the host's, so the host decides again what the engine - * decided: the permission allows it, or the person approved a call of exactly these arguments, and that approval is - * used up here, so it adds one server once. The entry is checked by the config's own schema first (a call refused for - * it keeps its approval), and a name already declared is not replaced. The server's own permission is left to its - * default, ask, so each of its tools still asks until the person allows it. - */ -async function addMcpServer(tx: Repositories, dotId: string, args: Record): Promise { - const name = typeof args.name === "string" ? args.name : ""; - const refuse = (reason: string): AppliedGuestEvent => ({ taskSettled: false, hostEvents: [{ type: "mcp.server.refused", data: { name, reason } }] }); - const dot = await tx.dots.get(dotId); - if (!dot) return refuse("the Dot no longer exists"); - const decision = resolvePermission(dot.config, ADD_PERMISSION); - if (decision === "deny") return refuse(`the Dot's config denies ${ADD_PERMISSION}`); - if (!isMcpServerName(name)) return refuse(`"${name}" is not an MCP server's name`); - if (name in dot.config.mcp_servers) return refuse(`the Dot's config already declares an MCP server "${name}"`); - const { name: _name, reason: _reason, ...entry } = args; - const parsed = parseMcpServerEntry(entry); - if (!parsed.ok) return refuse(parsed.problem); - const config = parseDotConfig({ ...dot.config, mcp_servers: { ...dot.config.mcp_servers, [name]: parsed.server } }); - let approvalId: string | undefined; - if (decision === "ask") { - const approval = await tx.approvals.useApproved(dotId, ADD_MCP_SERVER_TOOL, args); - if (!approval) return refuse(`${ADD_PERMISSION} asks, and no unused approval of exactly these arguments was given`); - approvalId = approval.id; - } - const reconfigured = await tx.dots.updateConfig(dotId, config); - if (!reconfigured) return refuse("the Dot no longer exists"); - const added = { name: dot.name, mcp_server_added: name, ...(approvalId && { approval_id: approvalId }) }; - return { taskSettled: false, reconfigured, hostEvents: [{ type: "dot.updated", data: added }] }; -} - -/** The spend a task event reports (architecture section 5.4), onto the task's row; events without it change nothing. */ -async function recordTaskSpend(tx: Repositories, dotId: string, data: { task_id: string; spent_usd?: number }): Promise { - if (data.spent_usd !== undefined) await tx.tasks.recordSpend(data.task_id, dotId, data.spent_usd); -} - -/** Apply one newly stored guest event inside the transaction that stored it. */ -export async function applyGuestEvent(tx: Repositories, dotId: string, event: OutboundEvent): Promise { - switch (event.type) { - case "agent.state": { - const dot = await tx.dots.get(dotId); - if (dot && !STICKY_DOT_STATES.includes(dot.status)) { - const next = dotStatusForAgent(event.data.state); - if (next !== dot.status) await tx.dots.setStatus(dotId, next); - } - return { taskSettled: false }; - } - case "task.started": - await tx.tasks.transition(event.data.task_id, "RUNNING", { dotId }); - return { taskSettled: false }; - case "task.progress": - await recordTaskSpend(tx, dotId, event.data); - return { taskSettled: false }; - case "task.completed": - await recordTaskSpend(tx, dotId, event.data); - return { taskSettled: (await tx.tasks.transition(event.data.task_id, "COMPLETED", { summary: event.data.summary, dotId })) !== null }; - case "task.failed": - await recordTaskSpend(tx, dotId, event.data); - return { taskSettled: (await tx.tasks.transition(event.data.task_id, "FAILED", { error: event.data.error, dotId })) !== null }; - case "automation.next_run": - // The guest's report of when its earliest automation is due, so a stopped computer is started shortly before it. - await tx.computers.setNextAutomation(dotId, event.data.next_run_at_ms); - return { taskSettled: false }; - case "approval.requested": { - // A request for a task that already ended (cancelled while the guest - // was asking) is kept for the record but never listed as pending. - const task = event.data.task_id ? await tx.tasks.get(event.data.task_id) : null; - const ended = task !== null && task.dot_id === dotId && TERMINAL_TASK_STATES.includes(task.status); - await tx.approvals.insertRequested(dotId, event.data, ended ? "expired" : "pending"); - if (event.data.task_id && !ended) await tx.tasks.transition(event.data.task_id, "WAITING_APPROVAL", { dotId }); - return { taskSettled: false }; - } - case "mcp.server.added": - return addMcpServer(tx, dotId, event.data.arguments); - default: - return { taskSettled: false }; - } -} diff --git a/apps/scheduler/src/inbound.ts b/apps/scheduler/src/inbound.ts deleted file mode 100644 index d0aa41dd..00000000 --- a/apps/scheduler/src/inbound.ts +++ /dev/null @@ -1,301 +0,0 @@ -/** - * Delivery of host to guest events (sections 5.4 and 9.5) from the durable - * outbox in `inbound_events`. Whatever the control plane decides to tell a - * guest (a claimed task, a message, an approval, a cancel) is stored first, - * in the transaction that decided it; this class only sends what is stored. - * - * Per Dot, one flush runs at a time and sends the pending rows in order, - * stopping at the first failure so a later event never overtakes an earlier - * one. A Dot that is not READY is woken first. A row is marked delivered - * only on the guest's 202; the guest keeps every event id it accepted, so a - * row whose send had an unknown outcome (a timeout, a reset connection) is - * simply sent again. Every READY transition flushes the Dot again, so - * nothing waits for a timer once the guest is back. - */ -import { TASK_ENDED_BEFORE_DELIVERY, type Database, type InboundRecord } from "@invisible-dots/database"; -import type { EventLog } from "@invisible-dots/events"; -import { guestErrorCode, guestErrorStatus } from "./driver.js"; -import type { Lifecycle } from "./lifecycle.js"; -import { errorMessage, type Clock, type Logger } from "@invisible-dots/shared"; - -export interface InboundDeliveryOptions { - /** Failed sends with a known outcome before a task.created gives its task up as FAILED. */ - maxDeliveryAttempts: number; - /** Delay before a failed row is retried, multiplied by its number of failures. */ - retryDelayMs: number; -} - -export const DEFAULT_INBOUND_OPTIONS: InboundDeliveryOptions = { - maxDeliveryAttempts: 3, - retryDelayMs: 30_000, -}; - -export interface InboundDeliveryDeps { - db: Database; - events: EventLog; - lifecycle: Lifecycle; - clock: Clock; - logger: Logger; - /** Resolves once a Dot that is still being provisioned is (successfully or not). */ - provisioned: (dotId: string) => Promise; - /** A task reached a terminal state here: the dispatcher may have work to hand out. */ - onTaskSettled: () => void; - options?: Partial; -} - -/** - * Whether a failed send certainly did NOT hand the event to the guest: the - * guest answered with an error, or nothing listened on the guest port. A - * timeout or a dropped connection may have delivered it, so it is not - * counted against the row. - */ -function knownUndelivered(error: unknown): boolean { - return guestErrorStatus(error) >= 400 || (error as { code?: unknown })?.code === "ECONNREFUSED"; -} - -/** Rolls back a give-up whose task ended before it: the task keeps that end and logs no failure. */ -class TaskAlreadyEnded extends Error {} - -export class InboundDelivery { - readonly #db: Database; - readonly #events: EventLog; - readonly #lifecycle: Lifecycle; - readonly #clock: Clock; - readonly #log: Logger; - readonly #provisioned: (dotId: string) => Promise; - readonly #onTaskSettled: () => void; - readonly #opts: InboundDeliveryOptions; - readonly #flushing = new Map>(); - readonly #again = new Set(); - readonly #timers = new Set(); - /** Dots whose deliveries wait for something that has to reach the guest first (`hold`). */ - readonly #holds = new Map; release: () => void }>(); - #closed = false; - - constructor(deps: InboundDeliveryDeps) { - this.#db = deps.db; - this.#events = deps.events; - this.#lifecycle = deps.lifecycle; - this.#clock = deps.clock; - this.#log = deps.logger; - this.#provisioned = deps.provisioned; - this.#onTaskSettled = deps.onTaskSettled; - this.#opts = { ...DEFAULT_INBOUND_OPTIONS, ...deps.options }; - } - - get maxDeliveryAttempts(): number { - return this.#opts.maxDeliveryAttempts; - } - - /** - * Send nothing to the Dot's guest until the returned function is called. For a caller that stores a row and - * then has to do something to the guest that the row's meaning depends on: the config an approval's "always - * allow" changed has to be there before the guest hears the approval, or the approved call's next use of the - * permission is asked again. Take the hold before the row is stored: a flush that is already running - * waits for it before every row it sends, so no later flush or timer can overtake the work. Several holds - * on one Dot add up. The hold is in memory: after a restart, the guest's next READY does what the caller - * would have done. - */ - hold(dotId: string): () => void { - let entry = this.#holds.get(dotId); - if (!entry) { - let release!: () => void; - const open = new Promise((resolve) => { - release = resolve; - }); - entry = { holders: 0, open, release }; - this.#holds.set(dotId, entry); - } - entry.holders++; - const held = entry; - let released = false; - return () => { - if (released) return; - released = true; - if (--held.holders === 0) { - this.#holds.delete(dotId); - held.release(); - } - }; - } - - async #unheld(dotId: string): Promise { - for (let entry = this.#holds.get(dotId); entry; entry = this.#holds.get(dotId)) await entry.open; - } - - /** - * Send the Dot's pending rows. A call while a flush runs makes that flush - * look once more when it is done, so nothing stored meanwhile is missed. - * Resolves when no flush of the Dot is running any more. - */ - kick(dotId: string): Promise { - if (this.#closed) return Promise.resolve(); - const running = this.#flushing.get(dotId); - if (running) { - this.#again.add(dotId); - return running; - } - const flush = (async () => { - try { - do { - this.#again.delete(dotId); - await this.#flush(dotId); - } while (this.#again.has(dotId) && !this.#closed); - } catch (error) { - this.#log.error("inbound delivery failed", { dotId, error: errorMessage(error) }); - } finally { - this.#flushing.delete(dotId); - } - })(); - this.#flushing.set(dotId, flush); - return flush; - } - - /** Kick every Dot whose oldest pending row is due for another try (the periodic pass and the start). */ - async kickDue(): Promise { - for (const dotId of await this.#db.inbound.dueDots(this.#opts.maxDeliveryAttempts)) void this.kick(dotId); - } - - /** Whether any flush is running. */ - get busy(): boolean { - return this.#flushing.size > 0; - } - - /** Whether a flush of the Dot is running. */ - isFlushing(dotId: string): boolean { - return this.#flushing.has(dotId); - } - - /** Wait for every flush running now (tests and shutdown). */ - async settle(): Promise { - while (this.#flushing.size > 0) await Promise.allSettled([...this.#flushing.values()]); - } - - close(): void { - this.#closed = true; - for (const timer of this.#timers) clearTimeout(timer); - this.#timers.clear(); - } - - async #flush(dotId: string): Promise { - await this.#unheld(dotId); - let rows = await this.#db.inbound.pending(dotId); - if (rows.length === 0) return; - await this.#provisioned(dotId); - try { - await this.#lifecycle.ensureReady(dotId); - } catch (error) { - await this.#notReady(dotId, rows, error); - return; - } - // Rows stored while the Dot woke up are sent in the same pass. - rows = await this.#db.inbound.pending(dotId); - for (const row of rows) { - await this.#unheld(dotId); - if (this.#closed) return; - if ((await this.#db.inbound.beginSend(row.event.id)) === "skip") { - this.#log.info("inbound event not sent: its task ended first", { dotId, id: row.event.id, type: row.event.type }); - continue; - } - try { - await (await this.#lifecycle.guest(dotId)).postEvent(row.event); - } catch (error) { - this.#lifecycle.markSuspect(dotId); - await this.#sendFailed(row, error); - return; - } - await this.#db.inbound.markDelivered(row.event.id); - await this.#db.computers.touch(dotId, this.#clock.now()); - this.#log.info("inbound event delivered", { dotId, id: row.event.id, type: row.event.type }); - } - } - - /** The Dot could not be made READY: every pending row had one more failed attempt. */ - async #notReady(dotId: string, rows: InboundRecord[], error: unknown): Promise { - const reason = errorMessage(error); - this.#log.warn("inbound events wait: the Dot is not READY", { dotId, pending: rows.length, error: reason }); - for (const row of rows) await this.#countFailure(row, reason); - } - - async #sendFailed(row: InboundRecord, error: unknown): Promise { - const reason = errorMessage(error); - if (!knownUndelivered(error)) { - // The guest may hold it: never counted, sent again with the same id. - this.#log.warn("inbound event outcome unknown, will send it again", { dotId: row.dot_id, id: row.event.id, error: reason }); - const retryAt = this.#retryAt(1); - await this.#db.inbound.postpone(row.event.id, reason, retryAt); - this.#schedule(row.dot_id, this.#opts.retryDelayMs); - return; - } - const status = guestErrorStatus(error); - if (status >= 400 && status < 500) { - // The guest refused this event as such: sending it again cannot change that. - const code = guestErrorCode(error) ?? String(status); - this.#log.error("the guest refused an inbound event; it is dropped", { dotId: row.dot_id, id: row.event.id, error: reason }); - await this.#giveUp(row, `refused by the guest (${code}): ${reason}`); - return; - } - await this.#countFailure(row, reason); - } - - async #countFailure(row: InboundRecord, reason: string): Promise { - const failures = await this.#db.inbound.recordFailure(row.event.id, reason, this.#retryAt(row.failures + 1)); - if (failures >= this.#opts.maxDeliveryAttempts) { - if (row.event.type === "task.created") { - await this.#giveUp(row, `could not be delivered to the Dot after ${failures} attempts: ${reason}`); - } else { - // A message or a decision is never given up: the next READY sends it. - this.#log.warn("inbound event waits for the next READY", { dotId: row.dot_id, id: row.event.id, failures }); - } - return; - } - this.#schedule(row.dot_id, this.#opts.retryDelayMs * failures); - } - - /** - * Drop a row for good; a task.created takes its task with it, logged as `task.failed` in the same transaction like - * every task state change. A task that ended meanwhile (cancelled) keeps its end: only the row is dropped. - */ - async #giveUp(row: InboundRecord, reason: string): Promise { - const taskId = row.event.type === "task.created" ? row.task_id : null; - if (taskId) { - const logged = await this.#db - .transaction(async (tx) => { - // The event first: its insert takes the event-order lock (database events.ts). - const event = await this.#events.appendHostIn(tx, row.dot_id, "task.failed", { task_id: taskId, error: reason }); - if (!(await tx.tasks.transition(taskId, "FAILED", { error: reason }))) throw new TaskAlreadyEnded(); - await tx.inbound.drop(row.event.id, reason); - return event; - }) - .catch((error: unknown) => { - if (error instanceof TaskAlreadyEnded) return null; - throw error; - }); - if (logged) { - this.#events.publish(logged); - this.#log.error("task delivery failed for good", { taskId, error: reason }); - this.#onTaskSettled(); - return; - } - } - await this.#db.inbound.drop(row.event.id, reason); - } - - /** scheduled_at style: compared with the database's now(), so a zero delay is "no date" rather than this host's now. */ - #retryAt(failures: number): Date | null { - const delay = this.#opts.retryDelayMs * failures; - return delay > 0 ? new Date(Date.now() + delay) : null; - } - - #schedule(dotId: string, delayMs: number): void { - if (this.#closed) return; - const timer = setTimeout(() => { - this.#timers.delete(timer); - void this.kick(dotId); - }, delayMs); - timer.unref(); - this.#timers.add(timer); - } -} - -export { TASK_ENDED_BEFORE_DELIVERY }; diff --git a/apps/scheduler/src/index.ts b/apps/scheduler/src/index.ts deleted file mode 100644 index 4d6571a4..00000000 --- a/apps/scheduler/src/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -export * from "./dispatcher.js"; -export * from "./driver.js"; -export * from "./guest-events.js"; -export * from "./inbound.js"; -export * from "./lifecycle.js"; -export * from "./scheduler.js"; -export * from "./support.js"; diff --git a/apps/scheduler/src/lifecycle.ts b/apps/scheduler/src/lifecycle.ts deleted file mode 100644 index 64872c0c..00000000 --- a/apps/scheduler/src/lifecycle.ts +++ /dev/null @@ -1,878 +0,0 @@ -/** - * The life of a Dot's computer on the host: create (section 9.4), the READY - * procedure (9.3), sleep and wake (9.5), reboot, delete, the guest event - * pump, and reconciliation with the running QEMU processes (found through - * their pid files) after a control plane restart. - * - * Every operation that changes a computer's state runs under a per-Dot lock, - * so a wake and an idle sleep of the same Dot can never interleave. A Dot is - * in `#ready` only between a completed READY procedure and the moment an - * operation under that lock takes it out again, so anything that skips the - * lock because the Dot is READY (a delivery, a push) can only meet a stop - * that has not started yet: every stop takes the Dot out of `#ready` first. - */ -import { VM_PROXY_NAME, type Database, type Repositories } from "@invisible-dots/database"; -import type { EventLog } from "@invisible-dots/events"; -import { - COMPUTER_STOPPED, - computerIsUp, - computerResources, - PREPARE_SLEEP_TIMEOUT_MS, - toRuntimeConfig, - type DotState, - type HealthAnswer, - type HostEventDataMap, - type HostEventType, - type OutboundEvent, - type RefusedEvent, - type StopReason, - type StoredEvent, - type VmState, -} from "@invisible-dots/shared"; -import { guestErrorCode, guestErrorStatus, type ComputerDriver, type ComputerSpecInput, type ComputerState, type GuestApi } from "./driver.js"; -import { applyGuestEvent, dotStatusForAgent, type AppliedGuestEvent } from "./guest-events.js"; -import { abortableSleep, ControlPlaneError, errorMessage, type Clock, type Logger } from "@invisible-dots/shared"; -import { BackgroundWork, KeyedMutex } from "./support.js"; - -export interface LifecycleOptions { - /** How long a started VM may take to pass the READY procedure. Default 10 minutes (a first boot runs cloud-init). */ - readyTimeoutMs: number; - /** Interval between health polls while waiting for READY. Default 2 s. */ - healthPollMs: number; - /** Per-request timeout of a health poll. Default 5 s. */ - healthRequestTimeoutMs: number; - /** First retry delay of the event pump after a failure; doubles up to `pumpMaxRetryMs`. */ - pumpRetryMs: number; - pumpMaxRetryMs: number; - /** Time the agent gets to flush its state before a shutdown (section 9.5). Default PREPARE_SLEEP_TIMEOUT_MS of packages/shared, which the engine's own steps fit inside. */ - prepareSleepTimeoutMs: number; - /** - * How long before an automation is due a stopped computer is started, and how close to one a running computer is - * kept awake (section 9.5). It is the time a start takes to reach READY with some to spare; the passes that look - * for the Dots to start run every few seconds, so the computer is up between this long and a pass before the run. - * Default 90 s. - */ - automationWakeLeadMs: number; -} - -export const DEFAULT_LIFECYCLE_OPTIONS: LifecycleOptions = { - readyTimeoutMs: 10 * 60_000, - healthPollMs: 2_000, - healthRequestTimeoutMs: 5_000, - pumpRetryMs: 1_000, - pumpMaxRetryMs: 30_000, - prepareSleepTimeoutMs: PREPARE_SLEEP_TIMEOUT_MS, - automationWakeLeadMs: 90_000, -}; - -export interface LifecycleDeps { - db: Database; - events: EventLog; - driver: ComputerDriver; - clock: Clock; - logger: Logger; - options?: Partial; - /** Called when a task reached a terminal state or a Dot became READY: the dispatcher may have work to hand out. */ - onWorkPossible?: () => void; - /** Called every time a Dot becomes READY: what waits in its inbound outbox can go now. */ - onReady?: (dotId: string) => void; -} - -/** Why a stop is asked for: the idle sleep, or the person (`exited` is only ever recorded, never asked for). */ -export type StopRequest = Exclude; - -/** Part of the error a Dot gets when READY failed only for want of a key; setting a key retries those Dots. */ -export const MISSING_KEY_MESSAGE = "no OpenRouter API key is configured"; - -/** The READY procedure failed; the message says which step and why. */ -export class NotReadyError extends ControlPlaneError { - constructor(dotId: string, message: string) { - super(503, "computer_not_ready", `Dot ${dotId}: ${message}`); - this.name = "NotReadyError"; - } -} - -/** - * A failed guest call described by its status and code only. Used where the - * request carried a secret: the error text of a guest or a proxy may echo - * the body, and this message goes into logs, `dots.error` and the event log. - */ -function describeWithoutBody(error: unknown): string { - const status = guestErrorStatus(error); - const code = guestErrorCode(error) ?? (error as { code?: unknown })?.code; - return status === 0 ? `the guest was not reached${typeof code === "string" ? ` (${code})` : ""}` : `status ${status}${typeof code === "string" ? `, ${code}` : ""}`; -} - -/** Store a host event through the transaction a change is made in (`Lifecycle.#commit`). */ -type HostLog = (dotId: string, type: K, data: HostEventDataMap[K]) => Promise; - -export class Lifecycle { - readonly #db: Database; - readonly #events: EventLog; - readonly #driver: ComputerDriver; - readonly #clock: Clock; - readonly #log: Logger; - readonly #opts: LifecycleOptions; - readonly #onWorkPossible: () => void; - readonly #onReady: (dotId: string) => void; - readonly #mutex = new KeyedMutex(); - /** Pushes of the key and the config to a READY guest, one at a time per Dot, so the last one sent is the newest. */ - readonly #pushes = new KeyedMutex(); - /** Bumped by every change of what a guest must hold; READY completes only on an unchanged generation. */ - readonly #generation = new Map(); - /** Dots whose READY procedure completed since their computer last started. */ - readonly #ready = new Set(); - readonly #pumps = new Map(); - readonly #pumpDone = new Map>(); - readonly #background: BackgroundWork; - #closed = false; - - constructor(deps: LifecycleDeps) { - this.#db = deps.db; - this.#events = deps.events; - this.#driver = deps.driver; - this.#clock = deps.clock; - this.#log = deps.logger; - this.#background = new BackgroundWork(this.#log); - this.#opts = { ...DEFAULT_LIFECYCLE_OPTIONS, ...deps.options }; - this.#onWorkPossible = deps.onWorkPossible ?? (() => {}); - this.#onReady = deps.onReady ?? (() => {}); - } - - isReady(dotId: string): boolean { - return this.#ready.has(dotId); - } - - /** Whether a lifecycle operation is queued or running for the Dot. */ - isBusy(dotId: string): boolean { - return this.#mutex.isBusy(dotId); - } - - /** - * The guest of a running computer, reached through the port recorded at its - * start. The port is read for every call: it changes with every start. - */ - async guest(dotId: string): Promise { - const computer = await this.#db.computers.get(dotId); - if (!computer) throw new ControlPlaneError(404, "not_found", `Dot ${dotId} has no computer`); - if (computer.guest_port === null) { - throw new ControlPlaneError(409, COMPUTER_STOPPED, `the computer of Dot ${dotId} is ${computer.state}, it has no guest port`); - } - return this.#driver.guest({ dotId, port: computer.guest_port }, await this.#db.computers.token(dotId)); - } - - /** - * Run `write` in one transaction, and publish what it logged after COMMIT, in order. A change of state and the host - * events that tell it commit together or not at all: a control plane killed between two writes leaves the state it - * was in before the change (which recovery knows how to finish), never a state nobody was told of. `log` stores a host - * event through the transaction. Like every transaction that inserts events and changes other rows, `write` logs first - * and writes its rows after (the event insert takes the event-order lock, database events.ts). - */ - async #commit(write: (tx: Repositories, log: HostLog) => Promise): Promise { - const { value, logged } = await this.#db.transaction(async (tx) => { - const logged: StoredEvent[] = []; - const log: HostLog = async (dotId, type, data) => { - logged.push(await this.#events.appendHostIn(tx, dotId, type, data)); - }; - return { value: await write(tx, log), logged }; - }); - for (const event of logged) this.#events.publish(event); - return value; - } - - /** Log `computer.state` when the computer's row holds another state now; the caller writes the row after its events. */ - async #logState(tx: Repositories, log: HostLog, dotId: string, state: VmState): Promise { - if ((await tx.computers.get(dotId))?.state !== state) await log(dotId, "computer.state", { state }); - } - - /** Log `dot.updated` for the status ERROR (the other statuses are told by the events of what caused them); the caller writes the row after its events. */ - async #logDotStatus(tx: Repositories, log: HostLog, dotId: string, status: DotState, error: string | null): Promise { - const dot = status === "ERROR" ? await tx.dots.get(dotId) : null; - if (dot) await log(dotId, "dot.updated", { name: dot.name, status, error }); - } - - async #setVmState(dotId: string, state: VmState, lastError?: string | null, stopReason?: StopReason): Promise { - await this.#commit(async (tx, log) => { - await this.#logState(tx, log, dotId, state); - await tx.computers.setState(dotId, state, lastError, stopReason); - }); - } - - async #setDotStatus(dotId: string, status: DotState, error: string | null = null): Promise { - await this.#commit(async (tx, log) => { - await this.#logDotStatus(tx, log, dotId, status, error); - await tx.dots.setStatus(dotId, status, error); - }); - } - - /** - * The computer is up: its process, its images and its state RUNNING, with `computer.state` and `computer.started`, - * in one write. From here on the guest is reached through this port. - */ - async #recordStarted( - dotId: string, - started: { guestPort: number; pid: number; runtimeImage: string }, - goldenImage: string, - extra: Record, - ): Promise { - await this.#commit(async (tx, log) => { - await this.#logState(tx, log, dotId, "RUNNING"); - await log(dotId, "computer.started", { guest_port: started.guestPort, runtime_image: started.runtimeImage, ...extra }); - await tx.computers.setProcess(dotId, { guestPort: started.guestPort, pid: started.pid }); - await tx.computers.setImages(dotId, goldenImage, started.runtimeImage); - await tx.computers.setState(dotId, "RUNNING", null); - }); - } - - /** The computer and the Dot are in ERROR for `message`, with `computer.state` and `dot.updated`, in one write. */ - async #recordError(dotId: string, message: string): Promise { - await this.#commit(async (tx, log) => { - await this.#logState(tx, log, dotId, "ERROR"); - await this.#logDotStatus(tx, log, dotId, "ERROR", message); - await tx.computers.setState(dotId, "ERROR", message); - await tx.dots.setStatus(dotId, "ERROR", message); - }); - } - - /** - * The computer is off: no process, the state STOPPED with the reason, `computer.state` and `computer.stopped`, and - * the Dot idle, in one write. Recovery, which finds a computer off that was not recorded as such, leaves a Dot that - * is disabled or in ERROR as it is (`keepFailedDot`). - */ - async #recordStopped(dotId: string, reason: StopReason, forced: boolean, keepFailedDot = false): Promise { - await this.#commit(async (tx, log) => { - await this.#logState(tx, log, dotId, "STOPPED"); - await log(dotId, "computer.stopped", { reason, forced }); - const dot = await tx.dots.get(dotId); - await tx.computers.setProcess(dotId, null); - await tx.computers.setState(dotId, "STOPPED", null, reason); - if (dot && !(keepFailedDot && dot.status === "ERROR")) await tx.dots.setStatus(dotId, "IDLE", null); - }); - } - - /** Record a lifecycle failure on both rows, log it, and rethrow it. */ - async #fail(dotId: string, step: string, error: unknown): Promise { - const message = `${step} failed: ${errorMessage(error)}`; - this.#log.error(message, { dotId }); - await this.#recordError(dotId, message).catch(() => {}); - throw error instanceof ControlPlaneError ? error : new ControlPlaneError(502, "computer_error", `Dot ${dotId}: ${message}`); - } - - async #spec(dotId: string): Promise { - const [dot, computer] = await Promise.all([this.#db.dots.get(dotId), this.#db.computers.get(dotId)]); - if (!dot || !computer) throw new ControlPlaneError(404, "not_found", `Dot ${dotId} has no computer`); - const proxy = await this.#db.secrets.get(dotId, VM_PROXY_NAME); - return { - dotId, - token: await this.#db.computers.token(dotId), - ...(proxy ? { proxy } : {}), - resources: computerResources(dot.config), - goldenImage: computer.golden_image, - state: computer.state, - }; - } - - /** Section 9.4: provision the computer of a Dot inserted as CREATING, start it and bring it to READY. */ - provision(dotId: string): Promise { - return this.#mutex.run(dotId, async () => { - const spec = await this.#spec(dotId); - this.#log.info("provisioning computer", { dotId }); - let created; - try { - created = await this.#driver.create(spec); - } catch (error) { - return this.#fail(dotId, "create", error); - } - await this.#db.computers.setImages(dotId, created.goldenImage, created.runtimeImage); - await this.#startLocked(dotId); - }); - } - - /** - * Make sure the Dot can take work: start a stopped computer, finish the - * READY procedure of a running one that is not READY yet. Resolves once - * READY; throws (and leaves the Dot in ERROR) when that fails. - */ - async ensureReady(dotId: string): Promise { - if (this.#ready.has(dotId)) return; - await this.#mutex.run(dotId, () => this.#ensureReadyLocked(dotId)); - } - - async #ensureReadyLocked(dotId: string): Promise { - if (this.#ready.has(dotId)) return; - const computer = await this.#db.computers.get(dotId); - if (!computer) throw new ControlPlaneError(404, "not_found", `Dot ${dotId} has no computer`); - if (computer.state === "DELETING") throw new ControlPlaneError(409, "dot_deleting", `Dot ${dotId} is being deleted`); - if (computer.state === "PROVISIONING") { - throw new ControlPlaneError(409, "dot_provisioning", `Dot ${dotId} is still being provisioned`); - } - const vm = await this.#driver.state(dotId); - if (vm.state === "RUNNING" && vm.guestPort !== null) { - await this.#adoptRunning(dotId, { ...vm, guestPort: vm.guestPort }); - } else { - await this.#startLocked(dotId); - } - } - - /** - * Start a stopped computer for an automation that is due, if it still is one: the pass that chose it read the - * database before this turn of the per-Dot lock came, and the person may have stopped the computer since (a stop - * that is not started for its automations), or the guest reported another time. The rule is read again, under the - * lock, from the same place the pass read it (`stoppedDotsWithAutomationDue`); true when the computer was started. - */ - async wakeForAutomation(dotId: string): Promise { - if (this.#ready.has(dotId)) return false; - return this.#mutex.run(dotId, async () => { - if (!(await this.stoppedDotsWithAutomationDue()).includes(dotId)) return false; - await this.#ensureReadyLocked(dotId); - return true; - }); - } - - async #startLocked(dotId: string): Promise { - const spec = await this.#spec(dotId); - if (!spec.goldenImage) { - return this.#fail(dotId, "start", new Error("the computer has no golden image recorded; it was never created")); - } - await this.#setVmState(dotId, "STARTING"); - let started; - try { - started = await this.#driver.start({ ...spec, goldenImage: spec.goldenImage }); - } catch (error) { - await this.#db.computers.setProcess(dotId, null).catch(() => {}); - return this.#fail(dotId, "start", error); - } - await this.#recordStarted(dotId, started, spec.goldenImage, started.alreadyRunning ? { already_running: true } : {}); - await this.#readyProcedure(dotId); - } - - /** - * A running VM this process did not start (after a control plane restart, - * or after a failed READY). Its port and pid are what QEMU and its pid - * file say now; the row is brought in line with them when it differs. - */ - async #adoptRunning(dotId: string, vm: ComputerState & { guestPort: number }): Promise { - const computer = await this.#db.computers.get(dotId); - const pid = vm.pid ?? computer?.pid ?? null; - if (pid === null) { - return this.#fail(dotId, "attach to the running computer", new Error("QEMU runs but neither its pid file nor the database names its pid")); - } - if (computer?.guest_port !== vm.guestPort || computer?.pid !== pid) { - this.#log.info("recording the running VM's process", { dotId, guestPort: vm.guestPort, pid }); - await this.#db.computers.setProcess(dotId, { guestPort: vm.guestPort, pid }); - } - await this.#setVmState(dotId, "RUNNING", null); - await this.#readyProcedure(dotId); - } - - #generationOf(dotId: string): number { - return this.#generation.get(dotId) ?? 0; - } - - /** - * Push what the guest must hold: the OpenRouter key and the secrets the - * config's MCP servers name (memory only in the guest, so lost with every - * agent restart), then the runtime config, all read from the database now. - * The one place that does this, for READY, for a changed key, secret or - * config, and after an agent restart. - */ - async #push(dotId: string, guest: GuestApi): Promise { - const key = await this.#db.secrets.openRouterKey(dotId); - if (!key) throw new NotReadyError(dotId, `${MISSING_KEY_MESSAGE}; set one with PUT /api/secrets/openrouter`); - const dot = await this.#db.dots.get(dotId); - if (!dot) throw new ControlPlaneError(404, "not_found", `Dot ${dotId} not found`); - const mcpSecrets = await this.#db.secrets.mcpSecrets(dotId, dot.config.mcp_servers); - try { - await guest.pushSecrets({ openrouter_api_key: key, mcp_secrets: mcpSecrets }); - } catch (error) { - // Never the guest's own words here: whatever answered may have echoed a secret. - throw new Error(`the guest did not take the OpenRouter key and the MCP secrets (${describeWithoutBody(error)})`); - } - await guest.putConfig(toRuntimeConfig(dot.config)); - } - - /** - * Section 9.3: wait for dot-agentd and the agent, push the OpenRouter key, - * push the runtime config, then require the agent to report the key and - * every self-check. Only then is the Dot READY and its events pumped. - */ - async #readyProcedure(dotId: string): Promise { - const computer = await this.#db.computers.get(dotId); - if (computer?.guest_port == null) { - return this.#fail(dotId, "READY", new Error("the computer has no guest port recorded")); - } - // A pump of an earlier start reads a port that is gone and never comes back on its own. - await this.#stopPump(dotId); - const endpoint = { dotId, port: computer.guest_port }; - const token = await this.#db.computers.token(dotId); - const guest = this.#driver.guest(endpoint, token); - let health: HealthAnswer; - try { - health = await this.#driver.waitForHealth(endpoint, token, { - timeoutMs: this.#opts.readyTimeoutMs, - intervalMs: this.#opts.healthPollMs, - requestTimeoutMs: this.#opts.healthRequestTimeoutMs, - }); - } catch (error) { - if ((error as { status?: unknown }).status === 401) { - return this.#fail(dotId, "READY", new Error("the guest refused the Dot token (401)")); - } - return this.#fail(dotId, "READY (waiting for the guest)", error); - } - - let agentState; - for (;;) { - const generation = this.#generationOf(dotId); - try { - await this.#push(dotId, guest); - health = await guest.health({ timeoutMs: this.#opts.healthRequestTimeoutMs }); - } catch (error) { - return this.#fail(dotId, error instanceof NotReadyError ? "READY" : "READY (secret and config push)", error); - } - const problems: string[] = []; - if (health.agent.status !== "ok") problems.push(`agent status ${health.agent.status}`); - else { - if (!health.agent.openrouter_configured) problems.push("the agent does not report the OpenRouter key as configured"); - const checks = health.agent.checks; - if (!checks) problems.push("the agent reports no self-checks"); - else { - if (!checks.filesystem_writable) problems.push("filesystem not writable"); - if (!checks.network_reachable) problems.push("network not reachable"); - if (!checks.browser_installed) problems.push("browser layer not installed"); - } - } - if (problems.length > 0) { - return this.#fail(dotId, "READY", new NotReadyError(dotId, problems.join("; "))); - } - try { - agentState = await guest.state(); - } catch (error) { - return this.#fail(dotId, "READY (agent state)", error); - } - // A key or a config stored while this procedure pushed was refused by - // syncGuest (the Dot was not READY yet): push again, then decide. - // The comparison and the add below run without an await between them. - if (this.#generationOf(dotId) === generation) break; - this.#log.info("key or config changed during READY, pushing again", { dotId }); - } - this.#ready.add(dotId); - await this.#db.computers.touch(dotId, this.#clock.now()); - // The status comes from the state snapshot before the pump starts, so a - // pumped agent.state, which is newer than the snapshot, always wins. - await this.#setDotStatus(dotId, dotStatusForAgent(agentState.state)); - this.#startPump(dotId); - this.#log.info("dot ready", { dotId, agent: agentState.state }); - this.#onReady(dotId); - this.#onWorkPossible(); - } - - /** - * Take the Dot out of READY before anything that ends its computer's run: - * from here on every delivery and every push goes through the per-Dot - * lock, which the caller holds. A push already on its way is waited for. - * Returns whether the Dot was READY. - */ - async #leaveReady(dotId: string): Promise { - const wasReady = this.#ready.delete(dotId); - await this.#pushes.run(dotId, async () => {}); - return wasReady; - } - - /** - * Whether an idle sleep may still go ahead, read again under the per-Dot - * lock: the Dot is READY with an IDLE agent and has no work. Work that - * arrived after the idle check (a claimed task, a message, an approval) - * is pending in the database by now, also when its send is in flight. - */ - async #stillIdle(dotId: string): Promise { - const [dot, computer] = await Promise.all([this.#db.dots.get(dotId), this.#db.computers.get(dotId)]); - if (dot?.status !== "READY" || computer?.state !== "RUNNING") return false; - return !(await this.keepsAwake(dotId)); - } - - /** - * Whether the Dot has something that makes an idle sleep pointless (section 9.5): work (`tasks.hasWork`), or an - * automation due within the wake lead time, past ones included (the guest makes a missed run when it starts, and a - * computer put to sleep now would be started again at once). - */ - async keepsAwake(dotId: string): Promise { - const now = this.#clock.now(); - if (await this.#db.tasks.hasWork(dotId, now)) return true; - const dueAt = (await this.#db.computers.get(dotId))?.next_automation_at; - return dueAt != null && Date.parse(dueAt) <= now.getTime() + this.#opts.automationWakeLeadMs; - } - - /** - * The Dots whose computer is stopped and has an automation due within the wake lead time: the ones to start so - * their guest is ready when the run is due (or, for a run already missed, makes it as it starts). - */ - stoppedDotsWithAutomationDue(): Promise { - return this.#db.computers.stoppedWithAutomationBy(new Date(this.#clock.now().getTime() + this.#opts.automationWakeLeadMs)); - } - - /** - * Put the Dot to sleep, or stop it on the user's request (section 9.5). The reason is recorded with the computer: a - * computer the person stopped is not started for its automations until the person starts it again, also when it was - * already asleep. - */ - stop(dotId: string, reason: StopRequest): Promise { - return this.#mutex.run(dotId, () => this.#stopLocked(dotId, reason, true)); - } - - /** - * The stop, with the per-Dot lock held. An idle stop that is asked for now checks again that the Dot is idle - * (`recheckIdle`); one that recovery finds half done does not: the shutdown began before the control plane went - * down, the guest may already have taken prepare-sleep, and calling it off would leave a computer that runs without - * being READY. It is finished, and the pass wakes the computer if the work that came meanwhile needs it. - */ - async #stopLocked(dotId: string, reason: StopRequest, recheckIdle: boolean): Promise { - const computer = await this.#db.computers.get(dotId); - if (!computer) throw new ControlPlaneError(404, "not_found", `Dot ${dotId} has no computer`); - if (computer.state === "STOPPED") { - // The person's stop of a computer that is asleep: it is not started for its automations from now on, which every - // view of the computer (its automations note) learns from the event, as it learns of any other stop. - if (reason === "user" && computer.stop_reason !== "user") { - await this.#commit(async (tx, log) => { - await log(dotId, "computer.stopped", { reason, forced: false }); - await tx.computers.setState(dotId, "STOPPED", undefined, "user"); - }); - } - return; - } - if (computer.state === "DELETING" || computer.state === "PROVISIONING") { - throw new ControlPlaneError(409, "invalid_state", `Dot ${dotId} cannot be stopped while ${computer.state}`); - } - const wasReady = await this.#leaveReady(dotId); - if (reason === "idle" && recheckIdle && !(await this.#stillIdle(dotId))) { - if (wasReady) this.#ready.add(dotId); - this.#log.info("idle sleep called off: work arrived meanwhile", { dotId }); - return; - } - this.#log.info("stopping computer", { dotId, reason }); - await this.#setVmState(dotId, "STOPPING", undefined, reason); - if (wasReady) { - try { - const guest = await this.guest(dotId); - await guest.prepareSleep(this.#opts.prepareSleepTimeoutMs); - } catch (error) { - // The shutdown still happens: the guest flushes its own state on SIGTERM too. - this.#log.warn("prepare-sleep failed, shutting down anyway", { dotId, error: errorMessage(error) }); - } - } - await this.#stopPump(dotId); - let result; - try { - result = await this.#driver.stop(dotId, await this.#db.computers.token(dotId)); - } catch (error) { - return this.#fail(dotId, "stop", error); - } - await this.#recordStopped(dotId, reason, result.forced); - } - - /** - * A reboot is a clean stop and a start of the same VM (the agent flushes - * its state first, as before a sleep), so a new runtime ISO and new - * resources apply. The port changes; the READY procedure runs against the - * new one. - */ - reboot(dotId: string): Promise { - return this.#mutex.run(dotId, async () => { - const computer = await this.#db.computers.get(dotId); - if (!computer) throw new ControlPlaneError(404, "not_found", `Dot ${dotId} has no computer`); - if (!computerIsUp(computer.state)) { - throw new ControlPlaneError(409, COMPUTER_STOPPED, `Dot ${dotId} cannot be rebooted while ${computer.state}`); - } - const spec = await this.#spec(dotId); - if (!spec.goldenImage) { - return this.#fail(dotId, "reboot", new Error("the computer has no golden image recorded; it was never created")); - } - if (await this.#leaveReady(dotId)) { - try { - await (await this.guest(dotId)).prepareSleep(this.#opts.prepareSleepTimeoutMs); - } catch (error) { - this.#log.warn("prepare-sleep before reboot failed, rebooting anyway", { dotId, error: errorMessage(error) }); - } - } - await this.#stopPump(dotId); - await this.#setVmState(dotId, "STARTING"); - let started; - try { - started = await this.#driver.reboot({ ...spec, goldenImage: spec.goldenImage }); - } catch (error) { - await this.#db.computers.setProcess(dotId, null).catch(() => {}); - return this.#fail(dotId, "reboot", error); - } - await this.#recordStarted(dotId, started, spec.goldenImage, { reboot: true }); - await this.#readyProcedure(dotId); - }); - } - - /** Destroy the VM and its disk, then remove the Dot's rows (the event log keeps its history). */ - remove(dotId: string): Promise { - return this.#mutex.run(dotId, async () => { - const dot = await this.#db.dots.get(dotId); - if (!dot) return; - await this.#setVmState(dotId, "DELETING"); - await this.#leaveReady(dotId); - await this.#stopPump(dotId); - try { - await this.#driver.destroy(dotId); - } catch (error) { - return this.#fail(dotId, "delete", error); - } - await this.#commit(async (tx, log) => { - await log(dotId, "dot.deleted", { name: dot.name }); - await tx.dots.delete(dotId); - }); - this.#log.info("dot deleted", { dotId, name: dot.name }); - }); - } - - /** - * The key or the config the guest must hold changed (a PATCH, a new key, - * an agent that restarted and lost its key): push both to a READY guest - * now. A Dot that is not READY gets them from its READY procedure, which - * sees the change through the generation and pushes again if it was - * already past its own push. True when this call pushed. A push that fails - * may mean the guest is gone, so the Dot is suspect (`markSuspect`): its next - * delivery checks READY again, which pushes both again. - */ - async syncGuest(dotId: string): Promise { - this.#generation.set(dotId, this.#generationOf(dotId) + 1); - if (!this.#ready.has(dotId)) return false; - return this.#pushes.run(dotId, async () => { - if (!this.#ready.has(dotId)) return false; - try { - await this.#push(dotId, await this.guest(dotId)); - } catch (error) { - this.markSuspect(dotId); - throw error; - } - return true; - }); - } - - readyDots(): string[] { - return [...this.#ready]; - } - - /** A failed call means the guest may be gone: the next delivery re-checks READY instead of trusting it. */ - markSuspect(dotId: string): void { - this.#ready.delete(dotId); - } - - /** Whether work this class started in the background is still running. */ - get busy(): boolean { - return this.#background.busy; - } - - /** Wait for the work this class started in the background (tests and shutdown). */ - async settle(): Promise { - await this.#background.settle(); - } - - #startPump(dotId: string): void { - if (this.#pumps.has(dotId) || this.#closed) return; - const controller = new AbortController(); - this.#pumps.set(dotId, controller); - const done = this.#pump(dotId, controller.signal).finally(() => { - if (this.#pumps.get(dotId) === controller) this.#pumps.delete(dotId); - if (this.#pumpDone.get(dotId) === done) this.#pumpDone.delete(dotId); - }); - this.#pumpDone.set(dotId, done); - } - - async #stopPump(dotId: string): Promise { - this.#pumps.get(dotId)?.abort(); - await this.#pumpDone.get(dotId); - } - - /** - * Consume the guest's outbound stream from the stored cursor. Every event - * is stored, applied and the cursor advanced in ONE transaction, so a crash - * between two events loses nothing and applies nothing twice. On a failure - * the loop starts over from the cursor in the database, unless the VM's - * QEMU is gone: then the pump ends and the computer is recorded as stopped. - */ - async #pump(dotId: string, signal: AbortSignal): Promise { - let delay = this.#opts.pumpRetryMs; - while (!signal.aborted) { - try { - const computer = await this.#db.computers.get(dotId); - if (!computer) return; - const guest = await this.guest(dotId); - this.#log.debug("event pump connecting", { dotId, after: computer.event_cursor }); - for await (const event of guest.events({ after: computer.event_cursor, signal, onRefused: (info) => this.#refused(dotId, info) })) { - if (signal.aborted) return; - await this.handleGuestEvent(dotId, event); - delay = this.#opts.pumpRetryMs; - } - if (signal.aborted) return; - this.#log.warn("guest event stream ended", { dotId }); - } catch (error) { - if (signal.aborted) return; - this.#log.warn("event pump failed, retrying", { dotId, error: errorMessage(error), retryMs: delay }); - } - if (await this.#qemuGone(dotId)) { - this.#background.run("recording a VM that stopped by itself", dotId, () => this.#exited(dotId)); - return; - } - await abortableSleep(delay, signal); - delay = Math.min(delay * 2, this.#opts.pumpMaxRetryMs); - } - } - - async #qemuGone(dotId: string): Promise { - try { - return (await this.#driver.state(dotId)).state === "STOPPED"; - } catch { - return false; - } - } - - /** - * The QEMU of a READY Dot exited without a stop from the control plane: the - * guest powered itself off, or QEMU crashed or was killed. The computer is - * recorded as stopped; when the Dot still has work (`keepsAwake`: a task, or - * an automation due within the wake lead time), it is started again so its - * guest picks that work up (section 9.5). - */ - async #exited(dotId: string): Promise { - const recorded = await this.#mutex.run(dotId, async () => { - const computer = await this.#db.computers.get(dotId); - // A stop, reboot or start under the lock already dealt with it. - if (computer?.state !== "RUNNING" || !(await this.#qemuGone(dotId))) return false; - this.#ready.delete(dotId); - await this.#stopPump(dotId); - this.#log.warn("the VM stopped without being asked to", { dotId }); - await this.#recordStopped(dotId, "exited", false); - return true; - }); - if (recorded && (await this.keepsAwake(dotId))) { - this.#log.info("the Dot still has work, starting its computer again", { dotId }); - this.#background.run("restart after an unexpected stop", dotId, () => this.ensureReady(dotId)); - } - } - - /** - * The guest sent a message the host's schema refused. It is not stored as the guest's event (it is not one), but it is - * not lost with a log line either: a `guest.event.refused` host event says that it happened and what was wrong, and - * the cursor moves past it in the same transaction, so a reconnect does not read it again. An engine and a host that - * disagree on an event would otherwise leave the Dot with a state nobody can see (an approval that is never shown). - */ - async #refused(dotId: string, info: RefusedEvent): Promise { - const now = this.#clock.now(); - const stored = await this.#db.transaction(async (tx) => { - const logged = await this.#events.appendHostIn(tx, dotId, "guest.event.refused", info); - if (info.seq !== null) await tx.computers.advanceCursor(dotId, info.seq, now); - return logged; - }); - this.#events.publish(stored); - } - - /** Store and apply one guest event; exported for recovery tools and tests. */ - async handleGuestEvent(dotId: string, event: OutboundEvent): Promise { - const now = this.#clock.now(); - const { stored, applied, logged } = await this.#db.transaction(async (tx) => { - // The event first: its insert takes the event-order lock (events.ts). - const stored = await this.#events.appendGuest(tx, dotId, event); - await tx.computers.advanceCursor(dotId, event.seq, now); - const applied: AppliedGuestEvent = stored ? await applyGuestEvent(tx, dotId, event) : { taskSettled: false }; - const logged: StoredEvent[] = []; - for (const host of applied.hostEvents ?? []) logged.push(await this.#events.appendHostIn(tx, dotId, host.type, host.data)); - return { stored, applied, logged }; - }); - if (stored) this.#events.publish(stored); - for (const event of logged) this.#events.publish(event); - if (applied.taskSettled) this.#onWorkPossible(); - if (applied.reconfigured) { - // The Dot added an MCP server to its config: the guest gets the config (and the secrets it names) now. - this.#background.run("pushing the config with the MCP server the Dot added", dotId, () => this.syncGuest(dotId)); - } - if (stored && event.type === "agent.started") { - // The agent process restarted inside a running VM: it lost the key it keeps in memory. - this.#background.run("pushing the key again after an agent restart", dotId, () => this.syncGuest(dotId)); - } - return stored; - } - - /** - * After a control plane restart: bring the database in line with the QEMU - * processes that are actually running (pid files and process liveness, - * never the rows), - * finish interrupted operations, and run the READY procedure for every - * computer that is still running. Returns once each Dot's recovery has - * started; the slow parts continue in the background. - */ - async recover(): Promise[]> { - const background: Promise[] = []; - const track = (dotId: string, what: string, work: Promise) => - background.push(work.catch((error) => this.#log.error(`recovery: ${what} failed`, { dotId, error: errorMessage(error) }))); - - for (const computer of await this.#db.computers.list()) { - const dotId = computer.dot_id; - if (computer.state === "PROVISIONING") { - this.#log.info("recovery: resuming provisioning", { dotId }); - track(dotId, "provisioning", this.provision(dotId)); - continue; - } - if (computer.state === "DELETING") { - this.#log.info("recovery: resuming deletion", { dotId }); - track(dotId, "deletion", this.remove(dotId)); - continue; - } - let vm; - try { - vm = await this.#driver.state(dotId); - } catch (error) { - this.#log.error("recovery: cannot read the VM state", { dotId, error: errorMessage(error) }); - continue; - } - if (!vm.exists) { - await this.#db.computers.setProcess(dotId, null); - await this.#recordError(dotId, "the VM disk is missing"); - continue; - } - if (vm.state === "RUNNING") { - if (computer.state === "STOPPING") { - this.#log.info("recovery: finishing an interrupted stop", { dotId }); - // The reason the stop was asked for was recorded with STOPPING; setting RUNNING clears it. - await this.#db.computers.setState(dotId, "RUNNING"); - const reason = computer.stop_reason === "idle" ? "idle" : "user"; - track(dotId, "stop", this.#mutex.run(dotId, () => this.#stopLocked(dotId, reason, false))); - } else if (vm.guestPort === null) { - await this.#recordError(dotId, "QEMU runs but has no forward to the guest port, so the guest cannot be reached"); - } else { - // The port comes from the pid file, so a row that missed it (the - // control plane stopped between the spawn and the update) is - // corrected here; the READY procedure then checks the guest. - const running = { ...vm, guestPort: vm.guestPort }; - this.#log.info("recovery: reattaching to a running computer", { dotId, was: computer.state, pid: vm.pid }); - track(dotId, "reattach", this.#mutex.run(dotId, () => this.#adoptRunning(dotId, running))); - } - } else if (vm.state === "STOPPED") { - if (computer.state !== "STOPPED") { - this.#log.info("recovery: computer is off", { dotId, was: computer.state }); - // A stop that was under way keeps its reason; a VM that went off with the control plane down exited by itself. - // Either way the stop is told now: the control plane went down before it could. - await this.#recordStopped(dotId, computer.stop_reason ?? "exited", false, true); - } else if (computer.guest_port !== null || computer.pid !== null) { - await this.#db.computers.setProcess(dotId, null); - } - } else { - await this.#recordError(dotId, `the VM is ${vm.state}${vm.detail ? `: ${vm.detail}` : ""}`); - } - } - return background; - } - - /** Stop every pump and release the driver's host resources. VMs keep running. */ - async close(): Promise { - this.#closed = true; - this.#background.close(); - for (const controller of this.#pumps.values()) controller.abort(); - await Promise.all(this.#pumpDone.values()); - await this.settle(); - await this.#driver.close(); - } -} diff --git a/apps/scheduler/src/scheduler.ts b/apps/scheduler/src/scheduler.ts deleted file mode 100644 index 5fc78b5b..00000000 --- a/apps/scheduler/src/scheduler.ts +++ /dev/null @@ -1,922 +0,0 @@ -/** - * The control plane's operations, one method per thing the API can ask for. - * The API is a thin HTTP layer over this class; everything that touches the - * database, the VM layer or a guest happens here. - */ -import { randomBytes } from "node:crypto"; -import { DotChangedError, DotNameTakenError, GLOBAL_SCOPE, mcpSecretName, OPENROUTER_KEY_NAME, VM_PROXY_NAME, type Database } from "@invisible-dots/database"; -import { EventLog, USER_MESSAGE_EVENT } from "@invisible-dots/events"; -import type { - AcceptedAnswer, - ApprovalRecord, - ComputerAnswer, - ConversationMessage, - CreateTaskRequest, - DotRecord, - DotSummary, - MessageAnswer, - TaskRecord, - UsageAnswer, -} from "@invisible-dots/shared"; -import { - APPROVAL_NOTE_MAX, - checkHomePath, - checkMcpSecret, - checkOpenRouterKey, - COMPUTER_STOPPED, - ControlPlaneError, - computerIsUp, - computerResources, - CONVERSATION_LIST_LIMIT, - DotConfigError, - errorMessage, - isIdentityAnswer, - isStoredEventType, - MAX_HOST_FILE_BYTES, - newId, - notFound, - parseDotConfig, - parseMessageOrigin, - mcpServerNames, - parseSize, - permissionNames, - silentLogger, - systemClock, - TASK_CANCELLED_SYSTEM_EVENT, - TERMINAL_TASK_STATES, - vmName, - type ApprovalStatus, - type BrowserIdentity, - type Clock, - type CreateBrowserIdentityRequest, - type DotConfig, - type FilesListAnswer, - type InboundEvent, - type ListOrder, - type Logger, - type McpSecretsAnswer, - type MessageOrigin, - type StoredEvent, - type SystemAnswer, - type Skill, - type TaskState, - type ToolListAnswer, -} from "@invisible-dots/shared"; -import { Dispatcher } from "./dispatcher.js"; -import { guestErrorCode, guestErrorStatus, type ComputerDriver, type GuestApi } from "./driver.js"; -import { InboundDelivery, type InboundDeliveryOptions } from "./inbound.js"; -import { Lifecycle, MISSING_KEY_MESSAGE, type LifecycleOptions } from "./lifecycle.js"; -import { BackgroundWork } from "./support.js"; - -export interface SchedulerOptions { - db: Database; - driver: ComputerDriver; - events?: EventLog; - clock?: Clock; - logger?: Logger; - /** How often PENDING tasks are looked for besides the immediate pass on every change. Default 5 s. */ - dispatchIntervalMs?: number; - /** How often idle Dots are looked for. Default 30 s. */ - idleCheckIntervalMs?: number; - lifecycle?: Partial; - /** How inbound events (tasks, messages, approvals) are retried, and when a task's delivery gives up. */ - dispatcher?: Partial; -} - -const TOKEN_BYTES = 32; - -/** A path the files routes may read, normalized; the 400 of the rule of `checkHomePath` otherwise. */ -function homePath(raw: unknown): string { - const checked = checkHomePath(raw); - if (!checked.ok) throw new ControlPlaneError(400, "invalid_path", checked.problem); - return checked.path; -} - -export class Scheduler { - readonly db: Database; - readonly events: EventLog; - readonly lifecycle: Lifecycle; - readonly dispatcher: Dispatcher; - /** Sends what is stored for the guests in `inbound_events`, waking Dots that sleep. */ - readonly inbound: InboundDelivery; - readonly #clock: Clock; - readonly #log: Logger; - readonly #dispatchIntervalMs: number; - readonly #idleCheckIntervalMs: number; - readonly #background: BackgroundWork; - readonly #provisioning = new Map>(); - readonly #timers: NodeJS.Timeout[] = []; - #started = false; - - constructor(options: SchedulerOptions) { - this.db = options.db; - this.#clock = options.clock ?? systemClock; - this.#log = options.logger ?? silentLogger; - this.#background = new BackgroundWork(this.#log); - this.events = - options.events ?? new EventLog(options.db.events, (line) => this.#log.warn(line)); - this.#dispatchIntervalMs = options.dispatchIntervalMs ?? 5_000; - this.#idleCheckIntervalMs = options.idleCheckIntervalMs ?? 30_000; - this.lifecycle = new Lifecycle({ - db: options.db, - events: this.events, - driver: options.driver, - clock: this.#clock, - logger: this.#log, - options: options.lifecycle, - onWorkPossible: () => void this.dispatcher.dispatch(), - onReady: (dotId) => void this.inbound.kick(dotId), - }); - this.inbound = new InboundDelivery({ - db: options.db, - events: this.events, - lifecycle: this.lifecycle, - clock: this.#clock, - logger: this.#log, - provisioned: async (dotId) => { - await this.#provisioning.get(dotId)?.catch(() => {}); - }, - onTaskSettled: () => void this.dispatcher.dispatch(), - options: options.dispatcher, - }); - this.dispatcher = new Dispatcher(options.db, this.#clock, this.#log, (dotId) => void this.inbound.kick(dotId)); - } - - /** - * Recover from the previous run, then start the timers. Nothing is - * requeued: a task claimed before the restart still has its task.created - * in the outbox, and the guest ignores it if an earlier send reached it. - */ - async start(): Promise { - if (this.#started) return; - this.#started = true; - for (const work of await this.lifecycle.recover()) this.#background.track(work); - this.#background.track(this.pass()); - const every = (ms: number, fn: () => Promise) => { - const timer = setInterval(() => void fn().catch((e) => this.#log.error("timer failed", { error: errorMessage(e) })), ms); - timer.unref(); - this.#timers.push(timer); - }; - every(this.#dispatchIntervalMs, () => this.pass()); - every(this.#idleCheckIntervalMs, () => this.idleCheck()); - } - - /** - * One round of looking for work (section 9.5): claim due tasks, send the - * inbound rows whose retry time came, wake stopped Dots whose new work - * waits behind a task their guest has not finished, and wake the ones whose - * next automation is due within the wake lead time. - */ - async pass(): Promise { - await this.dispatcher.dispatch(); - await this.inbound.kickDue(); - for (const dotId of await this.db.tasks.stoppedDotsWithBlockedWork()) { - if (this.lifecycle.isBusy(dotId)) continue; - this.#log.info("waking a stopped Dot: new work waits behind its unfinished task", { dotId }); - this.#background.run("wake for blocked work", dotId, () => this.lifecycle.ensureReady(dotId)); - } - for (const dotId of await this.lifecycle.stoppedDotsWithAutomationDue()) { - if (this.lifecycle.isBusy(dotId)) continue; - this.#log.info("waking a stopped Dot: an automation is due", { dotId }); - this.#background.run("wake for an automation", dotId, () => this.lifecycle.wakeForAutomation(dotId)); - } - } - - /** Stop the timers and the event pumps and wait for in-flight work. VMs keep running. */ - async close(): Promise { - for (const timer of this.#timers.splice(0)) clearInterval(timer); - this.inbound.close(); - await this.settle().catch(() => {}); - await this.lifecycle.close(); - } - - /** Wait until every background operation started so far has finished (used by tests and shutdown). */ - async settle(): Promise { - for (;;) { - await this.dispatcher.settle(); - await this.inbound.settle(); - await this.lifecycle.settle(); - if (!this.#background.busy) { - // Each of the three may have started work in another while it settled. - if (!this.dispatcher.busy && !this.inbound.busy && !this.lifecycle.busy) return; - continue; - } - await this.#background.settle(); - } - } - - // Dots - - async requireDot(idOrName: string): Promise { - const dot = await this.db.dots.resolve(idOrName); - if (!dot) throw notFound("Dot", idOrName); - return dot; - } - - listDots(): Promise { - return this.db.dots.list(); - } - - #parseConfig(input: unknown): DotConfig { - if (input === undefined || input === null) { - throw new ControlPlaneError(400, "invalid_request", "the body needs a config: YAML text or an object"); - } - try { - return parseDotConfig(input); - } catch (error) { - if (error instanceof DotConfigError) throw new ControlPlaneError(400, "invalid_config", error.message, error.issues); - throw error; - } - } - - /** - * Section 9.4: insert the Dot as CREATING with its computer row and a new - * token in one transaction, then provision it in the background (overlay, - * seed, port, QEMU, READY). - */ - async createDot(configInput: unknown): Promise { - const config = this.#parseConfig(configInput); - const id = newId("dot"); - const token = randomBytes(TOKEN_BYTES).toString("base64url"); - let dot: DotRecord; - try { - const created = await this.db.transaction(async (tx) => { - // The events first: their insert takes the event-order lock (database events.ts). They commit with the rows. - const logged = [ - await this.events.appendHostIn(tx, id, "dot.created", { name: config.name }), - await this.events.appendHostIn(tx, id, "computer.state", { state: "PROVISIONING" }), - ]; - const record = await tx.dots.insert({ id, config, status: "CREATING" }); - await tx.computers.insert({ dotId: id, vmName: vmName(id), state: "PROVISIONING", token }); - return { record, logged }; - }); - dot = created.record; - for (const event of created.logged) this.events.publish(event); - } catch (error) { - if (error instanceof DotNameTakenError) throw new ControlPlaneError(409, "name_taken", error.message); - throw error; - } - this.#log.info("dot created", { dotId: id, name: config.name }); - const provisioning = this.lifecycle.provision(id); - this.#provisioning.set(id, provisioning); - this.#background.run("provisioning", id, () => - provisioning.finally(() => { - this.#provisioning.delete(id); - }), - ); - return dot; - } - - /** - * Replace the Dot's config. `expectedConfigVersion` (the `config_version` of the Dot as the caller read it) makes the - * save conditional: when the config changed since (the person answered "Always allow" in another view, another - * save), it is a 409 `dot_changed` and nothing is written, so a form opened before cannot silently undo what - * happened after. A change of the Dot's status (a task turn, an approval waiting) is not a change of the config. - */ - async updateDot(idOrName: string, configInput: unknown, expectedConfigVersion?: unknown): Promise { - if (expectedConfigVersion !== undefined && (typeof expectedConfigVersion !== "number" || !Number.isInteger(expectedConfigVersion) || expectedConfigVersion < 1)) { - throw new ControlPlaneError(400, "invalid_request", "expected_config_version must be the config_version of the Dot, a positive integer"); - } - const current = await this.requireDot(idOrName); - const config = this.#parseConfig(configInput); - if (parseSize(config.computer.disk) < parseSize(current.config.computer.disk)) { - throw new ControlPlaneError( - 409, - "disk_shrink", - `computer.disk cannot shrink from ${current.config.computer.disk} to ${config.computer.disk}: the filesystem on it would be destroyed`, - ); - } - let saved: { updated: DotRecord; logged: StoredEvent }; - try { - saved = await this.db.transaction(async (tx) => { - // The event first (database events.ts); it commits with the config it tells of, or neither does. - const logged = await this.events.appendHostIn(tx, current.id, "dot.updated", { name: config.name }); - const updated = await tx.dots.updateConfig(current.id, config, expectedConfigVersion); - if (!updated) throw notFound("Dot", idOrName); - // A secret of a server the config no longer declares, or no longer names, goes with it. - await tx.secrets.deleteUndeclaredMcpSecrets(current.id, config.mcp_servers); - return { updated, logged }; - }); - } catch (error) { - if (error instanceof DotChangedError) { - throw new ControlPlaneError(409, "dot_changed", `Dot ${current.name} changed after you read it: read it again and apply the change to what it is now`); - } - if (error instanceof DotNameTakenError) throw new ControlPlaneError(409, "name_taken", error.message); - throw error; - } - this.events.publish(saved.logged); - await this.#pushToGuest(saved.updated); - return saved.updated; - } - - /** - * The Dot's saved config changed (a PATCH, an "always allow"; `dot.updated` was logged with the change), or a secret of - * its MCP servers did: push the secrets and the config to the guest. A failed push does not fail the change: the Dot - * is suspect then (`syncGuest`), and both are pushed again on its next READY. - */ - async #pushToGuest(dot: DotRecord): Promise { - try { - await this.lifecycle.syncGuest(dot.id); - } catch (error) { - this.#log.warn("saved, but the push to the guest failed; it is pushed again on the next READY", { - dotId: dot.id, - error: errorMessage(error), - }); - } - } - - async deleteDot(idOrName: string): Promise { - const dot = await this.requireDot(idOrName); - this.#background.run("deletion", dot.id, () => this.lifecycle.remove(dot.id)); - return { accepted: true }; - } - - // Messages - - /** - * Log the message and store it for the guest in one transaction, so a - * message the person saw accepted always reaches the Dot, also across a - * failed wake or a control plane restart. `origin` names the channel chat - * the message came from; it is kept in the event log (the one place a reply - * is routed back by) and never sent to the guest. Only code inside the - * control plane passes one: the HTTP route does not take it. - */ - async sendMessage(idOrName: string, text: unknown, origin?: MessageOrigin): Promise { - if (typeof text !== "string" || text.trim() === "") { - throw new ControlPlaneError(400, "invalid_request", "text must be a non-empty string"); - } - if (origin !== undefined && parseMessageOrigin(origin) === null) { - throw new ControlPlaneError(400, "invalid_request", "origin must be a channel, a binding id, a chat id and an external id"); - } - const dot = await this.requireDot(idOrName); - const messageId = newId("msg"); - const event: InboundEvent<"user.message"> = { - id: messageId, - type: "user.message", - ts: this.#clock.now().toISOString(), - data: { text }, - }; - const stored = await this.db.transaction(async (tx) => { - // The event first: its insert takes the event-order lock (database events.ts). A channel message - // already logged is not stored again, so what a redelivery finds is exactly what the first delivery - // committed: the message and its queue row together, or neither. - const logged = await this.events.appendUserMessageIn(tx, dot.id, { message_id: messageId, text, ...(origin && { origin }) }); - if (logged) await tx.inbound.enqueue(dot.id, event); - return logged; - }); - if (!stored) return this.#answerRedelivery(dot.id, origin!); - this.events.publish(stored); - const delivery = await this.#deliver(dot.id, event.id); - return { message_id: messageId, event_id: stored.id, delivery }; - } - - /** The answer for a channel message that was already handed to the Dot: the same message, delivered if it is not yet. */ - async #answerRedelivery(dotId: string, origin: MessageOrigin): Promise { - const logged = await this.events.userMessageOfOrigin(dotId, origin.binding_id, origin.external_id); - const messageId = logged?.data.message_id; - if (!logged || typeof messageId !== "string") throw new Error(`a channel message is stored twice but cannot be found: ${origin.binding_id}`); - return { message_id: messageId, event_id: logged.id, delivery: await this.#deliver(dotId, messageId) }; - } - - /** - * The Dot's one conversation, oldest first; `order: "desc"` the newest first, and `before` (the event id of the - * oldest message of the previous page) pages on, older, from there. A page holds at most CONVERSATION_LIST_LIMIT. - */ - async conversation(idOrName: string, page: { limit?: number; order?: ListOrder; before?: number } = {}): Promise { - const dot = await this.requireDot(idOrName); - const events = await this.events.query({ - dotId: dot.id, - types: [USER_MESSAGE_EVENT, "message.assistant"], - limit: page.limit ?? CONVERSATION_LIST_LIMIT, - order: page.order, - before: page.before, - }); - return events.map((e) => { - // StoredEvent.type lists the contract's event types; the user side is logged as USER_MESSAGE_EVENT. - const user = (e.type as string) === USER_MESSAGE_EVENT; - const origin = user ? parseMessageOrigin(e.data.origin) : null; - return { - event_id: e.id, - role: user ? "user" : "assistant", - text: String(e.data.text ?? ""), - in_reply_to: typeof e.data.in_reply_to === "string" ? e.data.in_reply_to : null, - ...(origin && { origin }), - created_at: e.created_at, - }; - }); - } - - /** - * Send what is stored for the Dot. A READY Dot gets it before this - * returns ("delivered"); otherwise it is sent once the Dot is woken, in the - * background ("queued"), and stays stored until then. - */ - async #deliver(dotId: string, eventId: string): Promise<"delivered" | "queued"> { - if (!this.lifecycle.isReady(dotId)) { - void this.inbound.kick(dotId); - return "queued"; - } - await this.inbound.kick(dotId); - return (await this.db.inbound.get(eventId))?.delivered_at ? "delivered" : "queued"; - } - - // Tasks - - async createTask(idOrName: string, body: Partial): Promise { - const dot = await this.requireDot(idOrName); - if (typeof body.description !== "string" || body.description.trim() === "") { - throw new ControlPlaneError(400, "invalid_request", "description must be a non-empty string"); - } - if (body.priority !== undefined && (!Number.isInteger(body.priority) || Math.abs(body.priority) > 1_000_000)) { - throw new ControlPlaneError(400, "invalid_request", "priority must be an integer from -1000000 to 1000000"); - } - let scheduledAt: Date | null = null; - if (body.scheduled_at !== undefined && body.scheduled_at !== null) { - scheduledAt = new Date(body.scheduled_at); - if (typeof body.scheduled_at !== "string" || Number.isNaN(scheduledAt.getTime())) { - throw new ControlPlaneError(400, "invalid_request", "scheduled_at must be an ISO 8601 timestamp"); - } - } - const id = newId("task"); - const { description } = body; - const priority = body.priority ?? 0; - const { task, logged } = await this.db.transaction(async (tx) => { - // The event first (database events.ts); it commits with the task it tells of, or neither does. - const logged = await this.events.appendHostIn(tx, dot.id, "task.created", { task_id: id, description, priority }); - const task = await tx.tasks.insert({ id, dotId: dot.id, description, priority, scheduledAt }); - return { task, logged }; - }); - this.events.publish(logged); - void this.dispatcher.dispatch(); - return task; - } - - /** The Dot's tasks, the newest created first, at most TASK_LIST_LIMIT a page; `before` (the id of the last task of the previous page) goes on, older. */ - async listTasks(idOrName: string, page: { limit?: number; before?: string } = {}): Promise { - const dot = await this.requireDot(idOrName); - return this.db.tasks.listByDot(dot.id, page); - } - - /** The tasks of every Dot that ended in a terminal `status` after `after`, the latest end first: one read for all of them. */ - async listEndedTasks(status: unknown, after: unknown): Promise { - if (!TERMINAL_TASK_STATES.includes(status as TaskState)) { - throw new ControlPlaneError(400, "invalid_request", `status must be one of ${TERMINAL_TASK_STATES.join(", ")}`); - } - const since = typeof after === "string" ? new Date(after) : new Date(Number.NaN); - if (Number.isNaN(since.getTime())) throw new ControlPlaneError(400, "invalid_request", "finished_after must be an ISO 8601 time"); - return this.db.tasks.listEnded(status as TaskState, since); - } - - async getTask(id: string): Promise { - const task = await this.db.tasks.get(id); - if (!task) throw notFound("task", id); - return task; - } - - /** - * Cancel a task. A PENDING one simply never runs. Otherwise, in the same - * transaction: when no send of its task.created ever began, that event is - * dropped and the guest never hears of the task; when one began, the - * guest may hold the task, so `system.event { name: "task.cancelled" }` - * is stored behind it and reaches the guest even if it sleeps now (the - * contract has no dedicated inbound cancel). The previous state comes from - * the update itself, never from an earlier read. - */ - async cancelTask(id: string): Promise { - const task = await this.getTask(id); - if (TERMINAL_TASK_STATES.includes(task.status)) { - throw new ControlPlaneError(409, "task_finished", `task ${id} is already ${task.status}`); - } - const { logged, cancelled, tellGuest } = await this.db.transaction(async (tx) => { - // The event first: its insert takes the event-order lock (database events.ts). - const logged = await this.events.appendHostIn(tx, task.dot_id, "task.cancelled", { task_id: id }); - const moved = await tx.tasks.transition(id, "CANCELLED", { error: "cancelled by the user" }); - if (!moved) throw new ControlPlaneError(409, "task_finished", `task ${id} finished meanwhile`); - let tellGuest = false; - if (moved.previous !== "PENDING" && !(await tx.inbound.dropUnsent(id, "the task was cancelled before it was delivered"))) { - const event: InboundEvent<"system.event"> = { - id: newId("evt"), - type: "system.event", - ts: this.#clock.now().toISOString(), - data: { name: TASK_CANCELLED_SYSTEM_EVENT, data: { task_id: id } }, - }; - await tx.inbound.enqueue(task.dot_id, event, { taskId: id }); - tellGuest = true; - } - return { logged, cancelled: moved.task, tellGuest }; - }); - this.events.publish(logged); - if (tellGuest) void this.inbound.kick(task.dot_id); - void this.dispatcher.dispatch(); - return cancelled; - } - - // Computer - - async computer(idOrName: string): Promise { - const dot = await this.requireDot(idOrName); - const computer = await this.db.computers.get(dot.id); - if (!computer) throw notFound("computer of Dot", idOrName); - const ready = this.lifecycle.isReady(dot.id); - let system: SystemAnswer | null = null; - if (ready) { - // Live figures are a convenience of this answer: a guest that is slow or - // briefly unreachable must not turn the whole route into an error. - try { - system = await (await this.lifecycle.guest(dot.id)).system(); - } catch (error) { - this.#log.debug("live system figures unavailable", { dotId: dot.id, error: errorMessage(error) }); - } - } - return { ...computer, ready, system }; - } - - async startComputer(idOrName: string): Promise { - const dot = await this.requireDot(idOrName); - if (dot.computer_state === "PROVISIONING" || dot.computer_state === "DELETING") { - throw new ControlPlaneError(409, "invalid_state", `the computer is ${dot.computer_state}`); - } - this.#background.run("start", dot.id, () => this.lifecycle.ensureReady(dot.id)); - return { accepted: true }; - } - - async stopComputer(idOrName: string): Promise { - const dot = await this.requireDot(idOrName); - if (dot.computer_state === "PROVISIONING" || dot.computer_state === "DELETING") { - throw new ControlPlaneError(409, "invalid_state", `the computer is ${dot.computer_state}`); - } - this.#background.run("stop", dot.id, () => this.lifecycle.stop(dot.id, "user")); - return { accepted: true }; - } - - async rebootComputer(idOrName: string): Promise { - const dot = await this.requireDot(idOrName); - if (!computerIsUp(dot.computer_state)) { - throw new ControlPlaneError(409, COMPUTER_STOPPED, `the computer is ${dot.computer_state ?? "missing"}`); - } - this.#background.run("reboot", dot.id, () => this.lifecycle.reboot(dot.id)); - return { accepted: true }; - } - - /** A guest for a Dot whose computer is running; 409 computer_stopped otherwise (section 9.6). */ - async #runningGuest(idOrName: string): Promise<{ dotId: string; guest: GuestApi }> { - const dot = await this.requireDot(idOrName); - if (!computerIsUp(dot.computer_state)) { - throw new ControlPlaneError( - 409, - COMPUTER_STOPPED, - `the computer of Dot ${dot.name} is ${dot.computer_state ?? "missing"}; start it first`, - ); - } - return { dotId: dot.id, guest: await this.lifecycle.guest(dot.id) }; - } - - /** - * Run a guest call and turn its failure into an API error: the guest's own 4xx passes through, and so do the - * engine's coded identity answers that are not 4xx (`IDENTITY_ERROR_STATUS` of packages/shared: `busy` and the browser's own failures): a coded answer is the engine - * speaking, not a silence, so the UI can tell a busy or crashed browser from an unreachable computer. - */ - async #guestCall(dotId: string, what: string, call: () => Promise): Promise { - try { - return await call(); - } catch (error) { - const status = guestErrorStatus(error); - const code = guestErrorCode(error); - if ((status >= 400 && status < 500 && status !== 401) || isIdentityAnswer(code, status)) { - throw new ControlPlaneError(status, code ?? "guest_error", errorMessage(error)); - } - this.#log.warn("guest call failed", { dotId, what, error: errorMessage(error) }); - throw new ControlPlaneError(502, "guest_unavailable", `${what}: the Dot's computer did not answer: ${errorMessage(error)}`); - } - } - - async screenshot(idOrName: string): Promise { - const { dotId, guest } = await this.#runningGuest(idOrName); - return this.#guestCall(dotId, "screenshot", () => guest.screenshot()); - } - - /** The files of a directory under /home/dot, as dot-agentd lists them; `path` defaults to home itself. */ - async listFiles(idOrName: string, path: string = "~"): Promise { - const checked = homePath(path); - const { dotId, guest } = await this.#runningGuest(idOrName); - const { entries } = await this.#guestCall(dotId, "list files", () => guest.listFiles(checked)); - return { path: checked, entries }; - } - - /** The bytes of a file under /home/dot; a file larger than MAX_HOST_FILE_BYTES is a 413. */ - async readFile(idOrName: string, path: unknown): Promise<{ path: string; content: Uint8Array }> { - const checked = homePath(path); - const { dotId, guest } = await this.#runningGuest(idOrName); - const content = await this.#guestCall(dotId, "read a file", () => guest.readFile(checked, { maxBytes: MAX_HOST_FILE_BYTES })); - return { path: checked, content }; - } - - async listIdentities(idOrName: string): Promise { - const { dotId, guest } = await this.#runningGuest(idOrName); - return (await this.#guestCall(dotId, "list browser identities", () => guest.listBrowserIdentities())).identities; - } - - async createIdentity(idOrName: string, body: Partial): Promise { - if (typeof body.name !== "string" || body.name.trim() === "") { - throw new ControlPlaneError(400, "invalid_request", "name must be a non-empty string"); - } - if (body.proxy != null && typeof body.proxy !== "string") { - throw new ControlPlaneError(400, "invalid_request", "proxy must be a string"); - } - const { dotId, guest } = await this.#runningGuest(idOrName); - const request: CreateBrowserIdentityRequest = { name: body.name, ...(body.proxy ? { proxy: body.proxy } : {}) }; - return this.#guestCall(dotId, "create a browser identity", () => guest.createBrowserIdentity(request)); - } - - async getIdentity(idOrName: string, identityId: string): Promise { - const { dotId, guest } = await this.#runningGuest(idOrName); - return this.#guestCall(dotId, "get a browser identity", () => guest.getBrowserIdentity(identityId)); - } - - async deleteIdentity(idOrName: string, identityId: string): Promise { - const { dotId, guest } = await this.#runningGuest(idOrName); - await this.#guestCall(dotId, "delete a browser identity", () => guest.deleteBrowserIdentity(identityId)); - } - - /** The JPEG of an open identity's window: 409 `not_open` when it is closed, 503 `busy` while a call holds it. */ - async identityFrame(idOrName: string, identityId: string): Promise { - const { dotId, guest } = await this.#runningGuest(idOrName); - return this.#guestCall(dotId, "get the frame of a browser identity", () => guest.getBrowserIdentityFrame(identityId)); - } - - async closeIdentity(idOrName: string, identityId: string): Promise { - const { dotId, guest } = await this.#runningGuest(idOrName); - await this.#guestCall(dotId, "close a browser identity", () => guest.closeBrowserIdentity(identityId)); - } - - // The tools and the skills: the Dot's engine keeps both, read through the computer - - /** - * The Dot's tools, each with the permission it exercises and whether the model is offered it now, and the MCP servers - * its config declares with where each is (connecting, connected, failed and why). - */ - async listTools(idOrName: string): Promise { - const { dotId, guest } = await this.#runningGuest(idOrName); - return this.#guestCall(dotId, "list tools", () => guest.listTools()); - } - - /** The Dot's skills, the built-in ones and its own, each with its whole file. */ - async listSkills(idOrName: string): Promise { - const { dotId, guest } = await this.#runningGuest(idOrName); - return (await this.#guestCall(dotId, "list skills", () => guest.listSkills())).skills; - } - - // Approvals - - /** The approvals of one status or several (every one when omitted); `page` is what the approvals repository's `list` takes. */ - async listApprovals( - status?: ApprovalStatus | readonly ApprovalStatus[], - page: { limit?: number; order?: ListOrder; before?: string; dot?: string } = {}, - ): Promise { - const { dot, ...rest } = page; - // Of one Dot (by id or name; the history of a deleted Dot is read by id, as the event log's is), in the database. - return this.db.approvals.list({ status, ...rest, ...(dot === undefined ? {} : { dotId: await this.historyDotId(dot) }) }); - } - - /** - * Record the decision, move the task back to RUNNING and store the - * `approval.received` for the guest, in one transaction: a decision the - * person saw accepted always reaches the guest, also when the Dot sleeps - * and its wake fails or the control plane restarts first. - * - * `always` (an approval only) is "allow this from now on": in the same - * transaction `permissions[]` becomes `allow` in - * the Dot's config, and the config is pushed to the guest before the answer - * is delivered, so what the approved call does next is not asked again: the - * Dot's deliveries are held (`InboundDelivery.hold`) from before the commit - * until the push is over, so no flush overtakes it. A push that fails does - * not fail the answer; the guest gets the config on its next READY. An - * answer that loses the race for the approval (409) changes nothing. - */ - async resolveApproval( - id: string, - decision: "approve" | "reject", - answer: { note?: unknown; always?: unknown } = {}, - ): Promise { - if (answer.note !== undefined && typeof answer.note !== "string") { - throw new ControlPlaneError(400, "invalid_request", "note must be a string"); - } - if (answer.always !== undefined && answer.always !== true) { - throw new ControlPlaneError(400, "invalid_request", "always must be true"); - } - const note = answer.note as string | undefined; - const always = answer.always as true | undefined; - if (note !== undefined && note.length > APPROVAL_NOTE_MAX) { - throw new ControlPlaneError(400, "invalid_request", `note must be at most ${APPROVAL_NOTE_MAX} characters`); - } - if (always && decision !== "approve") { - throw new ControlPlaneError(400, "invalid_request", "always applies to an approval, not to a rejection"); - } - const existing = await this.db.approvals.get(id); - if (!existing) throw notFound("approval", id); - const event: InboundEvent<"approval.received"> = { - id: newId("evt"), - type: "approval.received", - ts: this.#clock.now().toISOString(), - data: { approval_id: id, decision, ...(note !== undefined ? { note } : {}) }, - }; - // From its commit on, any flush may send the answer: the Dot's deliveries wait until the config that "always" - // changed has reached the guest, so the approved call's next use of the permission is not asked again. - const release = always ? this.inbound.hold(existing.dot_id) : undefined; - let resolved: ApprovalRecord; - try { - const stored = await this.db.transaction(async (tx) => { - // The event first: its insert takes the event-order lock (database events.ts). - const logged = await this.events.appendHostIn(tx, existing.dot_id, "approval.resolved", { - approval_id: id, - decision, - ...(existing.task_id !== null ? { task_id: existing.task_id } : {}), - ...(note !== undefined ? { note } : {}), - ...(always ? { always } : {}), - }); - // "Always allow" changes the Dot's config: that is told by the event of every config change, in the same commit. - const dot = always ? await tx.dots.get(existing.dot_id) : null; - if (always && !dot) throw notFound("Dot", existing.dot_id); - // The permission of an MCP server the config no longer declares has nothing left to allow. - if (dot && !permissionNames(dot.config).includes(existing.permission as never)) { - throw new ControlPlaneError(409, "permission_gone", `the Dot's config no longer has the permission ${existing.permission}, so it cannot be allowed always`); - } - const updated = dot ? await this.events.appendHostIn(tx, existing.dot_id, "dot.updated", { name: dot.name }) : null; - const resolved = await tx.approvals.resolve(id, decision === "approve" ? "approved" : "rejected", note ?? null); - if (!resolved) { - const current = await tx.approvals.get(id); - throw new ControlPlaneError(409, "already_resolved", `approval ${id} is already ${current?.status ?? existing.status}`); - } - const reconfigured = always ? await tx.dots.setPermission(resolved.dot_id, resolved.permission, "allow") : null; - if (always && !reconfigured) throw notFound("Dot", resolved.dot_id); - if (resolved.task_id) await tx.tasks.transition(resolved.task_id, "RUNNING", { dotId: resolved.dot_id }); - await tx.inbound.enqueue(resolved.dot_id, event); - return { logged: [logged, ...(updated ? [updated] : [])], resolved, reconfigured }; - }); - resolved = stored.resolved; - for (const logged of stored.logged) this.events.publish(logged); - if (stored.reconfigured) await this.#pushToGuest(stored.reconfigured); - } finally { - release?.(); - } - await this.#deliver(resolved.dot_id, event.id); - return resolved; - } - - // Events and secrets - - /** The id of a Dot whose history is read: a deleted Dot's history stays readable by id. */ - async historyDotId(idOrName: string): Promise { - const dot = await this.db.dots.resolve(idOrName); - const dotId = dot?.id ?? (idOrName.includes("_") ? idOrName : undefined); - if (!dotId) throw notFound("Dot", idOrName); - return dotId; - } - - /** - * The Dot's events, oldest first: those after an id, of these types (an unknown type name is a 400, so a typo - * does not look like a quiet Dot) and of this task. - */ - async listEvents( - idOrName: string, - filter: { after?: number; before?: number; limit?: number; types?: readonly string[]; tools?: readonly string[]; taskId?: string; order?: ListOrder } = {}, - ): Promise { - const unknown = filter.types?.filter((type) => !isStoredEventType(type)) ?? []; - if (unknown.length > 0) throw new ControlPlaneError(400, "invalid_request", `unknown event type: ${unknown.join(", ")}`); - if (filter.taskId === "") throw new ControlPlaneError(400, "invalid_request", "task_id must not be empty"); - const types = filter.types === undefined || filter.types.length === 0 ? undefined : filter.types; - const tools = filter.tools === undefined || filter.tools.length === 0 ? undefined : filter.tools; - return this.events.query({ dotId: await this.historyDotId(idOrName), after: filter.after, before: filter.before, limit: filter.limit, types, tools, taskId: filter.taskId, order: filter.order }); - } - - /** The model spend the Dot's guest reported since `since` (every event when omitted), from the event log. */ - async usage(idOrName: string, since?: Date): Promise { - const dotId = await this.historyDotId(idOrName); - return { dot_id: dotId, since: since?.toISOString() ?? null, spent_usd: await this.db.events.spentUsd(dotId, since) }; - } - - /** - * Set or clear a Dot's VM proxy (`socks5://[user:password@]host:port`; null or blank clears it). The VM uses it from - * its next start: the seed is written at every start. The answer never carries the value. - */ - async setVmProxy(idOrName: string, value: unknown): Promise<{ dot_id: string; proxy: boolean }> { - const dot = await this.requireDot(idOrName); - if (value === null || value === undefined || (typeof value === "string" && value.trim() === "")) { - await this.db.secrets.delete(dot.id, VM_PROXY_NAME); - return { dot_id: dot.id, proxy: false }; - } - if (typeof value !== "string" || !/^socks5:\/\/\S+$/.test(value.trim())) { - throw new ControlPlaneError(400, "invalid_request", "proxy must be a socks5:// URL"); - } - await this.db.secrets.put(dot.id, VM_PROXY_NAME, value.trim()); - return { dot_id: dot.id, proxy: true }; - } - - /** Whether a Dot has a VM proxy; never its value. */ - async vmProxy(idOrName: string): Promise<{ dot_id: string; proxy: boolean }> { - const dot = await this.requireDot(idOrName); - return { dot_id: dot.id, proxy: (await this.db.secrets.get(dot.id, VM_PROXY_NAME)) !== null }; - } - - /** Every secret the Dot's MCP servers name, and whether each is set; never a value. */ - async mcpSecrets(idOrName: string): Promise { - const dot = await this.requireDot(idOrName); - const set = await this.db.secrets.mcpSecrets(dot.id, dot.config.mcp_servers); - const secrets = mcpServerNames(dot.config).flatMap((server) => - dot.config.mcp_servers[server]!.secrets.map((name) => ({ server, name, set: set[server]?.[name] !== undefined })), - ); - return { dot_id: dot.id, secrets }; - } - - /** - * Set the value of a secret an MCP server of the Dot's config names, or clear it with null, and push the secrets to - * the guest when it runs: the server starts again with it. A secret the config does not name is refused, so none is - * stored that the config would not keep. The answer never carries a value. - */ - async setMcpSecret(idOrName: string, server: string, name: string, value: unknown): Promise { - const dot = await this.requireDot(idOrName); - const declared = dot.config.mcp_servers[server]; - if (!declared) throw new ControlPlaneError(404, "not_found", `Dot ${dot.name} declares no MCP server "${server}"`); - if (!declared.secrets.includes(name)) { - throw new ControlPlaneError(404, "not_found", `the MCP server "${server}" of Dot ${dot.name} names no secret "${name}"`); - } - if (value === null) { - await this.db.secrets.delete(dot.id, mcpSecretName(server, name)); - } else { - const checked = checkMcpSecret(value); - if (!checked.ok) throw new ControlPlaneError(400, "invalid_request", checked.problem); - await this.db.secrets.put(dot.id, mcpSecretName(server, name), checked.value); - } - await this.#pushToGuest(dot); - return this.mcpSecrets(dot.id); - } - - /** Store the OpenRouter key (global or per Dot) and push it to the READY guests it applies to. */ - async setOpenRouterKey(value: unknown, dotIdOrName?: unknown): Promise<{ pushed: number }> { - const checked = checkOpenRouterKey(value); - if (!checked.ok) throw new ControlPlaneError(400, "invalid_request", checked.problem); - let scope = GLOBAL_SCOPE; - if (dotIdOrName !== undefined && dotIdOrName !== null) { - if (typeof dotIdOrName !== "string") throw new ControlPlaneError(400, "invalid_request", "dot_id must be a string"); - scope = (await this.requireDot(dotIdOrName)).id; - } - await this.db.secrets.put(scope, OPENROUTER_KEY_NAME, checked.key); - let pushed = 0; - // Every Dot it applies to, READY or not: one whose READY is under way must notice the change too. - const targets = scope === GLOBAL_SCOPE ? (await this.db.dots.list()).map((d) => d.id) : [scope]; - for (const dotId of targets) { - try { - if (await this.lifecycle.syncGuest(dotId)) pushed++; - } catch (error) { - this.#log.warn("could not push the new OpenRouter key", { dotId, error: errorMessage(error) }); - } - } - // Dots that failed READY only for want of a key can be retried now. - for (const dot of await this.db.dots.list()) { - const missedKey = dot.error?.includes(MISSING_KEY_MESSAGE) ?? false; - if (dot.status === "ERROR" && missedKey && (scope === GLOBAL_SCOPE || scope === dot.id)) { - this.#background.run("READY after a new key", dot.id, () => this.lifecycle.ensureReady(dot.id)); - } - } - return { pushed }; - } - - // Idle sleep (section 9.5) - - /** - * Put every Dot to sleep that is READY with an IDLE agent, has no work - * (no due or active task, nothing waiting to reach its guest, no automation - * due within the wake lead time), and was not active for its `idle_timeout`. - * Returns the ids of the Dots it started stopping; the stop checks all of it - * again under the Dot's lock and is called off when work arrived meanwhile. - */ - async idleCheck(): Promise { - const now = this.#clock.now(); - const sleeping: string[] = []; - for (const dotId of this.lifecycle.readyDots()) { - if (this.lifecycle.isBusy(dotId) || this.inbound.isFlushing(dotId)) continue; - const [dot, computer] = await Promise.all([this.db.dots.get(dotId), this.db.computers.get(dotId)]); - if (!dot || !computer || computer.state !== "RUNNING" || dot.status !== "READY") continue; - const timeout = computerResources(dot.config).idleTimeoutMs; - if (timeout === null) continue; - const lastActive = new Date(computer.last_active_at ?? computer.updated_at).getTime(); - if (now.getTime() - lastActive < timeout) continue; - if (await this.lifecycle.keepsAwake(dotId)) continue; - this.#log.info("dot idle, going to sleep", { dotId, idleMs: now.getTime() - lastActive }); - sleeping.push(dotId); - this.#background.run("idle sleep", dotId, () => this.lifecycle.stop(dotId, "idle")); - } - return sleeping; - } - - /** The database answers, and whether the global OpenRouter key is stored (and decrypts). */ - async health(): Promise<{ database: "ok"; openrouter_configured: boolean }> { - let key: string | null; - try { - key = await this.db.secrets.get(GLOBAL_SCOPE, OPENROUTER_KEY_NAME); - } catch (error) { - throw new ControlPlaneError(503, "database_unavailable", `the database did not answer: ${errorMessage(error)}`); - } - return { database: "ok", openrouter_configured: key !== null }; - } -} diff --git a/apps/scheduler/src/support.ts b/apps/scheduler/src/support.ts deleted file mode 100644 index 96e15aff..00000000 --- a/apps/scheduler/src/support.ts +++ /dev/null @@ -1,66 +0,0 @@ -/** The scheduler's own building blocks: a per-key lock, and the work it starts without waiting for it. */ -import { errorMessage, type Logger } from "@invisible-dots/shared"; - -/** - * Serializes work per key: lifecycle operations on one Dot (create, start, - * stop, reboot, delete) run one after the other, operations on different - * Dots run in parallel. - */ -export class KeyedMutex { - readonly #tails = new Map>(); - - run(key: string, fn: () => Promise): Promise { - const previous = this.#tails.get(key) ?? Promise.resolve(); - const result = previous.then(fn, fn); - const tail = result.catch(() => {}); - this.#tails.set(key, tail); - void tail.then(() => { - if (this.#tails.get(key) === tail) this.#tails.delete(key); - }); - return result; - } - - isBusy(key: string): boolean { - return this.#tails.has(key); - } -} - -/** - * Work a caller starts without waiting for it (a provisioning, a stop, a push to a guest): its failure is logged, - * since nobody awaits it, and `settle` waits until all of it has finished (tests and shutdown). After `close` nothing - * new starts. - */ -export class BackgroundWork { - readonly #pending = new Set>(); - #closed = false; - - constructor(private readonly log: Logger) {} - - /** Start `work`; a failure is logged as " failed" with the Dot. */ - run(what: string, dotId: string, work: () => Promise): void { - if (this.#closed) return; - this.track(work().catch((error) => this.log.error(`${what} failed`, { dotId, error: errorMessage(error) }))); - } - - /** Count a promise as background work until it settles; it is returned as it is, for a caller that awaits it. */ - track(work: Promise): Promise { - const tracked = work.finally(() => this.#pending.delete(tracked)); - tracked.catch(() => {}); - this.#pending.add(tracked); - return work; - } - - /** Whether any of it is still running. */ - get busy(): boolean { - return this.#pending.size > 0; - } - - /** Wait until all of it has finished, including work started meanwhile. */ - async settle(): Promise { - while (this.#pending.size > 0) await Promise.allSettled([...this.#pending]); - } - - close(): void { - this.#closed = true; - } -} diff --git a/apps/scheduler/src/testing.ts b/apps/scheduler/src/testing.ts deleted file mode 100644 index 6ab5c763..00000000 --- a/apps/scheduler/src/testing.ts +++ /dev/null @@ -1,879 +0,0 @@ -/** - * In-process stand-ins for the VM layer and the guest agent, used by the - * scheduler and API tests. FakeGuest follows the guest protocol of - * architecture section 5: health that comes up after a boot, a key held in - * memory only (lost on every reboot and every agent restart), `agent.started` - * at every start of the agent, inbound events accepted once per id, an - * outbox with monotonically increasing `seq` replayed after a cursor, and - * browser identities under the same rules as the real agent's, and a small - * file system (`putFile`) served through the files routes with the real - * daemon's errors, the automations its engine runs (`putAutomation`) and a small tool table. - */ -import { crc32, deflateSync } from "node:zlib"; -import { - checkIdentityRequest, - IDENTITY_ERROR_STATUS, - FILE_TOO_LARGE, - mcpPermission, - mcpServerNames, - GUEST_PATHS, - IdentityRequestError, - newId, - newIdentityId, - pollGuestHealth, - type AgentState, - type AgentStateAnswer, - type BrowserIdentity, - type BrowserIdentityListAnswer, - type CreateBrowserIdentityRequest, - type DotRuntimeConfig, - type FileEntry, - type FileListAnswer, - type GuestChecks, - type HealthAnswer, - type IdentityErrorCode, - type InboundEvent, - type OutboundEvent, - type OutboundEventDataMap, - type OutboundEventType, - type PermissionName, - type RefusedEvent, - type SecretsRequest, - type SystemAnswer, - type ToolInfo, - type Skill, - type SkillListAnswer, - type ToolListAnswer, - type VmState, -} from "@invisible-dots/shared"; -import type { - ComputerDriver, - ComputerSpecInput, - ComputerState, - CreatedComputer, - GuestApi, - GuestEndpoint, - StartedComputer, - WaitForHealthOptions, -} from "./driver.js"; - -/** The fake desktop's size: a picture a browser draws with real proportions, so a page that shows it is laid out as it will be. */ -const FAKE_DESKTOP = { width: 1280, height: 800 } as const; - -/** - * A job of the fake's engine, with the fields of nanobot's CronJob that decide when it runs and that a test reads back. - * The host never lists the jobs (they are the Dot's, made and changed through its cron tool); it hears only when the - * earliest is next due (`automation.next_run`). - */ -export interface FakeAutomation { - id: string; - name: string; - enabled: boolean; - schedule: { kind: "at" | "every" | "cron"; at_ms?: number; every_ms?: number; expr?: string; tz?: string }; - message: string; - next_run_at_ms: number | null; - last_run_at_ms: number | null; - last_status: "ok" | "error" | "skipped" | null; - last_error: string | null; - delete_after_run: boolean; - created_at_ms: number; -} - -let desktop: Uint8Array | undefined; - -/** A real PNG of the fake desktop, one flat colour, made once. */ -function desktopPng(): Uint8Array { - if (desktop) return desktop; - const { width, height } = FAKE_DESKTOP; - const stride = 1 + width * 3; - const raw = Buffer.alloc(stride * height); - for (let y = 0; y < height; y++) for (let x = 0; x < width; x++) raw.set([24, 32, 48], y * stride + 1 + x * 3); - const chunk = (type: string, data: Buffer) => { - const head = Buffer.alloc(8); - head.writeUInt32BE(data.length, 0); - head.write(type, 4, "latin1"); - const crc = Buffer.alloc(4); - crc.writeUInt32BE(crc32(Buffer.concat([head.subarray(4), data])), 0); - return Buffer.concat([head, data, crc]); - }; - const ihdr = Buffer.alloc(13); - ihdr.writeUInt32BE(width, 0); - ihdr.writeUInt32BE(height, 4); - ihdr[8] = 8; // bits per channel - ihdr[9] = 2; // RGB - desktop = new Uint8Array(Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), chunk("IHDR", ihdr), chunk("IDAT", deflateSync(raw)), chunk("IEND", Buffer.alloc(0))])); - return desktop; -} - -/** The error shape of vm-manager's GuestRequestError: a status (0 = unreachable) and the guest's code. */ -/** The most browser identities the engine keeps (`DEFAULT_MAX_IDENTITIES` in nanobot/dots/main.py). */ -export const FAKE_MAX_IDENTITIES = 20; - -export class FakeGuestError extends Error { - constructor( - readonly status: number, - message: string, - readonly code?: string, - ) { - super(message); - this.name = "FakeGuestError"; - } -} - -export type InboundHandler = (event: InboundEvent, guest: FakeGuest) => void | Promise; - -/** - * The default agent: a task runs at once and completes; a message gets an - * answer; an approval resumes and completes the task it belongs to. - */ -export const completeEverything: InboundHandler = (event, guest) => { - switch (event.type) { - case "task.created": - guest.emit("agent.state", { state: "THINKING" }); - guest.emit("task.started", { task_id: event.data.task_id }); - guest.emit("task.completed", { task_id: event.data.task_id, summary: `done: ${event.data.description}` }); - guest.emit("agent.state", { state: "IDLE" }); - break; - case "user.message": - guest.emit("message.assistant", { text: `echo: ${event.data.text}`, in_reply_to: event.id }); - break; - case "approval.received": { - const pending = guest.pendingApproval; - guest.pendingApproval = null; - guest.emit("agent.state", { state: "EXECUTING" }); - if (pending?.task_id) { - guest.emit("task.completed", { task_id: pending.task_id, summary: `approval ${event.data.decision}d` }); - } - guest.emit("agent.state", { state: "IDLE" }); - break; - } - default: - break; - } -}; - -export class FakeGuest implements GuestApi { - /** Whether the VM is powered on; every call fails as unreachable when it is not. */ - running = false; - /** Health polls answered with agent "starting" after each boot before it reports "ok". */ - bootPolls = 1; - #pollsSinceBoot = 0; - #bootedAt = 0; - agentState: AgentState = "IDLE"; - openrouterKey: string | null = null; - /** The MCP servers' secrets the host pushed last, by server and name. */ - mcpSecrets: SecretsRequest["mcp_secrets"] = {}; - config: DotRuntimeConfig | null = null; - checks: GuestChecks = { filesystem_writable: true, network_reachable: true, browser_installed: true }; - readonly outbox: OutboundEvent[] = []; - readonly inbound: InboundEvent[] = []; - readonly identities = new Map(); - /** The files of the guest by absolute path; directories exist where a file is under them (and home always). */ - readonly files = new Map(); - /** Symbolic links of the guest: the absolute path of the link to its target (an absolute path). */ - readonly links = new Map(); - /** The automations of the Dot by id (its cron tool made them). */ - readonly automations = new Map(); - /** - * The tools the fake's engine has: the real table's shape, a few rows; `offered` follows the permissions of the - * config, as the engine's `offered_tools` does. - */ - readonly tools: Omit[] = [ - { name: "exec", permission: "computer.exec", description: "Run a shell command on the computer." }, - { name: "read_file", permission: "files.read", description: "Read a file." }, - { name: "write_file", permission: "files.write", description: "Write a file." }, - { name: "cron", permission: "automations", description: "Schedule reminders and recurring tasks." }, - { name: "browser_identity_list", permission: "browser.identity.list", description: "List the browser identities." }, - { name: "browser_identity_create", permission: "browser.identity.create", description: "Create a browser identity." }, - ]; - /** - * The tools each declared MCP server serves, by server; a server of the config with none here serves none. A server - * named in `mcpFailures` is failed with that error instead, as the engine reports a server it could not start. - */ - readonly mcpTools = new Map[]>(); - readonly mcpFailures = new Map(); - /** The skills the fake's engine shows: one built-in by default, as the real one ships; a test adds the Dot's own. */ - skills: Skill[] = [ - { - name: "invisible-playwright", - description: "Use the Dot's browser for any task on a website.", - source: "builtin", - path: "/opt/invisible-dots/engine/skills/invisible-playwright/SKILL.md", - content: "---\nname: invisible-playwright\ndescription: Use the Dot's browser for any task on a website.\n---\n\n# The browser\n", - }, - ]; - readonly calls: string[] = []; - pendingApproval: { approval_id: string; task_id?: string } | null = null; - onInbound: InboundHandler = completeEverything; - /** While set, a frame of an open identity answers 503 `busy`, as when a call of the Dot holds the browser. */ - identityBusy = false; - /** While set, a frame of an open identity answers with this code and its status of the shared table, as when the engine's browser fails. */ - identityFault: IdentityErrorCode | null = null; - /** When set, `postEvent` fails with it once. */ - failNextPost: Error | null = null; - boots = 0; - /** The guest's clock, in ms since the epoch: what its engine compares the jobs' times with at a boot. */ - now: () => number = () => Date.now(); - #seq = 0; - /** Messages the guest sent that the host will refuse, each after the `seq` of the last event the guest had written when it was sent. */ - #refusals: { info: RefusedEvent; after: number }[] = []; - #wakers = new Set<() => void>(); - #streams = new Set(); - - constructor(readonly token: string) {} - - boot(): void { - this.running = true; - this.boots++; - this.#pollsSinceBoot = 0; - this.#bootedAt = Date.now(); - this.openrouterKey = null; - this.mcpSecrets = {}; - this.agentState = "IDLE"; - this.emit("agent.started", {}); - this.#catchUp(); - } - - /** - * What the engine does at a start (CronService.start): a job whose time came while the computer was off runs once, - * late, and moves on (a recurring job counts its next run from now, a one-time one is over), then the engine reports - * the earliest next run when it differs from what the host was last told. - */ - #catchUp(): void { - const now = this.now(); - for (const job of [...this.automations.values()]) { - if (!job.enabled || job.next_run_at_ms === null || job.next_run_at_ms > now) continue; - const ran = { ...job, last_run_at_ms: now, last_status: "ok" as const, last_error: null }; - if (job.schedule.kind === "at") { - if (job.delete_after_run) this.automations.delete(job.id); - else this.automations.set(job.id, { ...ran, enabled: false, next_run_at_ms: null }); - } else { - const every = job.schedule.kind === "every" ? (job.schedule.every_ms ?? 60_000) : 24 * 3_600_000; - this.automations.set(job.id, { ...ran, next_run_at_ms: now + every }); - } - } - this.#reportNextRun(); - } - - /** - * The agent process restarts inside a running VM (systemd after a crash): - * its key is gone, its event stream drops, and it announces its start. - */ - restartAgent(): void { - this.agentRestarts++; - this.openrouterKey = null; - this.mcpSecrets = {}; - this.disconnectStreams(); - this.emit("agent.started", {}); - } - - agentRestarts = 0; - - powerOff(): void { - this.running = false; - this.disconnectStreams(); - } - - /** End every open event stream, as a dropped connection would. */ - disconnectStreams(): void { - for (const controller of this.#streams) controller.abort(); - this.#streams.clear(); - this.#wake(); - } - - #wake(): void { - for (const w of [...this.#wakers]) w(); - } - - #openIdentities(): BrowserIdentity[] { - return [...this.identities.values()].filter((identity) => identity.status === "open"); - } - - /** The Dot opens an identity's browser (the engine does it for a tool call; the host only observes it). */ - launchIdentity(id: string): void { - const identity = this.identities.get(id); - if (!identity) throw new Error(`no identity ${id}`); - this.identities.set(id, { ...identity, status: "open", lastUsedAt: new Date().toISOString() }); - this.emit("browser.identity.launched", { identity_id: id, name: identity.name }); - } - - #reachable(what: string): void { - this.calls.push(what); - if (!this.running) { - if (this.#rebootDelayMs !== null) { - setTimeout(() => this.boot(), this.#rebootDelayMs); - this.#rebootDelayMs = null; - } - throw new FakeGuestError(0, `${what}: connect ECONNREFUSED (the VM is off)`, "ECONNREFUSED"); - } - } - - #rebootDelayMs: number | null = null; - - /** - * Power off and come back `delayMs` after the first call that finds the - * guest down. A real reboot is seconds of downtime that a poll always sees; - * booting on a plain timer instead would let a slow test machine miss it. - */ - reboot(delayMs: number): void { - this.powerOff(); - this.#rebootDelayMs = delayMs; - } - - /** Append an outbound event to the outbox, as the agent does before streaming it. */ - emit(type: T, data: OutboundEventDataMap[T]): OutboundEvent { - if (type === "agent.state") this.agentState = (data as { state: AgentState }).state; - if (type === "approval.requested") { - const d = data as OutboundEventDataMap["approval.requested"]; - this.pendingApproval = { approval_id: d.approval_id, ...(d.task_id ? { task_id: d.task_id } : {}) }; - this.agentState = "WAITING_APPROVAL"; - } - const event = { seq: ++this.#seq, id: newId("evt"), type, ts: new Date().toISOString(), data } as OutboundEvent; - this.outbox.push(event); - this.#wake(); - return event; - } - - /** - * The engine writes an event of this type that the host's schema does not take: it has a `seq` (the next one) and the - * stream carries it, but the host refuses it. What follows is stored as usual. - */ - writeUnreadable(type: string, problem: string): void { - this.#refusals.push({ info: { seq: ++this.#seq, type, problem }, after: this.outbox.at(-1)?.seq ?? 0 }); - this.#wake(); - } - - /** The stream carries a message that is not JSON, so it has no `seq` and no type to name. */ - sendGarbage(problem: string): void { - this.#refusals.push({ info: { seq: null, type: null, problem }, after: this.outbox.at(-1)?.seq ?? 0 }); - this.#wake(); - } - - /** Ask for an approval from inside a task, as the policy engine does on `ask`. */ - requestApproval( - taskId: string | undefined, - tool = "browser_identity_delete", - permission: PermissionName = "browser.identity.delete", - args: Record = { identity_id: "shop-abc123" }, - ): string { - const approvalId = newId("apr"); - this.emit("agent.state", { state: "WAITING_APPROVAL" }); - this.emit("approval.requested", { - approval_id: approvalId, - ...(taskId ? { task_id: taskId } : {}), - tool, - permission, - arguments: args, - reason: "the tool needs approval", - }); - return approvalId; - } - - async health(): Promise { - this.#reachable("health"); - this.#pollsSinceBoot++; - // A real guest has been up for several seconds by the time its agent answers. - const uptime = 5 + Math.floor((Date.now() - this.#bootedAt) / 1000); - if (this.#pollsSinceBoot <= this.bootPolls) { - return { agentd: "ok", agent: { status: "down" }, uptime_s: uptime }; - } - return { - agentd: "ok", - agent: { - status: "ok", - state: this.agentState, - openrouter_configured: this.openrouterKey !== null, - browser: { identities: this.identities.size, open: this.#openIdentities().length }, - checks: { ...this.checks }, - }, - uptime_s: uptime, - }; - } - - async system(): Promise { - this.#reachable("system"); - return { - hostname: "invisible-dot-fake", - uptime_s: 5 + Math.floor((Date.now() - this.#bootedAt) / 1000), - cpus: 2, - mem_total_bytes: 4 * 1024 ** 3, - mem_available_bytes: 3 * 1024 ** 3, - disk_total_bytes: 40 * 1024 ** 3, - disk_free_bytes: 35 * 1024 ** 3, - }; - } - - async pushSecrets(secrets: SecretsRequest): Promise { - this.#reachable("pushSecrets"); - this.openrouterKey = secrets.openrouter_api_key; - this.mcpSecrets = structuredClone(secrets.mcp_secrets); - } - - - /** While set, `putConfig` waits for it (a slow push), and the config is stored when it resolves. */ - configPushGate: Promise | null = null; - - async putConfig(config: DotRuntimeConfig): Promise { - this.#reachable("putConfig"); - await this.configPushGate; - this.config = config; - } - - async postEvent(event: InboundEvent): Promise<{ accepted: true }> { - this.#reachable(`postEvent:${event.type}`); - if (this.failNextPost) { - const error = this.failNextPost; - this.failNextPost = null; - throw error; - } - // Like the agent's inbox: an id it already accepted is accepted again and ignored. - if (this.inbound.some((e) => e.id === event.id)) return { accepted: true }; - this.inbound.push(event); - await this.onInbound(event, this); - return { accepted: true }; - } - - async state(): Promise { - this.#reachable("state"); - return { state: this.agentState, current_task_id: null, pending_approval: null }; - } - - async listBrowserIdentities(): Promise { - this.#reachable("listBrowserIdentities"); - return { identities: [...this.identities.values()] }; - } - - async createBrowserIdentity(body: CreateBrowserIdentityRequest): Promise { - this.#reachable("createBrowserIdentity"); - let checked: { name: string; proxy?: string }; - try { - // The agent's own rules, with the engine's own limit (DEFAULT_MAX_IDENTITIES in nanobot/dots/main.py). - checked = checkIdentityRequest(body, this.identities.size, FAKE_MAX_IDENTITIES); - } catch (error) { - if (error instanceof IdentityRequestError) throw new FakeGuestError(error.code === "limit" ? 409 : 400, error.message, error.code); - throw error; - } - const id = newIdentityId(checked.name); - const identity: BrowserIdentity = { - id, - name: checked.name, - createdAt: new Date().toISOString(), - lastUsedAt: null, - status: "available", - profilePath: `/home/dot/browsers/${id}/profile`, - hasProxy: checked.proxy !== undefined, - }; - this.identities.set(id, identity); - this.emit("browser.identity.created", { identity_id: id, name: checked.name }); - return identity; - } - - async getBrowserIdentity(id: string): Promise { - this.#reachable("getBrowserIdentity"); - const identity = this.identities.get(id); - if (!identity) throw new FakeGuestError(404, `identity ${id} not found`, "not_found"); - return identity; - } - - async deleteBrowserIdentity(id: string): Promise { - this.#reachable("deleteBrowserIdentity"); - const identity = this.identities.get(id); - if (!identity) throw new FakeGuestError(404, `identity ${id} not found`, "not_found"); - this.identities.delete(id); - this.emit("browser.identity.deleted", { identity_id: id, name: identity.name }); - } - - async getBrowserIdentityFrame(id: string): Promise { - this.#reachable("getBrowserIdentityFrame"); - const identity = this.identities.get(id); - if (!identity) throw new FakeGuestError(404, `no browser identity "${id}"`, "not_found"); - if (identity.status !== "open") { - throw new FakeGuestError(409, `identity ${id} is not open; call browser_identity_launch first`, "not_open"); - } - if (this.identityBusy) { - throw new FakeGuestError(IDENTITY_ERROR_STATUS.busy, `browser identity "${id}" is busy with a call; ask again in a moment`, "busy"); - } - if (this.identityFault) { - throw new FakeGuestError(IDENTITY_ERROR_STATUS[this.identityFault], `browser identity "${id}" failed: ${this.identityFault}`, this.identityFault); - } - // The JPEG markers of an empty image: enough for a content check. - return Uint8Array.from([0xff, 0xd8, 0xff, 0xd9]); - } - - async closeBrowserIdentity(id: string): Promise { - this.#reachable("closeBrowserIdentity"); - const identity = this.identities.get(id); - if (!identity) throw new FakeGuestError(404, `no browser identity "${id}"`, "not_found"); - if (identity.status !== "open") return; - this.identities.set(id, { ...identity, status: "available" }); - this.emit("browser.identity.closed", { identity_id: id, name: identity.name }); - } - - /** Give the Dot an automation (its cron tool made it). */ - putAutomation(automation: FakeAutomation): void { - this.automations.set(automation.id, automation); - this.#reportNextRun(); - } - - /** Take an automation away (its cron tool removed it). */ - removeAutomation(id: string): void { - this.automations.delete(id); - this.#reportNextRun(); - } - - /** - * What the engine does after every change of its jobs: tell the host, with an `automation.next_run` event, when the - * earliest enabled one is due (null when none is), once per change (also at a boot, when a run made - * late moved it: a boot with nothing changed reports nothing, the engine has told the host what it has to). - */ - #reportNextRun(): void { - const due = [...this.automations.values()].flatMap((a) => (a.enabled && a.next_run_at_ms !== null ? [a.next_run_at_ms] : [])); - const next = due.length > 0 ? Math.min(...due) : null; - if (next === this.#reportedNextRun) return; - this.#reportedNextRun = next; - this.emit("automation.next_run", { next_run_at_ms: next }); - } - - #reportedNextRun: number | null = null; - - async listTools(): Promise { - this.#reachable("listTools"); - const permissions: Record = this.config?.permissions ?? {}; - const offered = (tool: Omit): boolean => - this.config !== null && (permissions[tool.permission] === "allow" || permissions[tool.permission] === "ask"); - const servers = this.config ? mcpServerNames(this.config) : []; - const connected = servers.filter((name) => !this.mcpFailures.has(name)); - const mcpTools = connected.flatMap((name) => (this.mcpTools.get(name) ?? []).map((tool) => ({ ...tool, permission: mcpPermission(name) }))); - return { - tools: [...this.tools, ...mcpTools].map((tool) => ({ ...tool, offered: offered(tool) })), - mcp_servers: servers.map((name) => { - const error = this.mcpFailures.get(name); - return error === undefined - ? { name, state: "connected" as const, error: null, tools: this.mcpTools.get(name)?.length ?? 0 } - : { name, state: "failed" as const, error, tools: 0 }; - }), - }; - } - - async listSkills(): Promise { - this.#reachable("listSkills"); - return { skills: [...this.skills] }; - } - - /** Put a file in the guest's file system (the Dot wrote it). */ - putFile(path: string, content: string | Uint8Array, mtime: Date = new Date()): void { - this.files.set(path, { content: typeof content === "string" ? new TextEncoder().encode(content) : content, mtime }); - } - - /** A symbolic link at `path` leading to `target` (absolute). */ - link(path: string, target: string): void { - this.links.set(path, target); - } - - /** - * What dot-agentd's TCP listener does with a path: follow the symbolic links, and refuse (403 `outside_home`) a - * real location that is not under home. The host's file routes always reach the guest through that listener. - */ - #realPath(path: string): string { - let real = path; - for (let hops = 0; ; hops++) { - if (hops > 40) throw new FakeGuestError(500, `${path}: too many levels of symbolic links`, "io_error"); - const key = [...this.links.keys()].filter((link) => real === link || real.startsWith(`${link}/`)).sort((a, b) => b.length - a.length)[0]; - if (key === undefined) break; - real = this.links.get(key)! + real.slice(key.length); - } - if (real !== GUEST_PATHS.home && !real.startsWith(`${GUEST_PATHS.home}/`)) { - throw new FakeGuestError(403, `the path leads outside ${GUEST_PATHS.home}`, "outside_home"); - } - return real; - } - - #isDirectory(path: string): boolean { - return path === GUEST_PATHS.home || [...this.files.keys()].some((file) => file.startsWith(`${path}/`)); - } - - async readFile(asked: string, options: { maxBytes?: number } = {}): Promise { - this.#reachable("readFile"); - const path = this.#realPath(asked); - if (this.#isDirectory(path)) throw new FakeGuestError(400, `${path} is a directory; use /v1/files/list`, "is_a_directory"); - const file = this.files.get(path); - if (!file) throw new FakeGuestError(404, `open ${path}: no such file or directory`, "not_found"); - if (options.maxBytes !== undefined && file.content.length > options.maxBytes) { - throw new FakeGuestError(413, `the answer is larger than ${options.maxBytes} bytes`, FILE_TOO_LARGE); - } - return file.content; - } - - async listFiles(asked: string): Promise { - this.#reachable("listFiles"); - const path = this.#realPath(asked); - if (this.files.has(path)) throw new FakeGuestError(400, `${path} is not a directory`, "not_a_directory"); - if (!this.#isDirectory(path)) throw new FakeGuestError(404, `stat ${path}: no such file or directory`, "not_found"); - const entries = new Map(); - for (const [file, { content, mtime }] of this.files) { - if (!file.startsWith(`${path}/`)) continue; - const [name, ...rest] = file.slice(path.length + 1).split("/"); - const modified = mtime.toISOString(); - const known = entries.get(name!); - if (rest.length > 0) entries.set(name!, { name: name!, type: "dir", size: 0, mtime: known && known.mtime > modified ? known.mtime : modified }); - else entries.set(name!, { name: name!, type: "file", size: content.length, mtime: modified }); - } - return { entries: [...entries.values()].sort((a, b) => (a.name < b.name ? -1 : 1)) }; - } - - async prepareSleep(): Promise { - this.#reachable("prepareSleep"); - } - - async screenshot(): Promise { - this.#reachable("screenshot"); - return desktopPng(); - } - - /** The outbox after `after`, then every new event, until aborted or disconnected. */ - async *events(options: { after?: number; signal?: AbortSignal; onRefused?: (info: RefusedEvent) => void | Promise }): AsyncGenerator { - this.#reachable("events"); - const own = new AbortController(); - this.#streams.add(own); - let after = options.after ?? 0; - const handed = new Set(); - try { - for (;;) { - if (options.signal?.aborted) return; - if (own.signal.aborted) throw new FakeGuestError(0, "event stream disconnected"); - // A refused message is handed over when its turn comes (after the events written before it), as the real - // stream does, and a reconnect resumes after its `seq`. - const refusal = this.#refusals.find((r) => !handed.has(r) && !this.outbox.some((e) => e.seq > after && e.seq <= r.after)); - if (refusal) { - handed.add(refusal); - if (refusal.info.seq !== null && refusal.info.seq <= after) continue; - await options.onRefused?.(refusal.info); - if (refusal.info.seq !== null) after = refusal.info.seq; - continue; - } - const next = this.outbox.find((e) => e.seq > after); - if (next) { - after = next.seq; - yield next; - continue; - } - await new Promise((resolve) => { - const done = () => { - this.#wakers.delete(done); - options.signal?.removeEventListener("abort", done); - resolve(); - }; - this.#wakers.add(done); - options.signal?.addEventListener("abort", done, { once: true }); - }); - } - } finally { - this.#streams.delete(own); - } - } -} - -interface FakeVm { - state: VmState; - guestPort: number | null; - pid: number | null; - token: string; -} - -export type DriverOperation = "create" | "start" | "stop" | "reboot" | "destroy"; - -/** The first port FakeDriver hands out; far from anything a test binds, and never really bound. */ -const FAKE_FIRST_PORT = 47_000; - -/** - * A ComputerDriver that keeps its "VMs" in memory and boots a FakeGuest in - * each. Like QEMU with a fresh port forward, every start gets a new port, - * and a guest is reached only through the port its VM runs with now. - */ -export class FakeDriver implements ComputerDriver { - readonly vms = new Map(); - readonly guests = new Map(); - readonly calls: string[] = []; - readonly #failures = new Map(); - #nextPort = FAKE_FIRST_PORT; - #nextPid = 9_000; - /** Applied to every new guest, e.g. to install a custom agent behaviour. */ - configureGuest: (guest: FakeGuest) => void = () => {}; - rebootDelayMs = 30; - goldenImage = "images/golden-test.qcow2"; - runtimeImage = "images/runtime-test.iso"; - - /** Make the next `times` calls of `operation` fail with `error`. */ - failNext(operation: DriverOperation, error: Error = new Error(`${operation} failed (injected)`), times = 1): void { - this.#failures.set(operation, { error, times }); - } - - #enter(operation: DriverOperation, dotId: string): void { - this.calls.push(`${operation}:${dotId}`); - const failure = this.#failures.get(operation); - if (failure) { - if (--failure.times <= 0) this.#failures.delete(operation); - throw failure.error; - } - } - - guestOf(dotId: string): FakeGuest { - const guest = this.guests.get(dotId); - if (!guest) throw new Error(`no fake guest for ${dotId}`); - return guest; - } - - /** Power a VM off behind the control plane's back, as a crash or a host reboot would. */ - crash(dotId: string): void { - const vm = this.vms.get(dotId); - if (vm) Object.assign(vm, { state: "STOPPED", guestPort: null, pid: null }); - this.guests.get(dotId)?.powerOff(); - } - - async create(spec: ComputerSpecInput): Promise { - this.#enter("create", spec.dotId); - if (!this.vms.has(spec.dotId)) this.vms.set(spec.dotId, { state: "STOPPED", guestPort: null, pid: null, token: spec.token }); - if (!this.guests.has(spec.dotId)) { - const guest = new FakeGuest(spec.token); - this.configureGuest(guest); - this.guests.set(spec.dotId, guest); - } - return { goldenImage: this.goldenImage, runtimeImage: this.runtimeImage }; - } - - async start(spec: ComputerSpecInput & { goldenImage: string }): Promise { - this.#enter("start", spec.dotId); - const vm = this.vms.get(spec.dotId); - if (!vm) throw new Error(`the disk of ${spec.dotId} does not exist`); - if (vm.state === "RUNNING" && vm.guestPort !== null && vm.pid !== null) { - return { guestPort: vm.guestPort, pid: vm.pid, runtimeImage: this.runtimeImage, alreadyRunning: true }; - } - Object.assign(vm, { state: "RUNNING", guestPort: this.#nextPort++, pid: this.#nextPid++ }); - this.guestOf(spec.dotId).boot(); - return { guestPort: vm.guestPort!, pid: vm.pid!, runtimeImage: this.runtimeImage, alreadyRunning: false }; - } - - async stop(dotId: string, token: string) { - this.#enter("stop", dotId); - // Like dot-agentd: the poweroff needs the Dot's token. - if (this.vms.get(dotId) && token !== this.vms.get(dotId)!.token) throw new FakeGuestError(401, "unauthorized", "unauthorized"); - this.crash(dotId); - return { forced: false }; - } - - /** Like the real one: the guest goes down, comes back a moment later, and the port changes. */ - async reboot(spec: ComputerSpecInput & { goldenImage: string }): Promise { - this.#enter("reboot", spec.dotId); - const vm = this.vms.get(spec.dotId); - if (!vm) throw new Error(`the disk of ${spec.dotId} does not exist`); - Object.assign(vm, { state: "RUNNING", guestPort: this.#nextPort++, pid: this.#nextPid++ }); - this.guestOf(spec.dotId).reboot(this.rebootDelayMs); - return { guestPort: vm.guestPort!, pid: vm.pid!, runtimeImage: this.runtimeImage, alreadyRunning: false }; - } - - /** vm-manager's rule and loop (pollGuestHealth), with "QEMU exited" as the check, as VmManager has. */ - async waitForHealth(endpoint: GuestEndpoint, token: string, options: WaitForHealthOptions): Promise { - const source = { address: `fake:${endpoint.port}`, health: (o: { timeoutMs?: number }) => this.guest(endpoint, token).health(o) }; - return pollGuestHealth(source, { - timeoutMs: options.timeoutMs, - intervalMs: options.intervalMs, - requestTimeoutMs: options.requestTimeoutMs, - check: () => { - if (this.vms.get(endpoint.dotId)?.state !== "RUNNING") throw new Error(`QEMU of ${endpoint.dotId} is not running any more`); - }, - }); - } - - async destroy(dotId: string) { - this.#enter("destroy", dotId); - this.guests.get(dotId)?.powerOff(); - this.vms.delete(dotId); - this.guests.delete(dotId); - } - - async state(dotId: string): Promise { - const vm = this.vms.get(dotId); - if (!vm) return { exists: false, state: "STOPPED", pid: null, guestPort: null, detail: null }; - return { exists: true, state: vm.state, pid: vm.pid, guestPort: vm.guestPort, detail: vm.state === "RUNNING" ? "running" : null }; - } - - guest(endpoint: GuestEndpoint, token: string): GuestApi { - const vm = this.vms.get(endpoint.dotId); - const guest = this.guests.get(endpoint.dotId); - const refuse = (status: number, message: string, code?: string) => - new Proxy({} as GuestApi, { - // Not a thenable: `await` on the client must not call "then" and hang. - get: (_target, property) => - property === "then" - ? undefined - : async () => { - throw new FakeGuestError(status, message, code); - }, - }); - // A port the VM no longer runs with reaches nothing, like a stale forward after a restart. - if (!vm || !guest || vm.guestPort !== endpoint.port) { - return refuse(0, `connect ECONNREFUSED 127.0.0.1:${endpoint.port}`, "ECONNREFUSED"); - } - // Every call is refused, like dot-agentd answering 401 to a wrong token. - if (token !== guest.token) return refuse(401, "unauthorized", "unauthorized"); - return guest; - } - - async close() {} -} - -/** Poll until `predicate` holds; fails with `what` after `timeoutMs`. */ -export async function waitFor( - predicate: () => T | Promise, - what: string, - timeoutMs = 5_000, - intervalMs = 10, -): Promise> { - const deadline = Date.now() + timeoutMs; - for (;;) { - const value = await predicate(); - if (value) return value as NonNullable; - if (Date.now() > deadline) throw new Error(`timed out waiting for ${what}`); - await new Promise((resolve) => setTimeout(resolve, intervalMs)); - } -} - -/** - * Wait until the Dot is READY and doing nothing: no lifecycle operation - * under its lock, every event its fake guest emitted so far stored by the - * pump, and what READY and those events started in the background done too - * (the flush of its outbox, the push an agent.started asks for). A test that - * starts earlier races that work: an idle check skips a Dot whose outbox is - * being flushed, and a guest event stored later counts as activity, which is - * right in a server and a coin toss in a test. - */ -export async function waitUntilSettledReady( - scheduler: { - db: { dots: { get(id: string): Promise<{ status: string } | null> }; computers: { get(id: string): Promise<{ event_cursor: number } | null> } }; - lifecycle: { isBusy(id: string): boolean }; - settle(): Promise; - }, - driver: FakeDriver, - dotId: string, - what: string, -): Promise { - // A Dot's creation is dozens of queries: against a Postgres across WSL on a loaded host that has taken over 5 s. - await waitFor(async () => (await scheduler.db.dots.get(dotId))?.status === "READY" && !scheduler.lifecycle.isBusy(dotId), `${what} READY`, 15_000); - const lastSeq = driver.guestOf(dotId).outbox.at(-1)?.seq ?? 0; - await waitFor(async () => ((await scheduler.db.computers.get(dotId))?.event_cursor ?? 0) >= lastSeq, `${what}'s guest events stored`, 15_000); - await scheduler.settle(); -} - -/** A clock tests move by hand. */ -export class ManualClock { - #now: number; - constructor(start: Date = new Date()) { - this.#now = start.getTime(); - } - now(): Date { - return new Date(this.#now); - } - advance(ms: number): void { - this.#now += ms; - } -} diff --git a/apps/scheduler/test/commit-points.test.ts b/apps/scheduler/test/commit-points.test.ts deleted file mode 100644 index ffe657c2..00000000 --- a/apps/scheduler/test/commit-points.test.ts +++ /dev/null @@ -1,197 +0,0 @@ -/** - * A change of state and the host event that tells it commit together or not at all. The control plane can be killed - * (kill -9, a power cut) between any two statements; a transaction cannot be half done, so what is left is the state it - * was in before the change, which recovery knows how to finish, never a state nobody was told of. - * - * Each test plays the kill by making the last write of the change throw (`killedAt`, database/testing): the process - * dies there, and whatever was written before it in the same transaction must not have been committed. The events - * checked are those the change would have logged. - */ -import type { Database } from "@invisible-dots/database"; -import { createTestDatabase, killedAt, type TestDatabase } from "@invisible-dots/database/testing"; -import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; -import { Scheduler } from "../src/index.js"; -import { FakeDriver, ManualClock, waitFor, waitUntilSettledReady } from "../src/testing.js"; - -const yaml = (name: string) => `name: ${name}\nmodel:\n provider: openrouter\n id: test/model\n`; - -describe("the commit points of the host", () => { - let t: TestDatabase; - let db: Database; - const open: Scheduler[] = []; - - beforeAll(async () => { - t = await createTestDatabase("pglite"); - db = t.db; - await db.secrets.put("global", "openrouter_api_key", "sk-or-test"); - }); - - afterEach(async () => { - await Promise.all(open.splice(0).map((s) => s.close())); - }); - - afterAll(async () => { - await t?.drop(); - }); - - function make(driver: FakeDriver, over: Database = db) { - const scheduler = new Scheduler({ - db: over, - driver, - clock: new ManualClock(), - lifecycle: { healthPollMs: 5, readyTimeoutMs: 3_000, pumpRetryMs: 10, pumpMaxRetryMs: 50 }, - dispatcher: { retryDelayMs: 0, maxDeliveryAttempts: 2 }, - dispatchIntervalMs: 60_000, - idleCheckIntervalMs: 60_000, - }); - open.push(scheduler); - return scheduler; - } - - const typesOf = async (dotId: string, ...types: string[]) => (await db.events.list({ dotId, types })).map((e) => e.type); - - it("a stop killed at its last write leaves the computer STOPPING and says nothing of a stop; recovery finishes it, and tells of it once", async () => { - const driver = new FakeDriver(); - const killed = killedAt(db, "dots", "setStatus"); - const first = make(driver, killed.db); - const dot = await first.createDot(yaml("killed-stop")); - await waitUntilSettledReady(first, driver, dot.id, "killed-stop"); - const before = (await typesOf(dot.id, "computer.stopped")).length; - - killed.kill(); - await first.stopComputer(dot.id); - await first.settle(); - - // Before the change the stop's rows were four statements, so the state was STOPPED, the Dot's status old and no event. - expect(await db.computers.get(dot.id)).toMatchObject({ state: "STOPPING", stop_reason: "user" }); - expect((await db.events.list({ dotId: dot.id, types: ["computer.state"] })).map((e) => e.data.state)).not.toContain("STOPPED"); - expect(await typesOf(dot.id, "computer.stopped")).toHaveLength(before); - - // The control plane starts again: the interrupted stop is finished, and told once. - killed.revive(); - await first.close(); - open.splice(open.indexOf(first), 1); - const second = make(driver); - await second.start(); - await second.settle(); - expect(await db.computers.get(dot.id)).toMatchObject({ state: "STOPPED", stop_reason: "user", pid: null, guest_port: null }); - expect(await db.events.list({ dotId: dot.id, types: ["computer.stopped"] })).toHaveLength(before + 1); - expect((await db.events.list({ dotId: dot.id, types: ["computer.state"] })).map((e) => e.data.state).filter((state) => state === "STOPPED")).toHaveLength(1); - expect((await db.dots.get(dot.id))?.status).toBe("IDLE"); - }); - - it("a deletion killed at its last write leaves the Dot, and no dot.deleted; the event and the Dot go together", async () => { - const driver = new FakeDriver(); - const killed = killedAt(db, "dots", "delete"); - const scheduler = make(driver, killed.db); - const dot = await scheduler.createDot(yaml("killed-delete")); - await waitUntilSettledReady(scheduler, driver, dot.id, "killed-delete"); - - killed.kill(); - await scheduler.deleteDot(dot.id); - await scheduler.settle(); - expect(await db.dots.get(dot.id)).not.toBeNull(); - expect(await typesOf(dot.id, "dot.deleted")).toEqual([]); - - killed.revive(); - await scheduler.deleteDot(dot.id); - await scheduler.settle(); - expect(await db.dots.get(dot.id)).toBeNull(); - expect(await typesOf(dot.id, "dot.deleted")).toEqual(["dot.deleted"]); - }); - - it("a Dot killed at its second insert is not created and is not announced", async () => { - const killed = killedAt(db, "computers", "insert"); - const scheduler = make(new FakeDriver(), killed.db); - killed.kill(); - await expect(scheduler.createDot(yaml("killed-create"))).rejects.toThrow(/killed at computers.insert/); - expect(await db.dots.resolve("killed-create")).toBeNull(); - expect((await db.events.list({ types: ["dot.created"] })).filter((e) => e.data.name === "killed-create")).toEqual([]); - - killed.revive(); - const dot = await scheduler.createDot(yaml("killed-create")); - expect(await typesOf(dot.id, "dot.created", "computer.state")).toEqual(["dot.created", "computer.state"]); - }); - - it("a task killed at its insert is not announced, and an announced one exists", async () => { - const driver = new FakeDriver(); - const killed = killedAt(db, "tasks", "insert"); - const scheduler = make(driver, killed.db); - const dot = await scheduler.createDot(yaml("killed-task")); - await waitUntilSettledReady(scheduler, driver, dot.id, "killed-task"); - - killed.kill(); - await expect(scheduler.createTask(dot.id, { description: "never stored" })).rejects.toThrow(/killed at tasks.insert/); - expect(await typesOf(dot.id, "task.created")).toEqual([]); - - killed.revive(); - const task = await scheduler.createTask(dot.id, { description: "stored" }); - await waitFor(async () => (await typesOf(dot.id, "task.created")).length === 1, "the event of the task"); - expect((await db.events.list({ dotId: dot.id, types: ["task.created"] }))[0]!.data.task_id).toBe(task.id); - }); - - it("a task given up killed at its last write is neither FAILED nor told; given up whole, it is both", async () => { - const driver = new FakeDriver(); - const killed = killedAt(db, "inbound", "drop"); - const scheduler = make(driver, killed.db); - const dot = await scheduler.createDot(yaml("killed-give-up")); - await waitUntilSettledReady(scheduler, driver, dot.id, "killed-give-up"); - await scheduler.lifecycle.stop(dot.id, "user"); - driver.failNext("start", new Error("qemu-system-x86_64 exited: boom"), 3); - const failures = async (taskId: string) => - (await db.query<{ failures: number }>("SELECT failures FROM inbound_events WHERE task_id = $1", [taskId])).rows[0]?.failures ?? 0; - - killed.kill(); - const task = await scheduler.createTask(dot.id, { description: "undeliverable" }); - await waitFor(async () => (await failures(task.id)) >= 2, "its attempts used up"); - await scheduler.settle(); - expect((await db.tasks.get(task.id))?.status).toBe("RUNNING"); - expect(await typesOf(dot.id, "task.failed")).toEqual([]); - - // The next attempt gives it up whole: the state and its event in one commit. - killed.revive(); - await scheduler.inbound.kick(dot.id); - expect((await db.tasks.get(task.id))?.status).toBe("FAILED"); - expect(await typesOf(dot.id, "task.failed")).toEqual(["task.failed"]); - }); - - it("a config killed at its save is not announced, and a saved one is, in the same commit", async () => { - const driver = new FakeDriver(); - const killed = killedAt(db, "dots", "updateConfig"); - const scheduler = make(driver, killed.db); - const dot = await scheduler.createDot(yaml("killed-config")); - await waitUntilSettledReady(scheduler, driver, dot.id, "killed-config"); - const before = (await typesOf(dot.id, "dot.updated")).length; - - killed.kill(); - await expect(scheduler.updateDot(dot.id, `${yaml("killed-config")}instructions: be brief\n`)).rejects.toThrow(/killed at dots.updateConfig/); - expect(await typesOf(dot.id, "dot.updated")).toHaveLength(before); - expect((await db.dots.get(dot.id))?.config.instructions).toBeUndefined(); - - killed.revive(); - await scheduler.updateDot(dot.id, `${yaml("killed-config")}instructions: be brief\n`); - expect(await typesOf(dot.id, "dot.updated")).toHaveLength(before + 1); - expect((await db.dots.get(dot.id))?.config.instructions).toBe("be brief"); - }); - - it("the person's stop of a computer that is asleep is told, so that every view learns the automations will not wake it", async () => { - const driver = new FakeDriver(); - const scheduler = make(driver); - const dot = await scheduler.createDot(yaml("asleep-then-stopped")); - await waitUntilSettledReady(scheduler, driver, dot.id, "asleep-then-stopped"); - await scheduler.lifecycle.stop(dot.id, "idle"); - expect(await db.computers.get(dot.id)).toMatchObject({ state: "STOPPED", stop_reason: "idle" }); - const stops = async () => (await db.events.list({ dotId: dot.id, types: ["computer.stopped"] })).map((e) => e.data); - expect(await stops()).toEqual([{ reason: "idle", forced: false }]); - - await scheduler.stopComputer(dot.id); - await scheduler.settle(); - expect(await db.computers.get(dot.id)).toMatchObject({ state: "STOPPED", stop_reason: "user" }); - expect(await stops()).toEqual([{ reason: "idle", forced: false }, { reason: "user", forced: false }]); - - // Asked again, it changes nothing and says nothing more. - await scheduler.stopComputer(dot.id); - await scheduler.settle(); - expect(await stops()).toHaveLength(2); - }); -}); diff --git a/apps/scheduler/test/delivery.test.ts b/apps/scheduler/test/delivery.test.ts deleted file mode 100644 index 5028078e..00000000 --- a/apps/scheduler/test/delivery.test.ts +++ /dev/null @@ -1,379 +0,0 @@ -/** - * Races and failures around getting work to a guest: what is stored for a - * guest survives a failed wake and a restart, a cancel never loses to a - * slow wake, a sleep never swallows work, and what the guest must hold (its - * key, its config) is pushed again whenever it may have lost it. - */ -import type { Database } from "@invisible-dots/database"; -import { createTestDatabase, testAdapters, type TestDatabase } from "@invisible-dots/database/testing"; -import type { InboundEvent, OutboundEvent, SecretsRequest } from "@invisible-dots/shared"; -import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; -import { Scheduler, type SchedulerOptions } from "../src/index.js"; -import { type Logger } from "@invisible-dots/shared"; -import type { GuestApi, GuestEndpoint } from "../src/index.js"; -import { FakeDriver, FakeGuest, FakeGuestError, ManualClock, waitFor, waitUntilSettledReady } from "../src/testing.js"; - -const yaml = (name: string, instructions = "keep watch") => - `name: ${name}\ninstructions: ${instructions}\nmodel:\n provider: openrouter\n id: test/model\ncomputer:\n idle_timeout: 10m\n`; - -const KEY = "sk-or-delivery-test-key-0123456789"; - -/** A gate a fake waits on until the test opens it. */ -function gate(): { wait: Promise; open(): void; reached: Promise; arrive(): void } { - let open!: () => void; - let arrive!: () => void; - const wait = new Promise((resolve) => (open = resolve)); - const reached = new Promise((resolve) => (arrive = resolve)); - return { wait, open, reached, arrive }; -} - -const received = (guest: FakeGuest, type: InboundEvent["type"]) => guest.inbound.filter((e) => e.type === type); - -describe.each(testAdapters())("delivery to guests (%s)", (kind) => { - let t: TestDatabase; - let db: Database; - const open: Scheduler[] = []; - - beforeAll(async () => { - t = await createTestDatabase(kind); - db = t.db; - await db.secrets.put("global", "openrouter_api_key", KEY); - }); - - afterEach(async () => { - await Promise.all(open.splice(0).map((s) => s.close())); - }); - - afterAll(async () => { - await t?.drop(); - }); - - function make(driver: FakeDriver = new FakeDriver(), extra: Partial = {}) { - const clock = new ManualClock(); - const scheduler = new Scheduler({ - db, - driver, - clock, - dispatchIntervalMs: 60_000, - idleCheckIntervalMs: 60_000, - lifecycle: { healthPollMs: 5, readyTimeoutMs: 3_000, pumpRetryMs: 10, pumpMaxRetryMs: 50 }, - dispatcher: { retryDelayMs: 0, maxDeliveryAttempts: 2 }, - ...extra, - }); - open.push(scheduler); - drivers.set(scheduler, driver); - return { scheduler, driver, clock }; - } - - const drivers = new Map(); - const driverOf = (s: Scheduler) => drivers.get(s)!; - - async function readyDot(s: Scheduler, name: string) { - const dot = await s.createDot(yaml(name)); - await waitUntilSettledReady(s, driverOf(s), dot.id, name); - return dot; - } - - /** A guest that asks for an approval inside every task. */ - function asksForApproval(guest: FakeGuest): void { - const original = guest.onInbound; - guest.onInbound = (event, g) => { - if (event.type === "task.created") { - g.emit("task.started", { task_id: event.data.task_id }); - g.requestApproval(event.data.task_id); - } else { - return original(event, g); - } - }; - } - - it("an approval for a sleeping Dot whose wake fails is kept and reaches the guest at the next READY", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "approve-later"); - const guest = driver.guestOf(dot.id); - asksForApproval(guest); - const task = await scheduler.createTask(dot.id, { description: "needs a yes" }); - const approval = await waitFor(async () => (await scheduler.listApprovals("pending")).find((a) => a.task_id === task.id), "pending approval"); - await scheduler.lifecycle.stop(dot.id, "user"); - - driver.failNext("start", new Error("QEMU exited (injected)")); - await scheduler.resolveApproval(approval.id, "approve"); - await scheduler.settle(); - expect(received(guest, "approval.received")).toHaveLength(0); - expect((await db.inbound.pending(dot.id)).map((r) => r.event.type)).toEqual(["approval.received"]); - - await scheduler.startComputer(dot.id); - await waitFor(() => received(guest, "approval.received").length === 1, "approval delivered after the next READY"); - await waitFor(async () => (await db.tasks.get(task.id))?.status === "COMPLETED", "task completed"); - expect(await db.inbound.pending(dot.id)).toEqual([]); - }); - - it("a message stored before a restart is delivered by the next control plane", async () => { - const driver = new FakeDriver(); - const first = make(driver).scheduler; - const dot = await readyDot(first, "restart-chat"); - await first.lifecycle.stop(dot.id, "user"); - driver.failNext("start", new Error("QEMU exited (injected)")); - const answer = await first.sendMessage(dot.id, "are you there?"); - expect(answer.delivery).toBe("queued"); - await first.close(); - open.splice(open.indexOf(first), 1); - - const { scheduler } = make(driver); - await scheduler.start(); - await scheduler.startComputer(dot.id); - const guest = driver.guestOf(dot.id); - await waitFor(() => received(guest, "user.message").some((e) => e.id === answer.message_id), "message delivered"); - await waitFor(async () => (await scheduler.conversation(dot.id)).length === 2, "reply"); - }); - - it("a task cancelled while its Dot wakes is never sent to the guest", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "cancel-early"); - await scheduler.lifecycle.stop(dot.id, "user"); - const guest = driver.guestOf(dot.id); - guest.bootPolls = 40; - const task = await scheduler.createTask(dot.id, { description: "never mind" }); - await waitFor(async () => (await db.tasks.get(task.id))?.status === "RUNNING", "claimed"); - expect(scheduler.lifecycle.isReady(dot.id)).toBe(false); - await scheduler.cancelTask(task.id); - await waitFor(() => scheduler.lifecycle.isReady(dot.id), "woken anyway"); - await scheduler.settle(); - expect((await db.tasks.get(task.id))?.status).toBe("CANCELLED"); - expect(received(guest, "task.created")).toHaveLength(0); - // Nothing to cancel on a guest that never heard of the task. - expect(received(guest, "system.event")).toHaveLength(0); - }); - - it("a task cancelled after its send began is cancelled on the guest too", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "cancel-late"); - const guest = driver.guestOf(dot.id); - guest.onInbound = (event, g) => { - if (event.type === "task.created") g.emit("task.started", { task_id: event.data.task_id }); - }; - const task = await scheduler.createTask(dot.id, { description: "long job" }); - await waitFor(() => received(guest, "task.created").length === 1, "delivered"); - await scheduler.cancelTask(task.id); - await waitFor(() => received(guest, "system.event").length === 1, "cancel delivered"); - expect(received(guest, "system.event")[0]?.data).toMatchObject({ name: "task.cancelled", data: { task_id: task.id } }); - }); - - it("an idle sleep never swallows a task that arrives while the agent prepares to sleep", async () => { - const { scheduler, driver, clock } = make(); - const dot = await readyDot(scheduler, "light-sleeper"); - const guest = driver.guestOf(dot.id); - const bootsAtReceipt: number[] = []; - const complete = guest.onInbound; - guest.onInbound = (event, g) => { - if (event.type === "task.created") bootsAtReceipt.push(g.boots); - return complete(event, g); - }; - const sleeping = gate(); - const prepare = guest.prepareSleep.bind(guest); - guest.prepareSleep = async () => { - sleeping.arrive(); - await sleeping.wait; - return prepare(); - }; - clock.advance(11 * 60_000); - expect(await scheduler.idleCheck()).toEqual([dot.id]); - await sleeping.reached; - const task = await scheduler.createTask(dot.id, { description: "just in time" }); - await scheduler.dispatcher.dispatch(); - // Give a send that does not wait for the stop every chance to happen; a - // correct one waits for the Dot's lock, so nothing arrives in this window. - await waitFor(() => received(guest, "task.created").length > 0, "a send during prepare-sleep", 300).catch(() => undefined); - expect(received(guest, "task.created")).toHaveLength(0); - sleeping.open(); - await waitFor(async () => (await db.tasks.get(task.id))?.status === "COMPLETED", "task completed after the wake"); - // Received once, by the guest of the next boot: never by the one that was powering off. - expect(bootsAtReceipt).toEqual([2]); - expect((await db.computers.get(dot.id))?.state).toBe("RUNNING"); - }); - - it("an idle sleep is called off when work was stored after the idle check", async () => { - const { scheduler, driver, clock } = make(); - const dot = await readyDot(scheduler, "called-off"); - const guest = driver.guestOf(dot.id); - // Work the idle check did not see: a message stored straight into the outbox. - clock.advance(11 * 60_000); - const message: InboundEvent<"user.message"> = { id: "msg_late", type: "user.message", ts: new Date().toISOString(), data: { text: "hi" } }; - await db.inbound.enqueue(dot.id, message); - const stop = scheduler.lifecycle.stop(dot.id, "idle"); - await stop; - expect((await db.computers.get(dot.id))?.state).toBe("RUNNING"); - expect(guest.calls).not.toContain("prepareSleep"); - expect(scheduler.lifecycle.isReady(dot.id)).toBe(true); - }); - - it("a stopped Dot with an unfinished task is woken for a new task", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "blocked"); - const guest = driver.guestOf(dot.id); - guest.onInbound = (event, g) => { - if (event.type === "task.created") g.emit("task.started", { task_id: event.data.task_id }); - }; - const first = await scheduler.createTask(dot.id, { description: "unfinished" }); - await waitFor(() => received(guest, "task.created").length === 1, "first delivered"); - await scheduler.lifecycle.stop(dot.id, "user"); - await scheduler.createTask(dot.id, { description: "waits behind it" }); - await scheduler.pass(); - await waitFor(() => scheduler.lifecycle.isReady(dot.id), "woken for the new task"); - expect((await db.tasks.get(first.id))?.status).toBe("RUNNING"); - }); - - it("a send that reached the guest but timed out is sent again, accepted once, and never fails the task", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "slow-ack"); - const guest = driver.guestOf(dot.id); - const post = guest.postEvent.bind(guest); - let timeouts = 3; - guest.postEvent = async (event) => { - await post(event); - if (event.type === "task.created" && timeouts-- > 0) throw new FakeGuestError(0, "no answer within 30000 ms"); - return { accepted: true }; - }; - const task = await scheduler.createTask(dot.id, { description: "acknowledged late" }); - await waitFor(async () => (await db.tasks.get(task.id))?.status === "COMPLETED", "completed"); - expect(received(guest, "task.created")).toHaveLength(1); - expect((await db.tasks.runs(task.id)).map((r) => r.outcome)).toEqual(["completed"]); - }); - - it("a cancelled task leaves no pending approval, and a late request for it is never pending", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "expiring"); - const guest = driver.guestOf(dot.id); - asksForApproval(guest); - const task = await scheduler.createTask(dot.id, { description: "asks then gets cancelled" }); - const approval = await waitFor(async () => (await scheduler.listApprovals("pending")).find((a) => a.task_id === task.id), "pending"); - await scheduler.cancelTask(task.id); - expect((await scheduler.listApprovals("pending")).map((a) => a.id)).not.toContain(approval.id); - expect((await db.approvals.get(approval.id))?.status).toBe("expired"); - await expect(scheduler.resolveApproval(approval.id, "approve")).rejects.toMatchObject({ status: 409, code: "already_resolved" }); - - const late = guest.requestApproval(task.id); - await waitFor(async () => (await db.approvals.get(late)) !== null, "late request stored"); - expect((await db.approvals.get(late))?.status).toBe("expired"); - }); - - it("an agent that restarts inside a running VM gets its key again", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "phoenix"); - const guest = driver.guestOf(dot.id); - expect(guest.openrouterKey).toBe(KEY); - guest.restartAgent(); - expect(guest.openrouterKey).toBeNull(); - await waitFor(() => guest.openrouterKey === KEY, "key pushed again after the restart"); - expect(guest.config?.name).toBe("phoenix"); - }); - - it("a config changed while the Dot passes READY reaches the guest", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "late-config"); - const guest = driver.guestOf(dot.id); - // The boot's agent.started is stored, and the push it starts in the - // background has finished (it runs after the event's transaction, so the - // cursor alone does not say so), so nothing but READY pushes from here on. - await waitFor(async () => (await db.computers.get(dot.id))?.event_cursor === guest.outbox.at(-1)?.seq, "boot events stored"); - await scheduler.lifecycle.settle(); - const pushing = gate(); - const putConfig = guest.putConfig.bind(guest); - let held = false; - guest.putConfig = async (config) => { - if (!held) { - held = true; - pushing.arrive(); - await pushing.wait; - } - return putConfig(config); - }; - // A READY procedure on the running VM (as after a failed call or a control plane restart). - scheduler.lifecycle.markSuspect(dot.id); - const ready = scheduler.lifecycle.ensureReady(dot.id); - await pushing.reached; - const update = await scheduler.updateDot(dot.id, yaml("late-config", "a brand new instruction")); - expect(update.config.instructions).toBe("a brand new instruction"); - pushing.open(); - await ready; - expect(guest.config?.instructions).toBe("a brand new instruction"); - }); - - it("a guest that echoes the key in a failed secret push never puts it in the log, the rows or the events", async () => { - const lines: string[] = []; - const record = (level: string) => (message: string, fields?: Record) => lines.push(`${level} ${message} ${JSON.stringify(fields ?? {})}`); - const logger: Logger = { debug: record("debug"), info: record("info"), warn: record("warn"), error: record("error") }; - const driver = new FakeDriver(); - driver.configureGuest = (g) => { - g.pushSecrets = async (secrets: SecretsRequest) => { - throw new FakeGuestError(502, `upstream said: bad request body {"openrouter_api_key":"${secrets.openrouter_api_key}"}`); - }; - }; - const { scheduler } = make(driver, { logger }); - const dot = await scheduler.createDot(yaml("echoes")); - await waitFor(async () => (await db.dots.get(dot.id))?.status === "ERROR", "ERROR"); - await scheduler.settle(); - const dotRow = await db.dots.get(dot.id); - const computer = await db.computers.get(dot.id); - const events = await db.events.list({ dotId: dot.id }); - expect(dotRow?.error).toMatch(/did not take the OpenRouter key and the MCP secrets \(status 502\)/); - for (const text of [dotRow?.error ?? "", computer?.last_error ?? "", JSON.stringify(events), lines.join("\n")]) { - expect(text).not.toContain(KEY); - expect(text).not.toContain(KEY.slice(0, 12)); - } - }); - - it("a VM that dies is recorded as stopped, and a new task runs on a new port with a working event pump", async () => { - /** Like the real guest client: an event stream bound to the port it was opened on, reconnecting forever. */ - class StalePortDriver extends FakeDriver { - override guest(endpoint: GuestEndpoint, token: string): GuestApi { - const inner = super.guest(endpoint, token); - const vms = this.vms; - return new Proxy(inner, { - get: (target, property) => { - if (property === "then") return undefined; - if (property !== "events") { - const value: unknown = Reflect.get(target, property); - return typeof value === "function" ? value.bind(target) : value; - } - return async function* (options: { after?: number; signal?: AbortSignal }): AsyncGenerator { - while (!options.signal?.aborted) { - if (vms.get(endpoint.dotId)?.guestPort === endpoint.port) { - try { - yield* target.events(options); - } catch { - // reconnects, as GuestClient does on a reset - } - } - await new Promise((resolve) => setTimeout(resolve, 5)); - } - }; - }, - }); - } - } - const driver = new StalePortDriver(); - const { scheduler } = make(driver); - const dot = await readyDot(scheduler, "lazarus"); - const before = (await db.computers.get(dot.id))?.guest_port; - driver.crash(dot.id); - const task = await scheduler.createTask(dot.id, { description: "after the crash" }); - await waitFor(async () => (await db.tasks.get(task.id))?.status === "COMPLETED", "task completed on the new boot"); - expect((await db.computers.get(dot.id))?.guest_port).not.toBe(before); - }); - - it("a VM that powers itself off with a task in progress is started again", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "self-stopper"); - const guest = driver.guestOf(dot.id); - guest.onInbound = (event, g) => { - if (event.type === "task.created") g.emit("task.started", { task_id: event.data.task_id }); - }; - await scheduler.createTask(dot.id, { description: "keeps going" }); - await waitFor(() => received(guest, "task.created").length === 1, "delivered"); - const boots = guest.boots; - driver.crash(dot.id); - await waitFor(async () => (await db.events.list({ dotId: dot.id, types: ["computer.stopped"] })).some((e) => e.data.reason === "exited"), "stop recorded"); - await waitFor(() => guest.boots === boots + 1 && scheduler.lifecycle.isReady(dot.id), "started again"); - }); -}); diff --git a/apps/scheduler/test/engine-shapes.test.ts b/apps/scheduler/test/engine-shapes.test.ts deleted file mode 100644 index 397ae751..00000000 --- a/apps/scheduler/test/engine-shapes.test.ts +++ /dev/null @@ -1,152 +0,0 @@ -/** - * The engine's answer to `GET /tools` and the data of every event it writes, against the host's description of them. The engine's own - * test (`invisible_engine_dots/tests/dots/test_wire_shapes.py`) writes what it really answers, and one event of each - * type and each set of keys its own writers produce, into `wire_shapes.json`; here that file is parsed with the - * schemas of `packages/shared` and the host's fake guest is held to the same answers, so neither side can move a key, - * or a rule of what the model is offered, without a suite failing. The host's schemas strip a key they do not know - * and the host drops an event they refuse, so an event must come out of its parse exactly as it went in. - */ -import { readFileSync } from "node:fs"; -import { fileURLToPath } from "node:url"; -import { - MAX_RUN_AT_MS, - mcpServerStatusSchema, - OUTBOUND_EVENT_TYPES, - parseDotConfig, - parseOutboundEvent, - PERMISSIONS, - skillSchema, - toolInfoSchema, - toRuntimeConfig, - type DotRuntimeConfig, -} from "@invisible-dots/shared"; -import { describe, expect, it } from "vitest"; -import { FakeGuest, type FakeAutomation } from "../src/testing.js"; - -interface OfferingCase { - permissions: Record; - offered: string[]; - tools: unknown[]; -} - -const shapes = JSON.parse( - readFileSync(fileURLToPath(new URL("../../../invisible_engine_dots/tests/dots/wire_shapes.json", import.meta.url)), "utf8"), -) as { - limits: { max_run_at_ms: number }; - outbound_events: { type: string; data: Record }[]; - tool_offering: OfferingCase[]; - mcp: { tools: unknown[]; mcp_servers: unknown[] }; - skills: unknown[]; -}; - -const baseConfig = toRuntimeConfig(parseDotConfig("name: shapes\nmodel:\n provider: openrouter\n id: test/model\n")); - -function configFor(offering: OfferingCase): DotRuntimeConfig { - return { - ...baseConfig, - permissions: offering.permissions as DotRuntimeConfig["permissions"], - }; -} - -describe("what the engine answers, as the host describes it", () => { - it("the last time the engine lets an automation run is the last time the host's schemas accept", () => { - expect(shapes.limits.max_run_at_ms).toBe(MAX_RUN_AT_MS); - const nextRun = (next_run_at_ms: number) => ({ seq: 1, id: "evt_x", type: "automation.next_run", ts: "2026-10-06T09:00:00.000Z", data: { next_run_at_ms } }); - expect(parseOutboundEvent(nextRun(MAX_RUN_AT_MS)).data).toEqual({ next_run_at_ms: MAX_RUN_AT_MS }); - expect(() => parseOutboundEvent(nextRun(MAX_RUN_AT_MS + 1))).toThrow(); - expect(new Date(MAX_RUN_AT_MS).toISOString()).toBe("9999-12-31T23:59:59.999Z"); - }); - - it("every outbound event the engine writes comes out of the host's parse as it went in: none is refused and no key is stripped", () => { - for (const [index, { type, data }] of shapes.outbound_events.entries()) { - const event = { seq: index + 1, id: `evt_${index}`, type, ts: "2026-10-06T09:00:00.000Z", data }; - expect(parseOutboundEvent(event).data, JSON.stringify(event)).toEqual(data); - } - // One of every type the contract names, so a type the engine writes and the host does not know (or the reverse) is seen. - expect([...new Set(shapes.outbound_events.map((event) => event.type))].sort()).toEqual([...OUTBOUND_EVENT_TYPES].sort()); - }); - - it("a key the schema does not know is stripped by the parse, which the check above sees, and a key it needs and the engine lacks is refused", () => { - const called = shapes.outbound_events.find((event) => event.type === "tool.called")!; - const around = (event: { type: string; data: Record }, data: Record) => ({ seq: 1, id: "evt_x", type: event.type, ts: "2026-10-06T09:00:00.000Z", data }); - expect(parseOutboundEvent(around(called, { ...called.data, renamed_by_the_engine: true })).data).not.toEqual({ ...called.data, renamed_by_the_engine: true }); - const { tool: _tool, ...withoutTool } = called.data; - expect(() => parseOutboundEvent(around(called, withoutTool))).toThrow(); - }); - - it("the optional keys of the events are all written by the engine: a terminal, an interrupted call, a task's calls, an approval with and without a task, a reply to a message, the spend", () => { - const keys = (type: string) => shapes.outbound_events.filter((event) => event.type === type).map((event) => Object.keys(event.data).sort().join()); - expect(keys("tool.called")).toEqual(expect.arrayContaining(["decision,duration_ms,ok,permission,target,tool,tty", "decision,duration_ms,interrupted,ok,permission,target,task_id,tool"])); - expect(keys("approval.requested")).toEqual(expect.arrayContaining(["approval_id,arguments,permission,reason,task_id,tool", "approval_id,arguments,permission,reason,tool"])); - expect(keys("message.assistant")).toEqual(["in_reply_to,spent_usd,text"]); - expect(keys("task.progress")).toEqual(["spent_usd,task_id,text"]); - expect(keys("task.failed")).toEqual(["error,spent_usd,task_id"]); - expect(keys("task.completed")).toEqual(["spent_usd,summary,task_id"]); - }); - - it("the host's fake guest writes the automation events as the engine does", async () => { - const written = shapes.outbound_events.filter((event) => event.type === "automation.next_run"); - expect(written.map((event) => event.data.next_run_at_ms)).toEqual([1_790_000_000_000, null]); - const guest = new FakeGuest("token-for-shapes"); - guest.running = true; - const row: FakeAutomation = { - id: "job_every", - name: "check the shop", - enabled: true, - schedule: { kind: "every", every_ms: 3_600_000 }, - message: "look at the orders", - next_run_at_ms: 1_790_000_000_000, - last_run_at_ms: null, - last_status: null, - last_error: null, - delete_after_run: false, - created_at_ms: 1_789_990_000_000, - }; - guest.putAutomation(row); - expect(guest.outbox.map((event) => ({ type: event.type, data: event.data }))).toEqual([written[0]]); - guest.removeAutomation(row.id); - expect(guest.outbox.map((event) => ({ type: event.type, data: event.data }))).toEqual(written); - }); - - it("every skill the engine shows parses with the skill schema, and a key it lacks or has extra is refused", () => { - expect(shapes.skills.map((row) => (row as { name: string }).name)).toContain("invisible-playwright"); - for (const row of shapes.skills) expect(skillSchema.safeParse(row).error?.issues, JSON.stringify(row)).toBeUndefined(); - const first = shapes.skills[0] as Record; - const { content: _content, ...missing } = first; - expect(skillSchema.safeParse(missing).success).toBe(false); - expect(skillSchema.safeParse({ ...first, renamed_key: 1 }).success).toBe(false); - }); - - it("every tool row the engine shows parses with the tool schema, in the order of its table, under a permission the host knows", () => { - for (const offering of shapes.tool_offering) { - for (const row of offering.tools) expect(toolInfoSchema.safeParse(row).error?.issues, JSON.stringify(row)).toBeUndefined(); - const names = (offering.tools as { name: string }[]).map((row) => row.name); - expect(names).toEqual((shapes.tool_offering[0]!.tools as { name: string }[]).map((row) => row.name)); - expect((offering.tools as { permission: string }[]).every((row) => PERMISSIONS.includes(row.permission as never))).toBe(true); - } - }); - - it("what the engine says of the MCP servers a config declares parses with the host's schemas: their tools under `mcp.`, their states", () => { - expect(shapes.mcp.tools.length).toBeGreaterThan(0); - for (const row of shapes.mcp.tools) expect(toolInfoSchema.safeParse(row).error?.issues, JSON.stringify(row)).toBeUndefined(); - expect(new Set((shapes.mcp.tools as { permission: string }[]).map((row) => row.permission))).toEqual(new Set(["mcp.tools"])); - for (const status of shapes.mcp.mcp_servers) expect(mcpServerStatusSchema.safeParse(status).error?.issues, JSON.stringify(status)).toBeUndefined(); - expect((shapes.mcp.mcp_servers as { name: string; state: string }[]).map((status) => [status.name, status.state])).toEqual([["keyed", "failed"], ["tools", "connected"]]); - expect(toolInfoSchema.safeParse({ ...(shapes.mcp.tools[0] as object), permission: "mcp.Not_A_Server" }).success).toBe(false); - }); - - it("the fake guest holds the engine's tools to the same permissions, and offers what the engine offers for the same config", async () => { - const engineRows = new Map((shapes.tool_offering[0]!.tools as { name: string; permission: string }[]).map((row) => [row.name, row.permission])); - for (const offering of shapes.tool_offering) { - const guest = new FakeGuest("token-for-shapes"); - guest.running = true; - guest.config = configFor(offering); - const { tools } = await guest.listTools(); - expect(tools.length).toBeGreaterThan(0); - for (const tool of tools) { - expect(engineRows.get(tool.name), `the engine has no tool ${tool.name}`).toBe(tool.permission); - expect(tool.offered, `${tool.name} with ${JSON.stringify(offering.permissions)}`).toBe(offering.offered.includes(tool.name)); - } - } - }); -}); diff --git a/apps/scheduler/test/fixtures/upgrade/golden-b326f2a1d4fe/host.sql b/apps/scheduler/test/fixtures/upgrade/golden-b326f2a1d4fe/host.sql deleted file mode 100644 index fab18cb4..00000000 --- a/apps/scheduler/test/fixtures/upgrade/golden-b326f2a1d4fe/host.sql +++ /dev/null @@ -1,768 +0,0 @@ --- --- PostgreSQL database dump --- - - --- Dumped from database version 18.4 --- Dumped by pg_dump version 18.4 - -SET statement_timeout = 0; -SET lock_timeout = 0; -SET idle_in_transaction_session_timeout = 0; -SET transaction_timeout = 0; -SET client_encoding = 'UTF8'; -SET standard_conforming_strings = on; -SET check_function_bodies = false; -SET xmloption = content; -SET client_min_messages = warning; -SET row_security = off; - -SET default_tablespace = ''; - -SET default_table_access_method = heap; - --- --- Name: approvals; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.approvals ( - id text NOT NULL, - dot_id text NOT NULL, - task_id text, - tool text NOT NULL, - permission text NOT NULL, - arguments jsonb NOT NULL, - reason text NOT NULL, - status text NOT NULL, - note text, - created_at timestamp with time zone DEFAULT now() NOT NULL, - resolved_at timestamp with time zone, - CONSTRAINT approvals_status_check CHECK ((status = ANY (ARRAY['pending'::text, 'approved'::text, 'rejected'::text, 'expired'::text]))) -); - - --- --- Name: channel_bindings; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.channel_bindings ( - id text NOT NULL, - dot_id text NOT NULL, - kind text NOT NULL, - enabled boolean DEFAULT true NOT NULL, - settings jsonb NOT NULL, - status text NOT NULL, - status_detail text, - account text, - event_cursor bigint NOT NULL, - created_at timestamp with time zone DEFAULT now() NOT NULL, - CONSTRAINT channel_bindings_event_cursor_check CHECK ((event_cursor >= 0)), - CONSTRAINT channel_bindings_kind_check CHECK ((kind = ANY (ARRAY['telegram'::text, 'whatsapp'::text]))), - CONSTRAINT channel_bindings_status_check CHECK ((status = ANY (ARRAY['connecting'::text, 'connected'::text, 'needs_relink'::text, 'error'::text]))) -); - - --- --- Name: channel_pairings; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.channel_pairings ( - binding_id text NOT NULL, - code_hash text NOT NULL, - expires_at timestamp with time zone NOT NULL, - consumed_at timestamp with time zone -); - - --- --- Name: channel_peers; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.channel_peers ( - binding_id text NOT NULL, - peer_id text NOT NULL, - chat_id text NOT NULL, - role text NOT NULL, - label text NOT NULL, - created_at timestamp with time zone DEFAULT now() NOT NULL, - CONSTRAINT channel_peers_role_check CHECK ((role = ANY (ARRAY['owner'::text, 'user'::text]))) -); - - --- --- Name: channel_prompts; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.channel_prompts ( - binding_id text NOT NULL, - approval_id text NOT NULL, - chat_id text NOT NULL, - ref text NOT NULL, - created_at timestamp with time zone DEFAULT now() NOT NULL -); - - --- --- Name: computers; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.computers ( - dot_id text NOT NULL, - vm_name text NOT NULL, - guest_port integer, - pid integer, - state text NOT NULL, - golden_image text, - runtime_image text, - token_enc bytea NOT NULL, - event_cursor bigint DEFAULT 0 NOT NULL, - last_active_at timestamp with time zone, - last_error text, - updated_at timestamp with time zone DEFAULT now() NOT NULL, - next_automation_at timestamp with time zone, - stop_reason text, - CONSTRAINT computers_guest_port_check CHECK (((guest_port >= 1) AND (guest_port <= 65535))), - CONSTRAINT computers_pid_check CHECK ((pid > 0)), - CONSTRAINT computers_state_check CHECK ((state = ANY (ARRAY['PROVISIONING'::text, 'STARTING'::text, 'RUNNING'::text, 'IDLE'::text, 'STOPPING'::text, 'STOPPED'::text, 'ERROR'::text, 'DELETING'::text]))), - CONSTRAINT computers_stop_reason_check CHECK ((stop_reason = ANY (ARRAY['idle'::text, 'user'::text, 'exited'::text]))) -); - - --- --- Name: dots; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.dots ( - id text NOT NULL, - name text NOT NULL, - config jsonb NOT NULL, - status text NOT NULL, - error text, - created_at timestamp with time zone DEFAULT now() NOT NULL, - updated_at timestamp with time zone DEFAULT now() NOT NULL, - config_version integer DEFAULT 1 NOT NULL, - CONSTRAINT dots_status_check CHECK ((status = ANY (ARRAY['CREATING'::text, 'READY'::text, 'IDLE'::text, 'RUNNING'::text, 'WAITING_APPROVAL'::text, 'ERROR'::text, 'DISABLED'::text]))) -); - - --- --- Name: events; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.events ( - id bigint NOT NULL, - dot_id text NOT NULL, - type text NOT NULL, - data jsonb NOT NULL, - source text NOT NULL, - guest_seq bigint, - created_at timestamp with time zone DEFAULT now() NOT NULL, - CONSTRAINT events_check CHECK (((source = 'guest'::text) = (guest_seq IS NOT NULL))), - CONSTRAINT events_source_check CHECK ((source = ANY (ARRAY['host'::text, 'guest'::text]))) -); - - --- --- Name: events_id_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE public.events_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: events_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - --- - -ALTER SEQUENCE public.events_id_seq OWNED BY public.events.id; - - --- --- Name: inbound_events; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.inbound_events ( - seq bigint NOT NULL, - id text NOT NULL, - dot_id text NOT NULL, - type text NOT NULL, - data jsonb NOT NULL, - ts text NOT NULL, - task_id text, - run_id text, - created_at timestamp with time zone DEFAULT now() NOT NULL, - sent_at timestamp with time zone, - delivered_at timestamp with time zone, - dropped_at timestamp with time zone, - drop_reason text, - failures integer DEFAULT 0 NOT NULL, - last_error text, - retry_at timestamp with time zone, - CONSTRAINT inbound_events_type_check CHECK ((type = ANY (ARRAY['user.message'::text, 'task.created'::text, 'approval.received'::text, 'system.event'::text]))) -); - - --- --- Name: inbound_events_seq_seq; Type: SEQUENCE; Schema: public; Owner: - --- - -CREATE SEQUENCE public.inbound_events_seq_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - - --- --- Name: inbound_events_seq_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - --- - -ALTER SEQUENCE public.inbound_events_seq_seq OWNED BY public.inbound_events.seq; - - --- --- Name: schema_migrations; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.schema_migrations ( - version text NOT NULL, - applied_at timestamp with time zone DEFAULT now() NOT NULL -); - - --- --- Name: secrets; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.secrets ( - scope text NOT NULL, - name text NOT NULL, - value_enc bytea NOT NULL, - updated_at timestamp with time zone DEFAULT now() NOT NULL -); - - --- --- Name: task_runs; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.task_runs ( - id text NOT NULL, - task_id text NOT NULL, - started_at timestamp with time zone DEFAULT now() NOT NULL, - delivered_at timestamp with time zone, - finished_at timestamp with time zone, - outcome text -); - - --- --- Name: tasks; Type: TABLE; Schema: public; Owner: - --- - -CREATE TABLE public.tasks ( - id text NOT NULL, - dot_id text NOT NULL, - description text NOT NULL, - priority integer DEFAULT 0 NOT NULL, - status text NOT NULL, - created_at timestamp with time zone DEFAULT now() NOT NULL, - scheduled_at timestamp with time zone, - started_at timestamp with time zone, - finished_at timestamp with time zone, - summary text, - error text, - spent_usd double precision DEFAULT 0 NOT NULL, - CONSTRAINT tasks_spent_usd_check CHECK ((spent_usd >= (0)::double precision)), - CONSTRAINT tasks_status_check CHECK ((status = ANY (ARRAY['PENDING'::text, 'RUNNING'::text, 'WAITING_APPROVAL'::text, 'COMPLETED'::text, 'FAILED'::text, 'CANCELLED'::text]))) -); - - --- --- Name: events id; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.events ALTER COLUMN id SET DEFAULT nextval('public.events_id_seq'::regclass); - - --- --- Name: inbound_events seq; Type: DEFAULT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.inbound_events ALTER COLUMN seq SET DEFAULT nextval('public.inbound_events_seq_seq'::regclass); - - --- --- Data for Name: approvals; Type: TABLE DATA; Schema: public; Owner: - --- - -INSERT INTO public.approvals VALUES ('apr_01m4ceepvn47ypbkcpd5cymdey', 'dot_01m4ceepkrt609r0tde6hm21hb', 'task_01m4ceeptfsmdeddevy4770sbk', 'browser_identity_delete', 'browser.identity.delete', '{"identity_id": "shop-abc123"}', 'the tool needs approval', 'approved', 'fine', '2026-10-08 00:26:18.038234+00', '2026-10-08 00:26:18.071102+00'); -INSERT INTO public.approvals VALUES ('apr_01m4ceepzq74326e3e6vrheqvj', 'dot_01m4ceepkrt609r0tde6hm21hb', 'task_01m4ceepz48cetk8s7k7zwkst5', 'browser_identity_delete', 'browser.identity.delete', '{"identity_id": "shop-abc123"}', 'the tool needs approval', 'pending', NULL, '2026-10-08 00:26:18.153524+00', NULL); - - --- --- Data for Name: channel_bindings; Type: TABLE DATA; Schema: public; Owner: - --- - -INSERT INTO public.channel_bindings VALUES ('chb_upgrade', 'dot_01m4ceepkrt609r0tde6hm21hb', 'telegram', true, '{"approvals": true, "notify_tasks": true, "show_arguments": false}', 'connecting', NULL, 'fare_watch_bot', 0, '2026-10-08 00:26:18.181416+00'); - - --- --- Data for Name: channel_pairings; Type: TABLE DATA; Schema: public; Owner: - --- - - - --- --- Data for Name: channel_peers; Type: TABLE DATA; Schema: public; Owner: - --- - -INSERT INTO public.channel_peers VALUES ('chb_upgrade', '4242', '4242', 'owner', 'Federico', '2026-10-08 00:26:18.187194+00'); - - --- --- Data for Name: channel_prompts; Type: TABLE DATA; Schema: public; Owner: - --- - - - --- --- Data for Name: computers; Type: TABLE DATA; Schema: public; Owner: - --- - -INSERT INTO public.computers VALUES ('dot_01m4ceepkrt609r0tde6hm21hb', 'invisible-dot-dot_01m4ceepkrt609r0tde6hm21hb', 47000, 9000, 'RUNNING', 'images/golden-test.qcow2', 'images/runtime-test.iso', '\xd8f1c54703e75012cfac1404e670ebfdb057db12ea08404cd0e3b7367c02826ead80d0e36069ab2c190ee026c9ef5059590e1c32c10d787db872b944b7da832b635e4b9117ced9', 16, '2026-10-07 09:00:00+00', NULL, '2026-10-08 00:26:18.181416+00', '2026-10-09 00:23:53.167+00', NULL); -INSERT INTO public.computers VALUES ('dot_01m4ceeq1d5fgv96gmxs71ht30', 'invisible-dot-dot_01m4ceeq1d5fgv96gmxs71ht30', NULL, NULL, 'STOPPED', 'images/golden-test.qcow2', 'images/runtime-test.iso', '\x292befaaec6513a1a36ee6a6efa250c5eaba66e80bc9c2e7bca66bf26f3ece3c52f63b6c00d9eaec7a44d828176b1994b779f5cc635738a95529d12917459e3b65ed4a130f74d4', 1, '2026-10-07 09:00:00+00', NULL, '2026-10-08 00:26:18.281007+00', NULL, 'user'); - - --- --- Data for Name: dots; Type: TABLE DATA; Schema: public; Owner: - --- - -INSERT INTO public.dots VALUES ('dot_01m4ceepkrt609r0tde6hm21hb', 'fare-watch', '{"name": "fare-watch", "model": {"id": "z-ai/glm-5.3-flash", "provider": "openrouter"}, "limits": {"context_tokens": 32000, "max_steps_per_task": 60, "max_cost_per_task_usd": 1}, "models": {}, "computer": {"cpu": 2, "disk": "40gb", "memory": "4gb", "idle_timeout": "15m"}, "permissions": {"files.write": "ask"}}', 'WAITING_APPROVAL', NULL, '2026-10-08 00:26:17.77945+00', '2026-10-08 00:26:18.145257+00', 3); -INSERT INTO public.dots VALUES ('dot_01m4ceeq1d5fgv96gmxs71ht30', 'minimal-dot', '{"name": "minimal-dot", "model": {"id": "z-ai/glm-5.3-flash", "provider": "openrouter"}, "limits": {"context_tokens": 32000, "max_steps_per_task": 60, "max_cost_per_task_usd": 1}, "models": {}, "computer": {"cpu": 2, "disk": "40gb", "memory": "4gb", "idle_timeout": "15m"}, "permissions": {"files.write": "ask"}}', 'IDLE', NULL, '2026-10-08 00:26:18.190622+00', '2026-10-08 00:26:18.281007+00', 1); - - --- --- Data for Name: events; Type: TABLE DATA; Schema: public; Owner: - --- - -INSERT INTO public.events VALUES (1, 'dot_01m4ceepkrt609r0tde6hm21hb', 'dot.created', '{"name": "fare-watch"}', 'host', NULL, '2026-10-08 00:26:17.77945+00'); -INSERT INTO public.events VALUES (2, 'dot_01m4ceepkrt609r0tde6hm21hb', 'computer.state', '{"state": "PROVISIONING"}', 'host', NULL, '2026-10-08 00:26:17.77945+00'); -INSERT INTO public.events VALUES (3, 'dot_01m4ceepkrt609r0tde6hm21hb', 'computer.state', '{"state": "STARTING"}', 'host', NULL, '2026-10-08 00:26:17.815323+00'); -INSERT INTO public.events VALUES (4, 'dot_01m4ceepkrt609r0tde6hm21hb', 'computer.state', '{"state": "RUNNING"}', 'host', NULL, '2026-10-08 00:26:17.823904+00'); -INSERT INTO public.events VALUES (5, 'dot_01m4ceepkrt609r0tde6hm21hb', 'computer.started', '{"guest_port": 47000, "runtime_image": "images/runtime-test.iso"}', 'host', NULL, '2026-10-08 00:26:17.823904+00'); -INSERT INTO public.events VALUES (6, 'dot_01m4ceepkrt609r0tde6hm21hb', 'agent.started', '{"guest_ts": "2026-10-08T00:26:17.640Z", "guest_event_id": "evt_01m4ceepn88vyrgn8zp99tx0zz"}', 'guest', 1, '2026-10-08 00:26:17.862351+00'); -INSERT INTO public.events VALUES (7, 'dot_01m4ceepkrt609r0tde6hm21hb', 'user.message', '{"text": "Remember: my favourite colour is teal.", "message_id": "msg_01m4ceepq4w4wdyp8tvy7v060r"}', 'host', NULL, '2026-10-08 00:26:17.880657+00'); -INSERT INTO public.events VALUES (8, 'dot_01m4ceepkrt609r0tde6hm21hb', 'message.assistant', '{"text": "echo: Remember: my favourite colour is teal.", "guest_ts": "2026-10-08T00:26:17.720Z", "in_reply_to": "msg_01m4ceepq4w4wdyp8tvy7v060r", "guest_event_id": "evt_01m4ceepqrxs5240rvb8cmrtee"}', 'guest', 2, '2026-10-08 00:26:17.899743+00'); -INSERT INTO public.events VALUES (9, 'dot_01m4ceepkrt609r0tde6hm21hb', 'task.created', '{"task_id": "task_01m4ceepr4pcpyejkrwmwszmvy", "priority": 0, "description": "List the workspace."}', 'host', NULL, '2026-10-08 00:26:17.910436+00'); -INSERT INTO public.events VALUES (10, 'dot_01m4ceepkrt609r0tde6hm21hb', 'agent.state', '{"state": "THINKING", "guest_ts": "2026-10-08T00:26:17.763Z", "guest_event_id": "evt_01m4ceeps3b18qmjz7x3acqxr7"}', 'guest', 3, '2026-10-08 00:26:17.940112+00'); -INSERT INTO public.events VALUES (11, 'dot_01m4ceepkrt609r0tde6hm21hb', 'task.started', '{"task_id": "task_01m4ceepr4pcpyejkrwmwszmvy", "guest_ts": "2026-10-08T00:26:17.763Z", "guest_event_id": "evt_01m4ceeps31c9j93jpd8nb3gz9"}', 'guest', 4, '2026-10-08 00:26:17.948021+00'); -INSERT INTO public.events VALUES (12, 'dot_01m4ceepkrt609r0tde6hm21hb', 'task.completed', '{"summary": "done: List the workspace.", "task_id": "task_01m4ceepr4pcpyejkrwmwszmvy", "guest_ts": "2026-10-08T00:26:17.763Z", "guest_event_id": "evt_01m4ceeps3hawexcrf77xqs9wq"}', 'guest', 5, '2026-10-08 00:26:17.956503+00'); -INSERT INTO public.events VALUES (13, 'dot_01m4ceepkrt609r0tde6hm21hb', 'agent.state', '{"state": "IDLE", "guest_ts": "2026-10-08T00:26:17.763Z", "guest_event_id": "evt_01m4ceeps3parat5eey42mqgsj"}', 'guest', 6, '2026-10-08 00:26:17.966985+00'); -INSERT INTO public.events VALUES (14, 'dot_01m4ceepkrt609r0tde6hm21hb', 'task.created', '{"task_id": "task_01m4ceeptfsmdeddevy4770sbk", "priority": 0, "description": "Delete the old identity."}', 'host', NULL, '2026-10-08 00:26:17.981828+00'); -INSERT INTO public.events VALUES (15, 'dot_01m4ceepkrt609r0tde6hm21hb', 'task.started', '{"task_id": "task_01m4ceeptfsmdeddevy4770sbk", "guest_ts": "2026-10-08T00:26:17.845Z", "guest_event_id": "evt_01m4ceepvn3vzncer00rmzep10"}', 'guest', 7, '2026-10-08 00:26:18.018234+00'); -INSERT INTO public.events VALUES (16, 'dot_01m4ceepkrt609r0tde6hm21hb', 'agent.state', '{"state": "WAITING_APPROVAL", "guest_ts": "2026-10-08T00:26:17.845Z", "guest_event_id": "evt_01m4ceepvnc1nwxjjf3q3w032b"}', 'guest', 8, '2026-10-08 00:26:18.029602+00'); -INSERT INTO public.events VALUES (17, 'dot_01m4ceepkrt609r0tde6hm21hb', 'approval.requested', '{"tool": "browser_identity_delete", "reason": "the tool needs approval", "task_id": "task_01m4ceeptfsmdeddevy4770sbk", "guest_ts": "2026-10-08T00:26:17.845Z", "arguments": {"identity_id": "shop-abc123"}, "permission": "browser.identity.delete", "approval_id": "apr_01m4ceepvn47ypbkcpd5cymdey", "guest_event_id": "evt_01m4ceepvnz1rg79k8y3p2hh5g"}', 'guest', 9, '2026-10-08 00:26:18.038234+00'); -INSERT INTO public.events VALUES (18, 'dot_01m4ceepkrt609r0tde6hm21hb', 'approval.resolved', '{"note": "fine", "always": true, "task_id": "task_01m4ceeptfsmdeddevy4770sbk", "decision": "approve", "approval_id": "apr_01m4ceepvn47ypbkcpd5cymdey"}', 'host', NULL, '2026-10-08 00:26:18.071102+00'); -INSERT INTO public.events VALUES (19, 'dot_01m4ceepkrt609r0tde6hm21hb', 'dot.updated', '{"name": "fare-watch"}', 'host', NULL, '2026-10-08 00:26:18.071102+00'); -INSERT INTO public.events VALUES (20, 'dot_01m4ceepkrt609r0tde6hm21hb', 'agent.state', '{"state": "EXECUTING", "guest_ts": "2026-10-08T00:26:17.924Z", "guest_event_id": "evt_01m4ceepy4fbsgfqd96zgj6t4v"}', 'guest', 10, '2026-10-08 00:26:18.091455+00'); -INSERT INTO public.events VALUES (21, 'dot_01m4ceepkrt609r0tde6hm21hb', 'task.completed', '{"summary": "approval approved", "task_id": "task_01m4ceeptfsmdeddevy4770sbk", "guest_ts": "2026-10-08T00:26:17.924Z", "guest_event_id": "evt_01m4ceepy4ea0fjr9b6g6kpk29"}', 'guest', 11, '2026-10-08 00:26:18.100452+00'); -INSERT INTO public.events VALUES (22, 'dot_01m4ceepkrt609r0tde6hm21hb', 'agent.state', '{"state": "IDLE", "guest_ts": "2026-10-08T00:26:17.924Z", "guest_event_id": "evt_01m4ceepy48vx98jjv03q60ajm"}', 'guest', 12, '2026-10-08 00:26:18.107693+00'); -INSERT INTO public.events VALUES (23, 'dot_01m4ceepkrt609r0tde6hm21hb', 'dot.updated', '{"name": "fare-watch"}', 'host', NULL, '2026-10-08 00:26:18.111019+00'); -INSERT INTO public.events VALUES (24, 'dot_01m4ceepkrt609r0tde6hm21hb', 'task.created', '{"task_id": "task_01m4ceepz48cetk8s7k7zwkst5", "priority": 0, "description": "Delete the shop identity."}', 'host', NULL, '2026-10-08 00:26:18.121808+00'); -INSERT INTO public.events VALUES (25, 'dot_01m4ceepkrt609r0tde6hm21hb', 'task.started', '{"task_id": "task_01m4ceepz48cetk8s7k7zwkst5", "guest_ts": "2026-10-08T00:26:17.975Z", "guest_event_id": "evt_01m4ceepzqq05k889g7kweerrc"}', 'guest', 13, '2026-10-08 00:26:18.139836+00'); -INSERT INTO public.events VALUES (26, 'dot_01m4ceepkrt609r0tde6hm21hb', 'agent.state', '{"state": "WAITING_APPROVAL", "guest_ts": "2026-10-08T00:26:17.975Z", "guest_event_id": "evt_01m4ceepzqxj1pzcgx1zba0mcr"}', 'guest', 14, '2026-10-08 00:26:18.145257+00'); -INSERT INTO public.events VALUES (27, 'dot_01m4ceepkrt609r0tde6hm21hb', 'approval.requested', '{"tool": "browser_identity_delete", "reason": "the tool needs approval", "task_id": "task_01m4ceepz48cetk8s7k7zwkst5", "guest_ts": "2026-10-08T00:26:17.975Z", "arguments": {"identity_id": "shop-abc123"}, "permission": "browser.identity.delete", "approval_id": "apr_01m4ceepzq74326e3e6vrheqvj", "guest_event_id": "evt_01m4ceepzq7pb07ekz37zhskey"}', 'guest', 15, '2026-10-08 00:26:18.153524+00'); -INSERT INTO public.events VALUES (28, 'dot_01m4ceepkrt609r0tde6hm21hb', 'task.created', '{"task_id": "task_01m4ceeq0s2zdfycr7vwnwy9s8", "priority": 0, "description": "Check the fares again."}', 'host', NULL, '2026-10-08 00:26:18.171462+00'); -INSERT INTO public.events VALUES (29, 'dot_01m4ceepkrt609r0tde6hm21hb', 'task.cancelled', '{"task_id": "task_01m4ceeq0s2zdfycr7vwnwy9s8"}', 'host', NULL, '2026-10-08 00:26:18.176182+00'); -INSERT INTO public.events VALUES (30, 'dot_01m4ceepkrt609r0tde6hm21hb', 'automation.next_run', '{"guest_ts": "2026-10-08T00:26:18.019Z", "guest_event_id": "evt_01m4ceeq13g8b426wcn3432mbv", "next_run_at_ms": 1791505433167}', 'guest', 16, '2026-10-08 00:26:18.181416+00'); -INSERT INTO public.events VALUES (31, 'dot_01m4ceeq1d5fgv96gmxs71ht30', 'dot.created', '{"name": "minimal-dot"}', 'host', NULL, '2026-10-08 00:26:18.190622+00'); -INSERT INTO public.events VALUES (32, 'dot_01m4ceeq1d5fgv96gmxs71ht30', 'computer.state', '{"state": "PROVISIONING"}', 'host', NULL, '2026-10-08 00:26:18.190622+00'); -INSERT INTO public.events VALUES (33, 'dot_01m4ceeq1d5fgv96gmxs71ht30', 'computer.state', '{"state": "STARTING"}', 'host', NULL, '2026-10-08 00:26:18.203391+00'); -INSERT INTO public.events VALUES (34, 'dot_01m4ceeq1d5fgv96gmxs71ht30', 'computer.state', '{"state": "RUNNING"}', 'host', NULL, '2026-10-08 00:26:18.208314+00'); -INSERT INTO public.events VALUES (35, 'dot_01m4ceeq1d5fgv96gmxs71ht30', 'computer.started', '{"guest_port": 47001, "runtime_image": "images/runtime-test.iso"}', 'host', NULL, '2026-10-08 00:26:18.208314+00'); -INSERT INTO public.events VALUES (36, 'dot_01m4ceeq1d5fgv96gmxs71ht30', 'agent.started', '{"guest_ts": "2026-10-08T00:26:18.048Z", "guest_event_id": "evt_01m4ceeq206y59sjefjm80tgar"}', 'guest', 1, '2026-10-08 00:26:18.237511+00'); -INSERT INTO public.events VALUES (37, 'dot_01m4ceeq1d5fgv96gmxs71ht30', 'computer.state', '{"state": "STOPPING"}', 'host', NULL, '2026-10-08 00:26:18.260481+00'); -INSERT INTO public.events VALUES (38, 'dot_01m4ceeq1d5fgv96gmxs71ht30', 'computer.state', '{"state": "STOPPED"}', 'host', NULL, '2026-10-08 00:26:18.281007+00'); -INSERT INTO public.events VALUES (39, 'dot_01m4ceeq1d5fgv96gmxs71ht30', 'computer.stopped', '{"forced": false, "reason": "user"}', 'host', NULL, '2026-10-08 00:26:18.281007+00'); - - --- --- Data for Name: inbound_events; Type: TABLE DATA; Schema: public; Owner: - --- - -INSERT INTO public.inbound_events VALUES (1, 'msg_01m4ceepq4w4wdyp8tvy7v060r', 'dot_01m4ceepkrt609r0tde6hm21hb', 'user.message', '{"text": "Remember: my favourite colour is teal."}', '2026-10-07T09:00:00.000Z', NULL, NULL, '2026-10-08 00:26:17.880657+00', '2026-10-08 00:26:17.892461+00', '2026-10-08 00:26:17.899719+00', NULL, NULL, 0, NULL, NULL); -INSERT INTO public.inbound_events VALUES (2, 'evt_01m4ceeprmq0dfftt3ejd0v7qc', 'dot_01m4ceepkrt609r0tde6hm21hb', 'task.created', '{"task_id": "task_01m4ceepr4pcpyejkrwmwszmvy", "priority": 0, "description": "List the workspace."}', '2026-10-07T09:00:00.000Z', 'task_01m4ceepr4pcpyejkrwmwszmvy', 'run_01m4ceeprj3zwtyjmrn8wpefzz', '2026-10-08 00:26:17.918409+00', '2026-10-08 00:26:17.933733+00', '2026-10-08 00:26:17.940076+00', NULL, NULL, 0, NULL, NULL); -INSERT INTO public.inbound_events VALUES (3, 'evt_01m4ceepv0asgrb9zz7j54fjsp', 'dot_01m4ceepkrt609r0tde6hm21hb', 'task.created', '{"task_id": "task_01m4ceeptfsmdeddevy4770sbk", "priority": 0, "description": "Delete the old identity."}', '2026-10-07T09:00:00.000Z', 'task_01m4ceeptfsmdeddevy4770sbk', 'run_01m4ceeptymc59t9c34rmqzq8v', '2026-10-08 00:26:17.989407+00', '2026-10-08 00:26:18.010516+00', '2026-10-08 00:26:18.018374+00', NULL, NULL, 0, NULL, NULL); -INSERT INTO public.inbound_events VALUES (4, 'evt_01m4ceepxe33m1qnqvsfbzvkvc', 'dot_01m4ceepkrt609r0tde6hm21hb', 'approval.received', '{"note": "fine", "decision": "approve", "approval_id": "apr_01m4ceepvn47ypbkcpd5cymdey"}', '2026-10-07T09:00:00.000Z', NULL, NULL, '2026-10-08 00:26:18.071102+00', '2026-10-08 00:26:18.087155+00', '2026-10-08 00:26:18.091459+00', NULL, NULL, 0, NULL, NULL); -INSERT INTO public.inbound_events VALUES (5, 'evt_01m4ceepzer7yzzxfn4t1hwway', 'dot_01m4ceepkrt609r0tde6hm21hb', 'task.created', '{"task_id": "task_01m4ceepz48cetk8s7k7zwkst5", "priority": 0, "description": "Delete the shop identity."}', '2026-10-07T09:00:00.000Z', 'task_01m4ceepz48cetk8s7k7zwkst5', 'run_01m4ceepzd75vf6eyq6s83tvxf', '2026-10-08 00:26:18.126436+00', '2026-10-08 00:26:18.135244+00', '2026-10-08 00:26:18.139817+00', NULL, NULL, 0, NULL, NULL); - - --- --- Data for Name: schema_migrations; Type: TABLE DATA; Schema: public; Owner: - --- - -INSERT INTO public.schema_migrations VALUES ('0001_initial', '2026-10-08 00:26:17.623866+00'); -INSERT INTO public.schema_migrations VALUES ('0002_inbound_events', '2026-10-08 00:26:17.666723+00'); -INSERT INTO public.schema_migrations VALUES ('0003_task_spend', '2026-10-08 00:26:17.680464+00'); -INSERT INTO public.schema_migrations VALUES ('0004_channels', '2026-10-08 00:26:17.688232+00'); -INSERT INTO public.schema_migrations VALUES ('0005_channel_prompts', '2026-10-08 00:26:17.707123+00'); -INSERT INTO public.schema_migrations VALUES ('0006_events_task', '2026-10-08 00:26:17.718165+00'); -INSERT INTO public.schema_migrations VALUES ('0007_dot_config_version', '2026-10-08 00:26:17.724644+00'); -INSERT INTO public.schema_migrations VALUES ('0008_computer_next_automation', '2026-10-08 00:26:17.730628+00'); -INSERT INTO public.schema_migrations VALUES ('0009_dot_keeps_its_memory', '2026-10-08 00:26:17.737384+00'); -INSERT INTO public.schema_migrations VALUES ('0010_dot_has_no_goal', '2026-10-08 00:26:17.745491+00'); -INSERT INTO public.schema_migrations VALUES ('0011_dot_has_no_browser_settings', '2026-10-08 00:26:17.753326+00'); - - --- --- Data for Name: secrets; Type: TABLE DATA; Schema: public; Owner: - --- - -INSERT INTO public.secrets VALUES ('global', 'openrouter_api_key', '\x83ecec4851c227f6a0848b8123934629df0b90a2a63abc4b38d243360f1593e3604f9658463bada064baf61238436101953919de8d0551c0946b3f7cb6adbefbbf3da3', '2026-10-08 00:26:17.762727+00'); -INSERT INTO public.secrets VALUES ('dot_01m4ceepkrt609r0tde6hm21hb', 'telegram_bot_token', '\xcb8b7dc5772170ab2190e83956526daf3705cd174d2c1d80cffa2d1dcd5709736fbde9a872341e083d0586749322d18eeede5662b8b012c46505bd92aba13e6a8417a8', '2026-10-08 00:26:18.184962+00'); - - --- --- Data for Name: task_runs; Type: TABLE DATA; Schema: public; Owner: - --- - -INSERT INTO public.task_runs VALUES ('run_01m4ceeprj3zwtyjmrn8wpefzz', 'task_01m4ceepr4pcpyejkrwmwszmvy', '2026-10-08 00:26:17.918409+00', '2026-10-08 00:26:17.942498+00', '2026-10-08 00:26:17.956503+00', 'completed'); -INSERT INTO public.task_runs VALUES ('run_01m4ceeptymc59t9c34rmqzq8v', 'task_01m4ceeptfsmdeddevy4770sbk', '2026-10-08 00:26:17.989407+00', '2026-10-08 00:26:18.023517+00', '2026-10-08 00:26:18.100452+00', 'completed'); -INSERT INTO public.task_runs VALUES ('run_01m4ceepzd75vf6eyq6s83tvxf', 'task_01m4ceepz48cetk8s7k7zwkst5', '2026-10-08 00:26:18.126436+00', '2026-10-08 00:26:18.14187+00', NULL, NULL); - - --- --- Data for Name: tasks; Type: TABLE DATA; Schema: public; Owner: - --- - -INSERT INTO public.tasks VALUES ('task_01m4ceepr4pcpyejkrwmwszmvy', 'dot_01m4ceepkrt609r0tde6hm21hb', 'List the workspace.', 0, 'COMPLETED', '2026-10-08 00:26:17.910436+00', NULL, '2026-10-08 00:26:17.918409+00', '2026-10-08 00:26:17.956503+00', 'done: List the workspace.', NULL, 0); -INSERT INTO public.tasks VALUES ('task_01m4ceeptfsmdeddevy4770sbk', 'dot_01m4ceepkrt609r0tde6hm21hb', 'Delete the old identity.', 0, 'COMPLETED', '2026-10-08 00:26:17.981828+00', NULL, '2026-10-08 00:26:17.989407+00', '2026-10-08 00:26:18.100452+00', 'approval approved', NULL, 0); -INSERT INTO public.tasks VALUES ('task_01m4ceepz48cetk8s7k7zwkst5', 'dot_01m4ceepkrt609r0tde6hm21hb', 'Delete the shop identity.', 0, 'WAITING_APPROVAL', '2026-10-08 00:26:18.121808+00', NULL, '2026-10-08 00:26:18.126436+00', NULL, NULL, NULL, 0); -INSERT INTO public.tasks VALUES ('task_01m4ceeq0s2zdfycr7vwnwy9s8', 'dot_01m4ceepkrt609r0tde6hm21hb', 'Check the fares again.', 0, 'CANCELLED', '2026-10-08 00:26:18.171462+00', NULL, NULL, '2026-10-08 00:26:18.176182+00', NULL, 'cancelled by the user', 0); - - --- --- Name: events_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - --- - -SELECT pg_catalog.setval('public.events_id_seq', 39, true); - - --- --- Name: inbound_events_seq_seq; Type: SEQUENCE SET; Schema: public; Owner: - --- - -SELECT pg_catalog.setval('public.inbound_events_seq_seq', 5, true); - - --- --- Name: approvals approvals_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.approvals - ADD CONSTRAINT approvals_pkey PRIMARY KEY (id); - - --- --- Name: channel_bindings channel_bindings_dot_id_kind_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.channel_bindings - ADD CONSTRAINT channel_bindings_dot_id_kind_key UNIQUE (dot_id, kind); - - --- --- Name: channel_bindings channel_bindings_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.channel_bindings - ADD CONSTRAINT channel_bindings_pkey PRIMARY KEY (id); - - --- --- Name: channel_pairings channel_pairings_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.channel_pairings - ADD CONSTRAINT channel_pairings_pkey PRIMARY KEY (binding_id, code_hash); - - --- --- Name: channel_peers channel_peers_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.channel_peers - ADD CONSTRAINT channel_peers_pkey PRIMARY KEY (binding_id, peer_id); - - --- --- Name: channel_prompts channel_prompts_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.channel_prompts - ADD CONSTRAINT channel_prompts_pkey PRIMARY KEY (binding_id, approval_id, chat_id); - - --- --- Name: computers computers_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.computers - ADD CONSTRAINT computers_pkey PRIMARY KEY (dot_id); - - --- --- Name: dots dots_name_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.dots - ADD CONSTRAINT dots_name_key UNIQUE (name); - - --- --- Name: dots dots_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.dots - ADD CONSTRAINT dots_pkey PRIMARY KEY (id); - - --- --- Name: events events_dot_id_guest_seq_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.events - ADD CONSTRAINT events_dot_id_guest_seq_key UNIQUE (dot_id, guest_seq); - - --- --- Name: events events_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.events - ADD CONSTRAINT events_pkey PRIMARY KEY (id); - - --- --- Name: inbound_events inbound_events_id_key; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.inbound_events - ADD CONSTRAINT inbound_events_id_key UNIQUE (id); - - --- --- Name: inbound_events inbound_events_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.inbound_events - ADD CONSTRAINT inbound_events_pkey PRIMARY KEY (seq); - - --- --- Name: schema_migrations schema_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.schema_migrations - ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version); - - --- --- Name: secrets secrets_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.secrets - ADD CONSTRAINT secrets_pkey PRIMARY KEY (scope, name); - - --- --- Name: task_runs task_runs_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.task_runs - ADD CONSTRAINT task_runs_pkey PRIMARY KEY (id); - - --- --- Name: tasks tasks_pkey; Type: CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.tasks - ADD CONSTRAINT tasks_pkey PRIMARY KEY (id); - - --- --- Name: approvals_status_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX approvals_status_idx ON public.approvals USING btree (status, created_at); - - --- --- Name: channel_bindings_account_key; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX channel_bindings_account_key ON public.channel_bindings USING btree (account) WHERE ((kind = 'telegram'::text) AND (account IS NOT NULL)); - - --- --- Name: channel_peers_chat_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX channel_peers_chat_idx ON public.channel_peers USING btree (binding_id, chat_id); - - --- --- Name: channel_prompts_approval_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX channel_prompts_approval_idx ON public.channel_prompts USING btree (approval_id); - - --- --- Name: events_dot_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX events_dot_idx ON public.events USING btree (dot_id, id); - - --- --- Name: events_task_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX events_task_idx ON public.events USING btree (dot_id, ((data ->> 'task_id'::text)), id) WHERE ((data ->> 'task_id'::text) IS NOT NULL); - - --- --- Name: events_user_message_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX events_user_message_idx ON public.events USING btree (dot_id, ((data ->> 'message_id'::text))) WHERE (type = 'user.message'::text); - - --- --- Name: events_user_message_origin_key; Type: INDEX; Schema: public; Owner: - --- - -CREATE UNIQUE INDEX events_user_message_origin_key ON public.events USING btree (dot_id, (((data -> 'origin'::text) ->> 'binding_id'::text)), (((data -> 'origin'::text) ->> 'external_id'::text))) WHERE ((type = 'user.message'::text) AND ((data -> 'origin'::text) IS NOT NULL)); - - --- --- Name: inbound_events_pending_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX inbound_events_pending_idx ON public.inbound_events USING btree (dot_id, seq) WHERE ((delivered_at IS NULL) AND (dropped_at IS NULL)); - - --- --- Name: task_runs_task_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX task_runs_task_idx ON public.task_runs USING btree (task_id); - - --- --- Name: tasks_dot_status_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX tasks_dot_status_idx ON public.tasks USING btree (dot_id, status); - - --- --- Name: tasks_pending_idx; Type: INDEX; Schema: public; Owner: - --- - -CREATE INDEX tasks_pending_idx ON public.tasks USING btree (priority DESC, created_at, id) WHERE (status = 'PENDING'::text); - - --- --- Name: approvals approvals_dot_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.approvals - ADD CONSTRAINT approvals_dot_id_fkey FOREIGN KEY (dot_id) REFERENCES public.dots(id) ON DELETE CASCADE; - - --- --- Name: channel_bindings channel_bindings_dot_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.channel_bindings - ADD CONSTRAINT channel_bindings_dot_id_fkey FOREIGN KEY (dot_id) REFERENCES public.dots(id) ON DELETE CASCADE; - - --- --- Name: channel_pairings channel_pairings_binding_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.channel_pairings - ADD CONSTRAINT channel_pairings_binding_id_fkey FOREIGN KEY (binding_id) REFERENCES public.channel_bindings(id) ON DELETE CASCADE; - - --- --- Name: channel_peers channel_peers_binding_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.channel_peers - ADD CONSTRAINT channel_peers_binding_id_fkey FOREIGN KEY (binding_id) REFERENCES public.channel_bindings(id) ON DELETE CASCADE; - - --- --- Name: channel_prompts channel_prompts_approval_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.channel_prompts - ADD CONSTRAINT channel_prompts_approval_id_fkey FOREIGN KEY (approval_id) REFERENCES public.approvals(id) ON DELETE CASCADE; - - --- --- Name: channel_prompts channel_prompts_binding_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.channel_prompts - ADD CONSTRAINT channel_prompts_binding_id_fkey FOREIGN KEY (binding_id) REFERENCES public.channel_bindings(id) ON DELETE CASCADE; - - --- --- Name: computers computers_dot_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.computers - ADD CONSTRAINT computers_dot_id_fkey FOREIGN KEY (dot_id) REFERENCES public.dots(id) ON DELETE CASCADE; - - --- --- Name: inbound_events inbound_events_dot_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.inbound_events - ADD CONSTRAINT inbound_events_dot_id_fkey FOREIGN KEY (dot_id) REFERENCES public.dots(id) ON DELETE CASCADE; - - --- --- Name: task_runs task_runs_task_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.task_runs - ADD CONSTRAINT task_runs_task_id_fkey FOREIGN KEY (task_id) REFERENCES public.tasks(id) ON DELETE CASCADE; - - --- --- Name: tasks tasks_dot_id_fkey; Type: FK CONSTRAINT; Schema: public; Owner: - --- - -ALTER TABLE ONLY public.tasks - ADD CONSTRAINT tasks_dot_id_fkey FOREIGN KEY (dot_id) REFERENCES public.dots(id) ON DELETE CASCADE; - - --- --- PostgreSQL database dump complete --- - - diff --git a/apps/scheduler/test/fixtures/upgrade/golden-b326f2a1d4fe/master.key b/apps/scheduler/test/fixtures/upgrade/golden-b326f2a1d4fe/master.key deleted file mode 100644 index a59389d8..00000000 --- a/apps/scheduler/test/fixtures/upgrade/golden-b326f2a1d4fe/master.key +++ /dev/null @@ -1 +0,0 @@ -f38457d1f14a7ea3ee7119f7dc463369ae540f0ba0036e1001a40dba8a997e56 diff --git a/apps/scheduler/test/fixtures/upgrade/make-host-state.ts b/apps/scheduler/test/fixtures/upgrade/make-host-state.ts deleted file mode 100644 index 84088a85..00000000 --- a/apps/scheduler/test/fixtures/upgrade/make-host-state.ts +++ /dev/null @@ -1,111 +0,0 @@ -/** - * Writes the control plane's database as a release leaves it, for test/upgrade.test.ts. - * - * Run it from the root of a checkout of the release (a worktree of its tag), against a PostgreSQL server - * (it makes a database of its own there and prints its name), then dump that database, leaving out psql's - * meta-commands and the dump's reset of search_path, which would outlive the load on PGlite: - * - * DATABASE_URL=postgres://... npx tsx fixtures/upgrade//master.key - * pg_dump --inserts --no-owner --no-privileges | grep -v -e '^\\' -e "set_config('search_path'" \ - * > fixtures/upgrade//host.sql - * - * It drives that release's scheduler, with a fake computer, through what a person's install holds: two - * Dots (one stopped), a chat answered, a completed task, a task waiting for an approval, an approval - * answered with "always", a cancelled task, an automation, a Telegram binding with its token, and the - * OpenRouter key. The master key it writes is a test key: it only opens the secrets of the dump. - */ -import { randomBytes } from "node:crypto"; -import { writeFile } from "node:fs/promises"; -import { Database } from "@invisible-dots/database"; -import { createScratchPostgres } from "@invisible-dots/database/testing"; -import { Scheduler } from "../../../src/index.js"; -import { FakeDriver, ManualClock, waitFor, waitUntilSettledReady } from "../../../src/testing.js"; - -const [keyFile] = process.argv.slice(2); -if (!keyFile) throw new Error("usage: make-host-state.ts "); -const masterKey = randomBytes(32); -const { url } = await createScratchPostgres(); -const db = await Database.open({ target: { kind: "pg", url }, masterKey, poolSize: 5 }); -await db.migrate(); -const driver = new FakeDriver(); -const scheduler = new Scheduler({ - db, - driver, - clock: new ManualClock(new Date("2026-10-07T09:00:00.000Z")), - lifecycle: { healthPollMs: 5, readyTimeoutMs: 3_000, pumpRetryMs: 10, pumpMaxRetryMs: 50 }, - dispatcher: { retryDelayMs: 0, maxDeliveryAttempts: 2 }, -}); -await scheduler.setOpenRouterKey("sk-or-v1-upgrade-fixture-not-a-real-key"); - -const yaml = (name: string) => - `name: ${name}\nmodel:\n provider: openrouter\n id: z-ai/glm-5.3-flash\ncomputer:\n idle_timeout: 15m\npermissions:\n files.write: ask\n`; -const ready = async (name: string) => { - const dot = await scheduler.createDot(yaml(name)); - await waitUntilSettledReady(scheduler, driver, dot.id, name); - return dot; -}; - -const watcher = await ready("fare-watch"); -const guest = driver.guestOf(watcher.id); - -// A chat answered, and a completed task. -await scheduler.sendMessage(watcher.id, "Remember: my favourite colour is teal."); -const done = await scheduler.createTask(watcher.id, { description: "List the workspace." }); -await waitFor(async () => (await db.tasks.get(done.id))?.status === "COMPLETED", "task completed"); - -// An approval answered with "always", and a task that waits for an approval when the release is upgraded. -const original = guest.onInbound; -guest.onInbound = (event, g) => { - if (event.type !== "task.created") return original(event, g); - g.emit("task.started", { task_id: event.data.task_id }); - g.requestApproval(event.data.task_id); -}; -const allowed = await scheduler.createTask(watcher.id, { description: "Delete the old identity." }); -const always = await waitFor(async () => (await scheduler.listApprovals("pending")).find((a) => a.task_id === allowed.id), "first approval"); -await scheduler.resolveApproval(always.id, "approve", { always: true, note: "fine" }); -await waitFor(async () => (await db.tasks.get(allowed.id))?.status === "COMPLETED", "approved task completed"); -await scheduler.updateDot(watcher.id, yaml("fare-watch")); -const waiting = await scheduler.createTask(watcher.id, { description: "Delete the shop identity." }); -await waitFor(async () => (await db.tasks.get(waiting.id))?.status === "WAITING_APPROVAL", "task waiting"); - -// A cancelled task, queued behind the waiting one. -const cancelled = await scheduler.createTask(watcher.id, { description: "Check the fares again." }); -await scheduler.cancelTask(cancelled.id); - -// An automation the Dot reported. -guest.putAutomation({ - id: "job_water", - name: "water the plants", - enabled: true, - schedule: { kind: "every", every_ms: 86_400_000 }, - message: "Water the plants", - next_run_at_ms: 1_791_505_433_167, - last_run_at_ms: null, - last_status: null, - last_error: null, - delete_after_run: false, - created_at_ms: 1_791_419_033_167, -}); - -// A Telegram binding with its token and a paired owner. -const binding = await db.channels.createBinding({ - id: "chb_upgrade", - dotId: watcher.id, - kind: "telegram", - settings: { approvals: true, notify_tasks: true, show_arguments: false }, - eventCursor: 0, - account: "fare_watch_bot", -}); -await db.secrets.put(watcher.id, "telegram_bot_token", "123456:upgrade-fixture-not-a-real-token"); -await db.channels.upsertPeer({ bindingId: binding.id, peerId: "4242", chatId: "4242", role: "owner", label: "Federico" }); - -// A second Dot, stopped. -const minimal = await ready("minimal-dot"); -await scheduler.stopComputer(minimal.id); -await waitFor(async () => (await db.computers.get(minimal.id))?.state === "STOPPED", "computer stopped"); - -await scheduler.settle(); -await scheduler.close(); -await db.close(); -await writeFile(keyFile, masterKey.toString("hex") + "\n"); -console.log(new URL(url).pathname.slice(1)); diff --git a/apps/scheduler/test/host-shapes.test.ts b/apps/scheduler/test/host-shapes.test.ts deleted file mode 100644 index cbf56cfb..00000000 --- a/apps/scheduler/test/host-shapes.test.ts +++ /dev/null @@ -1,148 +0,0 @@ -/** - * What the host sends the engine: the events (`POST /events`) and the Dot's config (`PUT /config`), pinned in a file - * the engine checks. They are not written by hand here: a real scheduler runs a Dot against the fake guest, and what - * its own writers send is what the guest received. The engine is Python and parses them with its own parsers - * (`invisible_engine_dots/tests/dots/test_host_wire_shapes.py`), so neither side can move a key, or a rule of what is - * accepted, without a suite failing. The other direction, what the engine writes, is `engine-shapes.test.ts`. - * - * A key renamed or added here fails this test until the file is written again - * (`UPDATE_HOST_WIRE_SHAPES=1 npx vitest run apps/scheduler/test/host-shapes.test.ts`), and then fails the engine's - * test until its parser says the same. - */ -import { readFileSync, writeFileSync } from "node:fs"; -import { fileURLToPath } from "node:url"; -import { createTestDatabase, type TestDatabase } from "@invisible-dots/database/testing"; -import { INBOUND_EVENT_TYPES } from "@invisible-dots/shared"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { Scheduler } from "../src/index.js"; -import { FakeDriver, ManualClock, waitFor, waitUntilSettledReady } from "../src/testing.js"; - -const FIXTURE = fileURLToPath(new URL("../../../invisible_engine_dots/tests/dots/host_wire_shapes.json", import.meta.url)); - -const MINIMAL = "name: shapes-min\nmodel:\n provider: openrouter\n id: test/model\n"; -const FULL = `name: shapes-full -instructions: Answer in one line. -model: - provider: openrouter - id: test/model -models: - summary: test/small -computer: - cpu: 4 - memory: 6G - disk: 30G - idle_timeout: 5m -permissions: - computer.exec: deny - files.write: ask - mcp.time: allow -mcp_servers: - time: - command: uvx - args: [mcp-server-time, --local-timezone, Europe/Rome] - env: - LOG_LEVEL: warning - secrets: [TIME_TOKEN] - search: - url: https://search.example/mcp - headers: - X-Client: dots - secrets: [Authorization] - timeout_s: 30 -limits: - max_steps_per_task: 30 - max_cost_per_task_usd: 2.5 -`; - -describe("what the host sends the engine", () => { - let t: TestDatabase; - let scheduler: Scheduler; - let written: Awaited>; - const driver = new FakeDriver(); - - beforeAll(async () => { - t = await createTestDatabase("pglite"); - await t.db.secrets.put("global", "openrouter_api_key", "sk-or-test"); - scheduler = new Scheduler({ - db: t.db, - driver, - clock: new ManualClock(), - lifecycle: { healthPollMs: 5, readyTimeoutMs: 3_000, pumpRetryMs: 10, pumpMaxRetryMs: 50 }, - dispatcher: { retryDelayMs: 0, maxDeliveryAttempts: 2 }, - }); - written = await shapes(); - }); - - afterAll(async () => { - await scheduler?.close(); - await t?.drop(); - }); - - async function readyDot(yaml: string, name: string) { - const dot = await scheduler.createDot(yaml); - await waitUntilSettledReady(scheduler, driver, dot.id, name); - return dot; - } - - async function shapes() { - const configs = []; - for (const [name, yaml] of [["shapes-min", MINIMAL], ["shapes-full", FULL]] as const) { - const dot = await readyDot(yaml, name); - configs.push({ name, config: structuredClone(driver.guestOf(dot.id).config) }); - } - // The secrets of the full config's servers, set as the person sets them: the push carries them with the key. - const full = await scheduler.requireDot("shapes-full"); - await scheduler.setMcpSecret(full.id, "time", "TIME_TOKEN", "tt-1"); - await scheduler.setMcpSecret(full.id, "search", "Authorization", "Bearer sk-search-1"); - const fullGuest = driver.guestOf(full.id); - const secrets_requests = [{ openrouter_api_key: fullGuest.openrouterKey, mcp_secrets: structuredClone(fullGuest.mcpSecrets) }]; - - const dot = await readyDot("name: shapes-talk\nmodel:\n provider: openrouter\n id: test/model\n", "shapes-talk"); - const guest = driver.guestOf(dot.id); - // A quiet guest: it takes a task up and says nothing else, so nothing but what the host sends is in `inbound`. - guest.onInbound = (event, g) => { - if (event.type === "task.created") g.emit("task.started", { task_id: event.data.task_id }); - }; - await scheduler.sendMessage(dot.id, "what is the cheapest fare to Lisbon?"); - const task = await scheduler.createTask(dot.id, { description: "compare the fares", priority: 5 }); - await waitFor(async () => (await scheduler.getTask(task.id)).status === "RUNNING", "the task running"); - const plain = await scheduler.createTask(dot.id, { description: "a task with the default priority" }); - await scheduler.cancelTask(task.id); - await waitFor(() => guest.inbound.some((event) => event.type === "system.event"), "the cancel reaching the guest"); - const approved = guest.requestApproval(undefined); - const rejected = guest.requestApproval(undefined); - await waitFor(async () => (await scheduler.listApprovals("pending")).filter((a) => a.dot_id === dot.id).length === 2, "two pending approvals"); - await scheduler.resolveApproval(approved, "approve", { note: "go on" }); - await scheduler.resolveApproval(rejected, "reject"); - await waitFor(() => guest.inbound.filter((event) => event.type === "approval.received").length === 2, "both answers reaching the guest"); - expect(plain.priority).toBe(0); - - // The id and the time of an event are the host's own and vary: the engine's parser takes any, and the file is about - // the data. The ids of tasks and approvals, drawn at random, are named in the order they appear. - const named = new Map([[task.id, "task-1"], [plain.id, "task-2"], [approved, "approval-1"], [rejected, "approval-2"]]); - const inbound_events = (JSON.parse( - [...named].reduce((text, [id, name]) => text.replaceAll(id, name), JSON.stringify(guest.inbound.map((event) => ({ type: event.type, data: event.data })))), - ) as { type: string; data: Record }[]); - return { runtime_configs: configs, secrets_requests, inbound_events }; - } - - it("writes what the host's own writers sent, as the file the engine's parsers check", async () => { - const text = `${JSON.stringify(written, null, 2)}\n`; - if (process.env.UPDATE_HOST_WIRE_SHAPES === "1") writeFileSync(FIXTURE, text); - expect(JSON.parse(readFileSync(FIXTURE, "utf8"))).toEqual(JSON.parse(text)); - }); - - it("holds one event of each type the host sends, the optional note of an answer with and without it, and a config with every section set", async () => { - expect([...new Set(written.inbound_events.map((event) => event.type))].sort()).toEqual([...INBOUND_EVENT_TYPES].sort()); - const answers = written.inbound_events.filter((event) => event.type === "approval.received").map((event) => Object.keys(event.data).sort().join()); - expect(answers).toEqual(["approval_id,decision,note", "approval_id,decision"]); - const priorities = written.inbound_events.filter((event) => event.type === "task.created").map((event) => event.data.priority); - expect(priorities).toEqual([5, 0]); - const [minimal, full] = written.runtime_configs.map((entry) => entry.config as Record); - expect(Object.keys(full!).sort()).toEqual(["instructions", "limits", "mcp_servers", "model", "models", "name", "permissions"]); - expect(Object.keys(full!.mcp_servers as object).sort()).toEqual(["search", "time"]); - expect(written.secrets_requests[0]!.mcp_secrets).toEqual({ time: { TIME_TOKEN: "tt-1" }, search: { Authorization: "Bearer sk-search-1" } }); - expect(Object.keys(minimal!)).not.toContain("instructions"); - expect(minimal).not.toHaveProperty("computer"); - }); -}); diff --git a/apps/scheduler/test/scheduler.test.ts b/apps/scheduler/test/scheduler.test.ts deleted file mode 100644 index d458e9a5..00000000 --- a/apps/scheduler/test/scheduler.test.ts +++ /dev/null @@ -1,1506 +0,0 @@ -import type { Database } from "@invisible-dots/database"; -import { createTestDatabase, testAdapters, type TestDatabase } from "@invisible-dots/database/testing"; -import { APPROVAL_NOTE_MAX, IDENTITY_ERROR_STATUS, type IdentityErrorCode, type StoredEvent } from "@invisible-dots/shared"; -import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; -import { Scheduler, type SchedulerOptions } from "../src/index.js"; -import { FakeDriver, FakeGuestError, ManualClock, waitFor, waitUntilSettledReady, type FakeAutomation } from "../src/testing.js"; - - -const yaml = (name: string, idle = "15m") => - `name: ${name}\nmodel:\n provider: openrouter\n id: test/model\ncomputer:\n idle_timeout: ${idle}\n`; - -const automation = (over: Partial = {}): FakeAutomation => ({ - id: "job_1", - name: "daily fares", - enabled: true, - schedule: { kind: "cron", expr: "0 9 * * 1-5", tz: "Europe/Rome" }, - message: "check the fares", - next_run_at_ms: 1_800_000_000_000, - last_run_at_ms: null, - last_status: null, - last_error: null, - delete_after_run: false, - created_at_ms: 1_700_000_000_000, - ...over, -}); - -describe.each(testAdapters())("Scheduler with a fake driver and a fake guest (%s)", (kind) => { - let t: TestDatabase; - let db: Database; - const open: Scheduler[] = []; - - beforeAll(async () => { - t = await createTestDatabase(kind); - db = t.db; - await db.secrets.put("global", "openrouter_api_key", "sk-or-test"); - }); - - afterEach(async () => { - await Promise.all(open.splice(0).map((s) => s.close())); - }); - - afterAll(async () => { - await t?.drop(); - }); - - function make(driver = new FakeDriver(), extra: Partial = {}) { - const clock = new ManualClock(); - const scheduler = new Scheduler({ - db, - driver, - clock, - lifecycle: { healthPollMs: 5, readyTimeoutMs: 3_000, pumpRetryMs: 10, pumpMaxRetryMs: 50 }, - dispatcher: { retryDelayMs: 0, maxDeliveryAttempts: 2 }, - ...extra, - }); - open.push(scheduler); - drivers.set(scheduler, driver); - return { scheduler, driver, clock }; - } - - const drivers = new Map(); - const driverOf = (s: Scheduler) => drivers.get(s)!; - - async function readyDot(s: Scheduler, name: string, idle?: string) { - const dot = await s.createDot(yaml(name, idle)); - await waitUntilSettledReady(s, driverOf(s), dot.id, name); - return dot; - } - - const types = (events: StoredEvent[]) => events.map((e) => e.type); - - // PGlite has one connection: no other transaction can hold a row while the claim runs. - if (kind === "pg") it("claims a task whose Dot's row another transaction held for a moment, without waiting for the timer", async () => { - const { scheduler } = make(); - const dot = await readyDot(scheduler, "held-row"); - // What the guest's next event does as a task ends (agent.state IDLE moves the Dot to READY): an update of the - // Dot's row, which the claim's FOR UPDATE OF d SKIP LOCKED skips over. - let release!: () => void; - const released = new Promise((resolve) => (release = resolve)); - let holding!: () => void; - const held = new Promise((resolve) => (holding = resolve)); - const holder = db.transaction(async (tx) => { - await tx.dots.setStatus(dot.id, "READY"); - holding(); - await released; - }); - await held; - const task = await scheduler.createTask(dot.id, { description: "after the held row" }); - // The dispatch the task asked for has tried, and skipped it: the row is still held. - await new Promise((resolve) => setTimeout(resolve, 100)); - expect((await db.tasks.get(task.id))?.status).toBe("PENDING"); - release(); - await holder; - // The scheduler is not started here: no timer passes; only the dispatch itself can come back for the task. - await waitFor(async () => (await db.tasks.get(task.id))?.status !== "PENDING", "the task claimed once the row was free", 3_000); - }); - - it("create -> READY: pushes the key and the runtime config, records images and state events", async () => { - const { scheduler, driver } = make(); - const dot = await scheduler.createDot(yaml("ready-one")); - expect(dot.status).toBe("CREATING"); - await waitFor(async () => (await db.dots.get(dot.id))?.status === "READY", "READY"); - - const guest = driver.guestOf(dot.id); - expect(guest.openrouterKey).toBe("sk-or-test"); - expect(guest.config?.name).toBe("ready-one"); - expect(guest.config).not.toHaveProperty("computer"); - const computer = await scheduler.computer(dot.id); - expect(computer).toMatchObject({ state: "RUNNING", ready: true, golden_image: driver.goldenImage, vm_name: `invisible-dot-${dot.id}` }); - expect(computer.guest_port).toBe(driver.vms.get(dot.id)?.guestPort); - expect(computer.pid).toBe(driver.vms.get(dot.id)?.pid); - const events = await db.events.list({ dotId: dot.id }); - expect(types(events)).toEqual( - expect.arrayContaining(["dot.created", "computer.state", "computer.started"]), - ); - expect(events.filter((e) => e.type === "computer.state").map((e) => e.data.state)).toEqual([ - "PROVISIONING", - "STARTING", - "RUNNING", - ]); - }); - - it("task -> guest events -> COMPLETED, with the cursor advanced and the run delivered", async () => { - const { scheduler } = make(); - const dot = await readyDot(scheduler, "task-one"); - const task = await scheduler.createTask(dot.id, { description: "check fares", priority: 2 }); - await waitFor(async () => (await db.tasks.get(task.id))?.status === "COMPLETED", "task COMPLETED"); - const done = await db.tasks.get(task.id); - expect(done?.summary).toBe("done: check fares"); - expect(done?.started_at).not.toBeNull(); - const runs = await db.tasks.runs(task.id); - expect(runs).toHaveLength(1); - expect(runs[0]?.delivered_at).not.toBeNull(); - expect(runs[0]?.outcome).toBe("completed"); - await waitFor(async () => (await db.computers.get(dot.id))?.event_cursor === 5, "every guest event stored"); - const guestEvents = (await db.events.list({ dotId: dot.id })).filter((e) => e.source === "guest"); - expect(types(guestEvents)).toEqual(["agent.started", "agent.state", "task.started", "task.completed", "agent.state"]); - expect((await db.dots.get(dot.id))?.status).toBe("READY"); - }); - - it("a message of the guest's stream that the host refuses is a host event of its own, the cursor moves past it, and what follows is stored", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "refused-one"); - const guest = driver.guestOf(dot.id); - await waitFor(async () => (await db.computers.get(dot.id))?.event_cursor === guest.outbox.length, "the guest's events stored"); - const stored = guest.outbox.length; - - guest.writeUnreadable("approval.requested", "data.permission: Invalid option"); - guest.sendGarbage("Unexpected token 'x', \"xyz\" is not valid JSON"); - await waitFor(async () => (await db.events.list({ dotId: dot.id, types: ["guest.event.refused"] })).length === 2, "both refusals recorded"); - const refused = await db.events.list({ dotId: dot.id, types: ["guest.event.refused"] }); - expect(refused.map((e) => e.data)).toEqual([ - { seq: stored + 1, type: "approval.requested", problem: "data.permission: Invalid option" }, - { seq: null, type: null, problem: "Unexpected token 'x', \"xyz\" is not valid JSON" }, - ]); - expect(refused.every((e) => e.source === "host" && e.guest_seq === null)).toBe(true); - // The one with a seq is behind the cursor, so a reconnect does not record it again; one without a seq cannot be - // told apart from the others, and is recorded again. - expect((await db.computers.get(dot.id))?.event_cursor).toBe(stored + 1); - guest.disconnectStreams(); - guest.emit("agent.state", { state: "THINKING" }); - await waitFor(async () => (await db.computers.get(dot.id))?.event_cursor === stored + 2, "the event after it stored"); - expect(await db.events.list({ dotId: dot.id, types: ["guest.event.refused"] })).toHaveLength(3); - }); - - it("the spend a task's events report lands on the task: the highest heard, whatever ends it", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "spend-one"); - const guest = driver.guestOf(dot.id); - guest.onInbound = () => {}; - const task = await scheduler.createTask(dot.id, { description: "costly" }); - await waitFor(async () => (await db.tasks.runs(task.id))[0]?.delivered_at, "delivered"); - const spent = async () => (await db.tasks.get(task.id))?.spent_usd; - guest.emit("task.started", { task_id: task.id }); - guest.emit("task.progress", { task_id: task.id, text: "first step", spent_usd: 0.25 }); - await waitFor(async () => (await spent()) === 0.25, "progress spend"); - guest.emit("task.progress", { task_id: task.id, text: "second step", spent_usd: 0.75 }); - await waitFor(async () => (await spent()) === 0.75, "ticking spend"); - guest.emit("task.completed", { task_id: task.id, summary: "done", spent_usd: 0.875 }); - await waitFor(async () => (await db.tasks.get(task.id))?.status === "COMPLETED", "completed"); - expect(await spent()).toBe(0.875); - expect((await db.tasks.get(task.id))?.summary).toBe("done"); - }); - - it("a failed task keeps the spend its failure reports, and an event without one changes nothing", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "spend-two"); - const guest = driver.guestOf(dot.id); - guest.onInbound = () => {}; - const task = await scheduler.createTask(dot.id, { description: "capped" }); - await waitFor(async () => (await db.tasks.runs(task.id))[0]?.delivered_at, "delivered"); - guest.emit("task.progress", { task_id: task.id, text: "x", spent_usd: 0.6 }); - guest.emit("task.progress", { task_id: task.id, text: "no spend reported" }); - guest.emit("task.failed", { task_id: task.id, error: "stopped: the task reached limits.max_cost_per_task_usd (spent 1.2000 USD of 1.00)", spent_usd: 1.2 }); - await waitFor(async () => (await db.tasks.get(task.id))?.status === "FAILED", "failed"); - expect((await db.tasks.get(task.id))?.spent_usd).toBe(1.2); - expect((await scheduler.getTask(task.id)).spent_usd).toBe(1.2); - }); - - it("a cancelled task still takes the spend its guest reports afterwards", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "spend-three"); - const guest = driver.guestOf(dot.id); - guest.onInbound = () => {}; - const task = await scheduler.createTask(dot.id, { description: "cancelled while spending" }); - await waitFor(async () => (await db.tasks.runs(task.id))[0]?.delivered_at, "delivered"); - guest.emit("task.progress", { task_id: task.id, text: "x", spent_usd: 0.25 }); - await waitFor(async () => (await db.tasks.get(task.id))?.spent_usd === 0.25, "first spend"); - await scheduler.cancelTask(task.id); - guest.emit("task.failed", { task_id: task.id, error: "cancelled", spent_usd: 0.4 }); - await waitFor(async () => (await db.tasks.get(task.id))?.spent_usd === 0.4, "spend after the cancel"); - expect((await db.tasks.get(task.id))?.status).toBe("CANCELLED"); - }); - - it("a guest cannot write the spend of another Dot's task", async () => { - const { scheduler, driver } = make(); - const a = await readyDot(scheduler, "spend-owner"); - const b = await readyDot(scheduler, "spend-other"); - driver.guestOf(a.id).onInbound = () => {}; - const task = await scheduler.createTask(a.id, { description: "mine" }); - await waitFor(async () => (await db.tasks.runs(task.id))[0]?.delivered_at, "delivered"); - driver.guestOf(b.id).emit("task.progress", { task_id: task.id, text: "forged", spent_usd: 99 }); - await waitFor(async () => (await db.computers.get(b.id))?.event_cursor === 1, "forged event stored"); - expect((await db.tasks.get(task.id))?.spent_usd).toBe(0); - }); - - it("a reconnecting event stream does not store an event twice", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "replay-one"); - const guest = driver.guestOf(dot.id); - const a = guest.emit("automation.next_run", { next_run_at_ms: 1_800_000_000_000 }); - await waitFor(async () => (await db.computers.get(dot.id))?.event_cursor === a.seq, "first event"); - guest.disconnectStreams(); - const b = guest.emit("automation.next_run", { next_run_at_ms: null }); - await waitFor(async () => (await db.computers.get(dot.id))?.event_cursor === b.seq, "second event after reconnect"); - const stored = (await db.events.list({ dotId: dot.id })).filter((e) => e.source === "guest"); - expect(stored.map((e) => e.guest_seq)).toEqual(guest.outbox.map((e) => e.seq)); - }); - - it("a guest cannot complete another Dot's task", async () => { - const { scheduler, driver } = make(); - const a = await readyDot(scheduler, "owner-a"); - const b = await readyDot(scheduler, "owner-b"); - driver.guestOf(a.id).onInbound = () => {}; - const task = await scheduler.createTask(a.id, { description: "slow" }); - await waitFor(async () => (await db.tasks.runs(task.id))[0]?.delivered_at, "delivered"); - driver.guestOf(b.id).emit("task.completed", { task_id: task.id, summary: "forged" }); - await waitFor(async () => (await db.computers.get(b.id))?.event_cursor === 1, "forged event stored"); - expect((await db.tasks.get(task.id))?.status).toBe("RUNNING"); - }); - - it("approval flow: requested -> WAITING_APPROVAL -> approve -> guest resumes -> COMPLETED", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "approver"); - const guest = driver.guestOf(dot.id); - const original = guest.onInbound; - guest.onInbound = (event, g) => { - if (event.type === "task.created") { - g.emit("task.started", { task_id: event.data.task_id }); - g.requestApproval(event.data.task_id); - } else { - return original(event, g); - } - }; - const task = await scheduler.createTask(dot.id, { description: "delete the old identity" }); - const approval = await waitFor(async () => (await scheduler.listApprovals("pending"))[0], "pending approval"); - expect(approval).toMatchObject({ dot_id: dot.id, task_id: task.id, tool: "browser_identity_delete" }); - await waitFor(async () => (await db.tasks.get(task.id))?.status === "WAITING_APPROVAL", "task waiting"); - await waitFor(async () => (await db.dots.get(dot.id))?.status === "WAITING_APPROVAL", "dot waiting"); - - const resolved = await scheduler.resolveApproval(approval.id, "approve", { note: "go ahead" }); - expect(resolved.status).toBe("approved"); - const received = guest.inbound.find((e) => e.type === "approval.received"); - expect(received?.data).toEqual({ approval_id: approval.id, decision: "approve", note: "go ahead" }); - await waitFor(async () => (await db.tasks.get(task.id))?.status === "COMPLETED", "task completed"); - expect((await db.tasks.get(task.id))?.summary).toBe("approval approved"); - await expect(scheduler.resolveApproval(approval.id, "reject")).rejects.toMatchObject({ - status: 409, - code: "already_resolved", - }); - expect(types(await db.events.list({ dotId: dot.id, types: ["approval.resolved"] }))).toEqual(["approval.resolved"]); - }); - - it("an approve and a reject sent at the same moment: one wins, the other is a 409, and the guest hears one decision", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "two-answers"); - const guest = driver.guestOf(dot.id); - expect(guest.config?.permissions["browser.identity.delete"]).toBe("ask"); - for (let round = 0; round < 5; round++) { - const id = guest.requestApproval(undefined); - await waitFor(async () => (await db.approvals.get(id))?.status === "pending", "pending approval"); - - const answers = await Promise.allSettled([ - scheduler.resolveApproval(id, "approve", { always: true }), - scheduler.resolveApproval(id, "reject", { note: "no" }), - ]); - - const won = answers.filter((a) => a.status === "fulfilled"); - const lost = answers.filter((a) => a.status === "rejected"); - expect([won.length, lost.length]).toEqual([1, 1]); - expect((lost[0] as PromiseRejectedResult).reason).toMatchObject({ status: 409, code: "already_resolved" }); - const winner = (won[0] as PromiseFulfilledResult<{ status: string }>).value.status; - expect((await db.approvals.get(id))?.status).toBe(winner); - const decision = winner === "approved" ? "approve" : "reject"; - await waitFor(() => guest.inbound.some((e) => e.type === "approval.received" && e.data.approval_id === id), "decision delivered"); - expect(guest.inbound.flatMap((e) => (e.type === "approval.received" && e.data.approval_id === id ? [e.data.decision] : []))).toEqual([decision]); - const resolved = (await db.events.list({ dotId: dot.id, types: ["approval.resolved"] })).filter((e) => e.data.approval_id === id); - expect(resolved.map((e) => e.data.decision)).toEqual([decision]); - // "Always" changes the permission only when the approve won. - expect((await db.dots.get(dot.id))?.config.permissions["browser.identity.delete"] === "allow").toBe(winner === "approved"); - await db.dots.setPermission(dot.id, "browser.identity.delete", "ask"); - } - }); - - it("approve with always: the permission becomes allow in the config and the guest, once, in the same transaction as the decision", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "always-allow"); - const guest = driver.guestOf(dot.id); - expect(guest.config?.permissions["browser.identity.delete"]).toBe("ask"); - const original = guest.onInbound; - guest.onInbound = (event, g) => { - if (event.type === "task.created") { - g.emit("task.started", { task_id: event.data.task_id }); - g.requestApproval(event.data.task_id); - } else { - return original(event, g); - } - }; - const task = await scheduler.createTask(dot.id, { description: "delete the old identity" }); - const approval = await waitFor(async () => (await scheduler.listApprovals("pending")).find((a) => a.task_id === task.id), "pending approval"); - const configBefore = (await db.dots.get(dot.id))!.config; - - const resolved = await scheduler.resolveApproval(approval.id, "approve", { always: true, note: "fine" }); - expect(resolved.status).toBe("approved"); - const stored = (await db.dots.get(dot.id))!; - expect(stored.config).toEqual({ ...configBefore, permissions: { ...configBefore.permissions, "browser.identity.delete": "allow" } }); - expect(guest.config?.permissions["browser.identity.delete"]).toBe("allow"); - expect(guest.config?.name).toBe("always-allow"); - const events = await db.events.list({ dotId: dot.id, types: ["approval.resolved", "dot.updated"] }); - expect(events.map((e) => [e.type, e.data])).toEqual([ - // The task the approval was asked in is named, so that the events of the task hold the answer. - ["approval.resolved", { approval_id: approval.id, decision: "approve", task_id: task.id, note: "fine", always: true }], - ["dot.updated", { name: "always-allow" }], - ]); - // The guest is told the plain decision: what the config says is the guest's own business. - expect(guest.inbound.find((e) => e.type === "approval.received")?.data).toEqual({ approval_id: approval.id, decision: "approve", note: "fine" }); - await waitFor(async () => (await db.tasks.get(task.id))?.status === "COMPLETED", "task completed"); - - // Answered once: the second try is a 409 and changes nothing, however it asks. - await db.dots.setPermission(dot.id, "browser.identity.delete", "ask"); - await expect(scheduler.resolveApproval(approval.id, "approve", { always: true })).rejects.toMatchObject({ status: 409, code: "already_resolved" }); - expect((await db.dots.get(dot.id))?.config.permissions["browser.identity.delete"]).toBe("ask"); - expect(types(await db.events.list({ dotId: dot.id, types: ["approval.resolved", "dot.updated"] }))).toEqual(["approval.resolved", "dot.updated"]); - }); - - it("approve with always: a flush that runs between the commit and the push does not send the answer ahead of the config", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "always-ordered"); - const guest = driver.guestOf(dot.id); - const id = guest.requestApproval(undefined); - await waitFor(async () => (await db.approvals.get(id)) !== null, "stored"); - let configWhenAnswered: string | undefined; - const original = guest.onInbound; - guest.onInbound = (event, g) => { - if (event.type === "approval.received") configWhenAnswered = g.config?.permissions["browser.identity.delete"]; - return original(event, g); - }; - let openGate!: () => void; - guest.configPushGate = new Promise((resolve) => { - openGate = resolve; - }); - - const answered = scheduler.resolveApproval(id, "approve", { always: true }); - // The answer is committed and waits in the outbox, the push is in flight: every way a delivery can start is tried. - await waitFor(async () => (await db.inbound.pending(dot.id)).some((row) => row.event.type === "approval.received"), "the answer stored"); - void scheduler.inbound.kick(dot.id); - await scheduler.inbound.kickDue(); - await new Promise((resolve) => setTimeout(resolve, 150)); - expect(guest.inbound.some((e) => e.type === "approval.received")).toBe(false); - - openGate(); - expect((await answered).status).toBe("approved"); - await scheduler.settle(); - expect(guest.inbound.some((e) => e.type === "approval.received")).toBe(true); - expect(configWhenAnswered).toBe("allow"); - // The hold is gone: the next answer for this Dot is not held up. - const next = guest.requestApproval(undefined); - await waitFor(async () => (await db.approvals.get(next)) !== null, "stored"); - await scheduler.resolveApproval(next, "reject"); - await scheduler.settle(); - expect(guest.inbound.filter((e) => e.type === "approval.received")).toHaveLength(2); - }); - - it("refuses a note longer than the one the web takes, and leaves the approval pending", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "long-note"); - const id = driver.guestOf(dot.id).requestApproval(undefined); - await waitFor(async () => (await db.approvals.get(id)) !== null, "stored"); - - await expect(scheduler.resolveApproval(id, "approve", { note: "x".repeat(APPROVAL_NOTE_MAX + 1) })).rejects.toMatchObject({ status: 400, code: "invalid_request", message: `note must be at most ${APPROVAL_NOTE_MAX} characters` }); - expect((await db.approvals.get(id))?.status).toBe("pending"); - expect((await scheduler.resolveApproval(id, "approve", { note: "x".repeat(APPROVAL_NOTE_MAX) })).note).toHaveLength(APPROVAL_NOTE_MAX); - }); - - it("approve with always refuses what cannot be allowed for good and leaves the approval pending", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "always-refused"); - const guest = driver.guestOf(dot.id); - const known = guest.requestApproval(undefined); - await waitFor(async () => (await db.approvals.get(known)) !== null, "stored"); - const configBefore = (await db.dots.get(dot.id))!.config; - - await expect(scheduler.resolveApproval(known, "reject", { always: true })).rejects.toMatchObject({ status: 400, code: "invalid_request" }); - await expect(scheduler.resolveApproval(known, "approve", { always: false as unknown as true })).rejects.toMatchObject({ status: 400 }); - await expect(scheduler.resolveApproval("apr_missing", "approve", { always: true })).rejects.toMatchObject({ status: 404 }); - expect((await db.approvals.get(known))?.status).toBe("pending"); - expect((await db.dots.get(dot.id))?.config).toEqual(configBefore); - expect(await db.events.list({ dotId: dot.id, types: ["approval.resolved", "dot.updated"] })).toEqual([]); - }); - - it("approve with always on a sleeping Dot saves the config, and the wake that delivers the answer pushes it", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "always-asleep"); - const approvalId = driver.guestOf(dot.id).requestApproval(undefined); - await waitFor(async () => (await db.approvals.get(approvalId)) !== null, "stored"); - await scheduler.stopComputer(dot.id); - await scheduler.settle(); - expect((await db.computers.get(dot.id))?.state).toBe("STOPPED"); - - await scheduler.resolveApproval(approvalId, "approve", { always: true }); - expect((await db.dots.get(dot.id))?.config.permissions["browser.identity.delete"]).toBe("allow"); - expect((await db.events.list({ dotId: dot.id, types: ["dot.updated"] })).at(-1)?.data).toEqual({ name: "always-asleep" }); - await waitFor(async () => (await db.computers.get(dot.id))?.state === "RUNNING", "woken for the answer"); - await waitFor(() => driver.guestOf(dot.id).config?.permissions["browser.identity.delete"] === "allow", "config pushed on the wake"); - }); - - it("sleeps after idle_timeout (fake clock) and wakes on a new task", async () => { - const { scheduler, driver, clock } = make(); - const dot = await readyDot(scheduler, "sleeper", "10m"); - expect(await scheduler.idleCheck()).toEqual([]); - clock.advance(9 * 60_000); - expect(await scheduler.idleCheck()).toEqual([]); - clock.advance(2 * 60_000); - expect(await scheduler.idleCheck()).toEqual([dot.id]); - await scheduler.settle(); - expect((await db.computers.get(dot.id))?.state).toBe("STOPPED"); - expect((await db.dots.get(dot.id))?.status).toBe("IDLE"); - expect(driver.guestOf(dot.id).calls).toContain("prepareSleep"); - expect(driver.calls).toContain(`stop:${dot.id}`); - const stopped = await db.events.list({ dotId: dot.id, types: ["computer.stopped"] }); - expect(stopped[0]?.data).toMatchObject({ reason: "idle" }); - - const bootsBefore = driver.guestOf(dot.id).boots; - const task = await scheduler.createTask(dot.id, { description: "after the nap" }); - await waitFor(async () => (await db.tasks.get(task.id))?.status === "COMPLETED", "task after wake"); - const guest = driver.guestOf(dot.id); - expect(guest.boots).toBe(bootsBefore + 1); - // The key lives in guest memory only and is pushed again on every READY. - expect(guest.openrouterKey).toBe("sk-or-test"); - expect((await db.computers.get(dot.id))?.state).toBe("RUNNING"); - }); - - it("does not sleep with idle_timeout 0, with a task in progress, or with a non-idle agent", async () => { - const { scheduler, driver, clock } = make(); - const never = await readyDot(scheduler, "insomniac", "0"); - const busy = await readyDot(scheduler, "busy-bee", "1m"); - driver.guestOf(busy.id).onInbound = (event, g) => { - if (event.type === "task.created") g.emit("agent.state", { state: "EXECUTING" }); - }; - const task = await scheduler.createTask(busy.id, { description: "long job" }); - await waitFor(async () => (await db.dots.get(busy.id))?.status === "RUNNING", "busy agent"); - clock.advance(24 * 3_600_000); - expect(await scheduler.idleCheck()).toEqual([]); - expect((await db.computers.get(never.id))?.state).toBe("RUNNING"); - expect((await db.tasks.get(task.id))?.status).toBe("RUNNING"); - }); - - describe("automations and the computer's sleep (section 9.5)", () => { - const LEAD_MS = 60_000; - const MIN = 60_000; - - function makeWithLead() { - return make(new FakeDriver(), { - dispatchIntervalMs: 60_000, - idleCheckIntervalMs: 60_000, - lifecycle: { healthPollMs: 5, readyTimeoutMs: 3_000, pumpRetryMs: 10, pumpMaxRetryMs: 50, automationWakeLeadMs: LEAD_MS }, - }); - } - - /** The Dot's engine reports that its earliest automation is due `inMs` from the clock's now, and the host has stored it. */ - async function reportDueIn(scheduler: Scheduler, clock: ManualClock, dotId: string, inMs: number, id = "job_1") { - const at = clock.now().getTime() + inMs; - driverOf(scheduler).guestOf(dotId).putAutomation(automation({ id, next_run_at_ms: at })); - await waitFor(async () => (await db.computers.get(dotId))?.next_automation_at === new Date(at).toISOString(), "the report stored"); - return at; - } - - it("stores what the engine reports as the time of its next automation, and clears it when none is due", async () => { - const { scheduler, driver, clock } = makeWithLead(); - const dot = await readyDot(scheduler, "reporter", "0"); - expect((await db.computers.get(dot.id))?.next_automation_at).toBeNull(); - - const at = await reportDueIn(scheduler, clock, dot.id, 3 * 60 * MIN); - const logged = await db.events.list({ dotId: dot.id, types: ["automation.next_run"] }); - expect(logged.map((e) => e.data)).toMatchObject([{ next_run_at_ms: at }]); - - const guest = driver.guestOf(dot.id); - guest.putAutomation(automation({ id: "job_2", name: "sooner", next_run_at_ms: at - 60 * MIN })); - await waitFor(async () => (await db.computers.get(dot.id))?.next_automation_at === new Date(at - 60 * MIN).toISOString(), "the earlier one"); - guest.removeAutomation("job_2"); - guest.removeAutomation("job_1"); - await waitFor(async () => (await db.computers.get(dot.id))?.next_automation_at === null, "cleared"); - }); - - it("does not put a Dot to sleep when a run is due within the wake lead time, and does when it is not", async () => { - const { scheduler, clock } = makeWithLead(); - const soon = await readyDot(scheduler, "run-soon", "10m"); - const far = await readyDot(scheduler, "run-far", "10m"); - const none = await readyDot(scheduler, "run-none", "10m"); - await reportDueIn(scheduler, clock, soon.id, 10 * MIN + LEAD_MS - 1_000); - await reportDueIn(scheduler, clock, far.id, 10 * MIN + LEAD_MS + 5 * MIN); - - // Idle for the whole timeout: the one whose run is within the lead stays up. - clock.advance(10 * MIN + 1_000); - expect((await scheduler.idleCheck()).sort()).toEqual([far.id, none.id].sort()); - await scheduler.settle(); - expect((await db.computers.get(soon.id))?.state).toBe("RUNNING"); - expect((await db.computers.get(far.id))?.state).toBe("STOPPED"); - }); - - it("does not put a Dot to sleep that has a run already due, and does once the engine has moved it far off", async () => { - const { scheduler, driver, clock } = makeWithLead(); - const dot = await readyDot(scheduler, "run-due", "10m"); - await reportDueIn(scheduler, clock, dot.id, 2 * MIN); - - clock.advance(20 * MIN); - expect(await scheduler.idleCheck()).toEqual([]); - - // The engine made the run and tells the host the next one is hours away. - const guest = driver.guestOf(dot.id); - const next = clock.now().getTime() + 6 * 60 * MIN; - guest.putAutomation(automation({ id: "job_next", next_run_at_ms: next })); - guest.removeAutomation("job_1"); - await waitFor(async () => (await db.computers.get(dot.id))?.next_automation_at === new Date(next).toISOString(), "the next run stored"); - // The reports were activity: the idle timeout counts from them. - clock.advance(11 * MIN); - expect(await scheduler.idleCheck()).toEqual([dot.id]); - }); - - /** The Dot sleeps as the idle check puts it to sleep: nothing due within the lead, idle for its whole timeout. */ - async function sleepIdle(scheduler: Scheduler, clock: ManualClock, dotId: string, idleMs = 11 * MIN) { - clock.advance(idleMs); - expect(await scheduler.idleCheck()).toEqual([dotId]); - await scheduler.settle(); - expect(await db.computers.get(dotId)).toMatchObject({ state: "STOPPED", stop_reason: "idle" }); - } - - it("starts a stopped Dot shortly before its automation is due, and not before", async () => { - const { scheduler, driver, clock } = makeWithLead(); - const dot = await readyDot(scheduler, "sleeping-job", "10m"); - const at = await reportDueIn(scheduler, clock, dot.id, 60 * MIN); - await sleepIdle(scheduler, clock, dot.id); - const boots = driver.guestOf(dot.id).boots; - - await scheduler.pass(); - clock.advance(28 * MIN); - await scheduler.pass(); - await scheduler.settle(); - expect((await db.computers.get(dot.id))?.state).toBe("STOPPED"); - expect(driver.guestOf(dot.id).boots).toBe(boots); - - // 30 s before the run, within the 60 s lead. - clock.advance(at - clock.now().getTime() - 30_000); - await scheduler.pass(); - await waitFor(async () => (await db.computers.get(dot.id))?.state === "RUNNING", "started for the automation"); - await waitFor(async () => (await db.dots.get(dot.id))?.status === "READY", "READY"); - expect(driver.guestOf(dot.id).boots).toBe(boots + 1); - // The key lives in guest memory only: the start pushed it as every start does. - expect(driver.guestOf(dot.id).openrouterKey).toBe("sk-or-test"); - }); - - it("wakes a Dot for a missed run once: the guest makes it, the stored time moves on, and the Dot sleeps again", async () => { - const { scheduler, driver, clock } = makeWithLead(); - const dot = await readyDot(scheduler, "missed-job", "10m"); - const guest = driver.guestOf(dot.id); - guest.now = () => clock.now().getTime(); - const at = clock.now().getTime() + 60 * MIN; - guest.putAutomation(automation({ schedule: { kind: "every", every_ms: 6 * 60 * MIN }, next_run_at_ms: at })); - await waitFor(async () => (await db.computers.get(dot.id))?.next_automation_at === new Date(at).toISOString(), "the report stored"); - await sleepIdle(scheduler, clock, dot.id); - clock.advance(3 * 60 * MIN); - const boots = guest.boots; - - await scheduler.pass(); - await waitFor(async () => (await db.computers.get(dot.id))?.state === "RUNNING", "started for the missed run"); - await scheduler.settle(); - // The guest made the run at its boot and told the host when the next one is: the past time is gone. - const ranAt = clock.now().getTime(); - const next = ranAt + 6 * 60 * MIN; - expect(guest.automations.get("job_1")).toMatchObject({ last_run_at_ms: ranAt, next_run_at_ms: next }); - await waitFor(async () => (await db.computers.get(dot.id))?.next_automation_at === new Date(next).toISOString(), "the next run stored"); - expect(await scheduler.lifecycle.keepsAwake(dot.id)).toBe(false); - - // So the passes that follow find nothing to start, and the idle check puts the Dot to sleep again. - await scheduler.pass(); - await scheduler.settle(); - expect(guest.boots).toBe(boots + 1); - await sleepIdle(scheduler, clock, dot.id); - await scheduler.pass(); - await scheduler.settle(); - expect(guest.boots).toBe(boots + 1); - expect(guest.automations.get("job_1")?.last_run_at_ms).toBe(ranAt); - }); - - it("does not start for an automation when the person's stop came in front of the wake the pass had chosen, and still starts one nothing came in front of", async () => { - const { scheduler, driver, clock } = makeWithLead(); - const stopped = await readyDot(scheduler, "stop-comes-first", "10m"); - const free = await readyDot(scheduler, "nothing-comes-first", "10m"); - for (const dot of [stopped, free]) { - const guest = driver.guestOf(dot.id); - guest.now = () => clock.now().getTime(); - guest.putAutomation(automation({ schedule: { kind: "every", every_ms: 60 * MIN }, next_run_at_ms: clock.now().getTime() + 20 * MIN })); - await waitFor(async () => (await db.computers.get(dot.id))?.next_automation_at !== null, "the report stored"); - } - clock.advance(11 * MIN); - expect((await scheduler.idleCheck()).sort()).toEqual([stopped.id, free.id].sort()); - await scheduler.settle(); - clock.advance(9 * MIN); - // The pass would wake both: it read the database, and the wakes wait for their turn of the per-Dot lock. - expect((await scheduler.lifecycle.stoppedDotsWithAutomationDue()).sort()).toEqual([stopped.id, free.id].sort()); - const bootsOf = (id: string) => driver.guestOf(id).boots; - const before = { stopped: bootsOf(stopped.id), free: bootsOf(free.id) }; - - // The person's stop gets there first. - await scheduler.stopComputer(stopped.id); - await scheduler.settle(); - expect(await db.computers.get(stopped.id)).toMatchObject({ state: "STOPPED", stop_reason: "user" }); - expect(await scheduler.lifecycle.wakeForAutomation(stopped.id)).toBe(false); - expect(await scheduler.lifecycle.wakeForAutomation(free.id)).toBe(true); - await scheduler.settle(); - expect(bootsOf(stopped.id)).toBe(before.stopped); - expect(bootsOf(free.id)).toBe(before.free + 1); - expect((await db.computers.get(stopped.id))?.state).toBe("STOPPED"); - }); - - it("keeps a Dot the person stopped off, whatever its automations say, until the person starts it again", async () => { - const { scheduler, driver, clock } = makeWithLead(); - const dot = await readyDot(scheduler, "stopped-by-me", "10m"); - const guest = driver.guestOf(dot.id); - guest.now = () => clock.now().getTime(); - const at = clock.now().getTime() + 5 * MIN; - guest.putAutomation(automation({ schedule: { kind: "every", every_ms: 60 * MIN }, next_run_at_ms: at })); - await waitFor(async () => (await db.computers.get(dot.id))?.next_automation_at === new Date(at).toISOString(), "the report stored"); - - await scheduler.stopComputer(dot.id); - await scheduler.settle(); - expect(await db.computers.get(dot.id)).toMatchObject({ state: "STOPPED", stop_reason: "user" }); - const boots = guest.boots; - - // The run is past, then an hour of runs are: the Dot stays as the person left it. - clock.advance(10 * MIN); - await scheduler.pass(); - clock.advance(60 * MIN); - await scheduler.pass(); - await scheduler.settle(); - expect(guest.boots).toBe(boots); - expect((await db.computers.get(dot.id))?.state).toBe("STOPPED"); - - // The person starts it: the guest makes the run it missed, and the reason is gone. - await scheduler.startComputer(dot.id); - await waitFor(async () => (await db.dots.get(dot.id))?.status === "READY", "READY"); - expect((await db.computers.get(dot.id))?.stop_reason).toBeNull(); - const ranAt = clock.now().getTime(); - expect(guest.automations.get("job_1")?.last_run_at_ms).toBe(ranAt); - - // From then on it sleeps and wakes for its automations like any other. The guest's report of the next run is - // activity: the idle timeout counts from it, so it is stored before the clock moves. - await waitFor(async () => (await db.computers.get(dot.id))?.next_automation_at === new Date(ranAt + 60 * MIN).toISOString(), "the next run stored"); - await scheduler.settle(); - clock.advance(11 * MIN); - expect(await scheduler.idleCheck()).toEqual([dot.id]); - await scheduler.settle(); - expect((await db.computers.get(dot.id))?.stop_reason).toBe("idle"); - clock.advance(60 * MIN); - await scheduler.pass(); - await waitFor(async () => (await db.computers.get(dot.id))?.state === "RUNNING", "woken for the next run"); - expect(guest.boots).toBe(boots + 2); - }); - - it("records the person's stop of a computer that is already asleep, and a message still starts it", async () => { - const { scheduler, driver, clock } = makeWithLead(); - const dot = await readyDot(scheduler, "asleep-then-stopped", "10m"); - await reportDueIn(scheduler, clock, dot.id, 60 * MIN); - await sleepIdle(scheduler, clock, dot.id); - const boots = driver.guestOf(dot.id).boots; - - await scheduler.stopComputer(dot.id); - await scheduler.settle(); - expect((await db.computers.get(dot.id))?.stop_reason).toBe("user"); - clock.advance(2 * 60 * MIN); - await scheduler.pass(); - await scheduler.settle(); - expect(driver.guestOf(dot.id).boots).toBe(boots); - - // The person's stop is about the automations: their own message still starts the computer. - await scheduler.sendMessage(dot.id, "are you there"); - await scheduler.settle(); - expect(driver.guestOf(dot.id).boots).toBe(boots + 1); - expect((await db.computers.get(dot.id))?.stop_reason).toBeNull(); - }); - - it("starts a VM that stopped by itself at once when a run is due, by the same rule as the idle sleep", async () => { - const { scheduler, driver, clock } = makeWithLead(); - const dot = await readyDot(scheduler, "self-stopper-job", "10m"); - await reportDueIn(scheduler, clock, dot.id, LEAD_MS - 10_000); - const guest = driver.guestOf(dot.id); - const boots = guest.boots; - - // No pass runs here: it is the exit itself that starts the computer again, not the next round of the scheduler. - driver.crash(dot.id); - await waitFor(async () => (await db.events.list({ dotId: dot.id, types: ["computer.stopped"] })).some((e) => e.data.reason === "exited"), "stop recorded"); - await waitFor(() => guest.boots === boots + 1 && scheduler.lifecycle.isReady(dot.id), "started again"); - }); - - it("leaves a VM that stopped by itself off when nothing is due, and records that it exited", async () => { - const { scheduler, driver, clock } = makeWithLead(); - const dot = await readyDot(scheduler, "self-stopper-idle", "10m"); - await reportDueIn(scheduler, clock, dot.id, 5 * 60 * MIN); - const boots = driver.guestOf(dot.id).boots; - - driver.crash(dot.id); - await waitFor(async () => (await db.computers.get(dot.id))?.state === "STOPPED", "stop recorded"); - await scheduler.settle(); - expect(await db.computers.get(dot.id)).toMatchObject({ state: "STOPPED", stop_reason: "exited" }); - expect(driver.guestOf(dot.id).boots).toBe(boots); - }); - - it("leaves a stopped Dot alone that has no automation to run, or whose Dot needs the person", async () => { - const { scheduler, driver, clock } = makeWithLead(); - const nothing = await readyDot(scheduler, "stopped-nothing", "10m"); - const broken = await readyDot(scheduler, "stopped-broken", "10m"); - await reportDueIn(scheduler, clock, broken.id, 30 * MIN); - // Both slept by themselves (nothing due within the lead): it is the Dot's status that keeps the second one off. - clock.advance(11 * MIN); - expect((await scheduler.idleCheck()).sort()).toEqual([nothing.id, broken.id].sort()); - await scheduler.settle(); - await db.dots.setStatus(broken.id, "ERROR", "the person has to look"); - const boots = [driver.guestOf(nothing.id).boots, driver.guestOf(broken.id).boots]; - - clock.advance(60 * MIN); - await scheduler.pass(); - await scheduler.settle(); - - expect([driver.guestOf(nothing.id).boots, driver.guestOf(broken.id).boots]).toEqual(boots); - expect((await db.computers.get(nothing.id))?.state).toBe("STOPPED"); - expect((await db.computers.get(broken.id))?.state).toBe("STOPPED"); - }); - - it("calls an idle sleep off under the Dot's lock when the run came within the lead meanwhile", async () => { - const { scheduler, clock } = makeWithLead(); - const dot = await readyDot(scheduler, "late-report", "10m"); - clock.advance(11 * MIN); - // The idle check saw nothing due; then the engine reported a run within the lead, before the stop took the lock. - await reportDueIn(scheduler, clock, dot.id, LEAD_MS - 5_000); - - await scheduler.lifecycle.stop(dot.id, "idle"); - - expect((await db.computers.get(dot.id))?.state).toBe("RUNNING"); - expect(scheduler.lifecycle.isReady(dot.id)).toBe(true); - }); - }); - - it("a message to a stopped Dot is queued, wakes it and is delivered", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "chatty"); - await scheduler.lifecycle.stop(dot.id, "user"); - const answer = await scheduler.sendMessage("chatty", "hello there"); - expect(answer.delivery).toBe("queued"); - await scheduler.settle(); - const guest = driver.guestOf(dot.id); - expect(guest.inbound.at(-1)).toMatchObject({ id: answer.message_id, type: "user.message", data: { text: "hello there" } }); - await waitFor(async () => (await scheduler.conversation(dot.id)).length === 2, "assistant reply"); - const conversation = await scheduler.conversation(dot.id); - expect(conversation.map((m) => [m.role, m.text])).toEqual([ - ["user", "hello there"], - ["assistant", "echo: hello there"], - ]); - const direct = await scheduler.sendMessage(dot.id, "again"); - expect(direct.delivery).toBe("delivered"); - }); - - it("keeps where a message came from in the log and answers it in the conversation, never to the guest", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "bridged"); - const origin = { channel: "telegram", binding_id: "chb_1", chat_id: "4242", external_id: "77" } as const; - const fromChannel = await scheduler.sendMessage(dot.id, "from my phone", origin); - const fromApi = await scheduler.sendMessage(dot.id, "from the web"); - await waitFor(async () => (await scheduler.conversation(dot.id)).length === 4, "both replies"); - // The event log owns the fact. - const logged = (await db.events.list({ dotId: dot.id, types: ["user.message"] })).map((e) => e.data); - expect(logged).toEqual([ - { message_id: fromChannel.message_id, text: "from my phone", origin }, - { message_id: fromApi.message_id, text: "from the web" }, - ]); - const conversation = await scheduler.conversation(dot.id); - expect(conversation.filter((m) => m.role === "user").map((m) => m.origin)).toEqual([origin, undefined]); - expect(conversation.some((m) => m.role === "assistant" && "origin" in m)).toBe(false); - // The guest still gets {text}: nothing in the Dot knows about a channel. - const received = driver.guestOf(dot.id).inbound.filter((e) => e.type === "user.message"); - expect(received.map((e) => e.data)).toEqual([{ text: "from my phone" }, { text: "from the web" }]); - }); - - it("hands a channel message to the Dot once however often it is offered, and answers a repeat with the first", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "repeated"); - const origin = { channel: "telegram", binding_id: "chb_1", chat_id: "4242", external_id: "77" } as const; - const first = await scheduler.sendMessage(dot.id, "from my phone", origin); - const again = await scheduler.sendMessage(dot.id, "from my phone", origin); - expect(again).toEqual({ message_id: first.message_id, event_id: first.event_id, delivery: "delivered" }); - // The same id on another binding is another message. - const other = await scheduler.sendMessage(dot.id, "from my phone", { ...origin, binding_id: "chb_2" }); - expect(other.message_id).not.toBe(first.message_id); - await scheduler.settle(); - expect((await db.events.list({ dotId: dot.id, types: ["user.message"] })).map((e) => e.data.message_id)).toEqual([first.message_id, other.message_id]); - expect(driver.guestOf(dot.id).inbound.filter((e) => e.type === "user.message").map((e) => e.id)).toEqual([first.message_id, other.message_id]); - expect((await db.inbound.get(first.message_id))?.delivered_at).toBeTruthy(); - }); - - it("refuses a message whose origin is not a channel chat, and logs nothing for it", async () => { - const { scheduler } = make(); - const dot = await readyDot(scheduler, "strict"); - const before = (await db.events.list({ dotId: dot.id })).length; - const bad = { channel: "sms", binding_id: "chb_1", chat_id: "1", external_id: "2" }; - await expect(scheduler.sendMessage(dot.id, "hi", bad as never)).rejects.toMatchObject({ status: 400, code: "invalid_request" }); - await expect(scheduler.sendMessage(dot.id, "hi", { ...bad, channel: "telegram", extra: 1 } as never)).rejects.toMatchObject({ status: 400 }); - expect((await db.events.list({ dotId: dot.id })).length).toBe(before); - }); - - it("leaves out an origin it cannot read in a stored message instead of failing the conversation", async () => { - const { scheduler } = make(); - const dot = await readyDot(scheduler, "old-rows"); - await scheduler.events.appendUserMessage(dot.id, { message_id: "msg_old", text: "odd", origin: { channel: "sms" } as never }); - const [message] = await scheduler.conversation(dot.id); - expect(message).toMatchObject({ role: "user", text: "odd" }); - expect("origin" in message!).toBe(false); - }); - - it("READY fails without an OpenRouter key and succeeds once one is set", async () => { - await db.secrets.delete("global", "openrouter_api_key"); - try { - const { scheduler } = make(); - const dot = await scheduler.createDot(yaml("keyless")); - await waitFor(async () => (await db.dots.get(dot.id))?.status === "ERROR", "ERROR"); - expect((await db.dots.get(dot.id))?.error).toMatch(/no OpenRouter API key/); - expect((await db.computers.get(dot.id))?.state).toBe("ERROR"); - await scheduler.setOpenRouterKey("sk-or-new", dot.id); - await waitFor(async () => (await db.dots.get(dot.id))?.status === "READY", "READY after key"); - } finally { - await db.secrets.put("global", "openrouter_api_key", "sk-or-test"); - } - }); - - it("READY fails when a guest self-check fails, naming it", async () => { - const driver = new FakeDriver(); - driver.configureGuest = (g) => { - g.checks = { ...g.checks, browser_installed: false }; - }; - const { scheduler } = make(driver); - const dot = await scheduler.createDot(yaml("no-browser")); - await waitFor(async () => (await db.dots.get(dot.id))?.status === "ERROR", "ERROR"); - expect((await db.dots.get(dot.id))?.error).toMatch(/browser layer not installed/); - }); - - it("every start records a new guest port and pid; a stop clears them", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "port-mover"); - const before = await db.computers.get(dot.id); - expect(before?.guest_port).toEqual(expect.any(Number)); - await scheduler.lifecycle.stop(dot.id, "user"); - expect(await db.computers.get(dot.id)).toMatchObject({ state: "STOPPED", guest_port: null, pid: null }); - await expect(scheduler.lifecycle.guest(dot.id)).rejects.toMatchObject({ status: 409, code: "computer_stopped" }); - - await scheduler.lifecycle.ensureReady(dot.id); - const after = await db.computers.get(dot.id); - expect(after?.guest_port).not.toBe(before?.guest_port); - expect(after?.pid).toBe(driver.vms.get(dot.id)?.pid); - const started = await db.events.list({ dotId: dot.id, types: ["computer.started"] }); - expect(started.map((e) => e.data.guest_port)).toEqual([before?.guest_port, after?.guest_port]); - // The old port reaches nothing any more: only the recorded one leads to the guest. - const token = await db.computers.token(dot.id); - await expect(driver.guest({ dotId: dot.id, port: before!.guest_port! }, token).health()).rejects.toMatchObject({ status: 0 }); - }); - - it("a failed delivery is retried, then the task fails with the reason", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "flaky"); - const guest = driver.guestOf(dot.id); - guest.failNextPost = new FakeGuestError(0, "connection reset"); - const ok = await scheduler.createTask(dot.id, { description: "eventually" }); - await waitFor(async () => (await db.tasks.get(ok.id))?.status === "COMPLETED", "retried task"); - // The same event went again: one run, never back to PENDING in between. - expect((await db.tasks.runs(ok.id)).map((r) => r.outcome)).toEqual(["completed"]); - - driver.failNext("start", new Error("qemu-system-x86_64 exited: boom"), 2); - await scheduler.lifecycle.stop(dot.id, "user"); - const doomed = await scheduler.createTask(dot.id, { description: "never" }); - await waitFor(async () => (await db.tasks.get(doomed.id))?.status === "FAILED", "failed task", 8_000); - expect((await db.tasks.get(doomed.id))?.error).toMatch(/after 2 attempts/); - // The host ended the task, so the host logs its end: every view that follows task.failed sees it. - const failed = await db.events.list({ dotId: dot.id, types: ["task.failed"], taskId: doomed.id }); - expect(failed).toHaveLength(1); - expect(failed[0]).toMatchObject({ source: "host", data: { task_id: doomed.id, error: expect.stringMatching(/after 2 attempts/) } }); - }); - - it("cancel: a pending task never runs, a finished one answers 409", async () => { - const { scheduler } = make(); - const dot = await readyDot(scheduler, "canceller"); - const later = await scheduler.createTask(dot.id, { - description: "tomorrow", - scheduled_at: new Date(Date.now() + 86_400_000).toISOString(), - }); - const cancelled = await scheduler.cancelTask(later.id); - expect(cancelled.status).toBe("CANCELLED"); - await expect(scheduler.cancelTask(later.id)).rejects.toMatchObject({ status: 409, code: "task_finished" }); - expect(types(await db.events.list({ dotId: dot.id, types: ["task.cancelled"] }))).toEqual(["task.cancelled"]); - }); - - it("identities need a running computer: 409 computer_stopped otherwise, guest 404 passes through", async () => { - const { scheduler } = make(); - const dot = await readyDot(scheduler, "browser-owner"); - const created = await scheduler.createIdentity(dot.id, { name: "Shop Account" }); - expect(created.id).toMatch(/^shop-account-[a-z0-9]{6}$/); - expect((await scheduler.listIdentities(dot.id)).map((i) => i.id)).toEqual([created.id]); - await expect(scheduler.getIdentity(dot.id, "missing-abc123")).rejects.toMatchObject({ status: 404, code: "not_found" }); - await scheduler.deleteIdentity(dot.id, created.id); - await scheduler.lifecycle.stop(dot.id, "user"); - await expect(scheduler.listIdentities(dot.id)).rejects.toMatchObject({ status: 409, code: "computer_stopped" }); - await expect(scheduler.screenshot(dot.id)).rejects.toMatchObject({ status: 409, code: "computer_stopped" }); - }); - - it("an identity is created with no proxy unless one is given: absent, null and blank are the same, a number is refused", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "no-proxy"); - const guest = driver.guestOf(dot.id); - const requests: unknown[] = []; - const create = guest.createBrowserIdentity.bind(guest); - guest.createBrowserIdentity = async (body) => { - requests.push(body); - return create(body); - }; - - for (const body of [{ name: "A" }, { name: "B", proxy: null }, { name: "C", proxy: "" }] as { name: string; proxy?: string }[]) { - expect(await scheduler.createIdentity(dot.id, body)).toMatchObject({ hasProxy: false }); - } - expect(requests).toEqual([{ name: "A" }, { name: "B" }, { name: "C" }]); - - const own = await scheduler.createIdentity(dot.id, { name: "D", proxy: "http://proxy.test:8080" }); - expect(own.hasProxy).toBe(true); - expect(requests.at(-1)).toEqual({ name: "D", proxy: "http://proxy.test:8080" }); - await expect(scheduler.createIdentity(dot.id, { name: "E", proxy: 8080 as never })).rejects.toMatchObject({ status: 400, code: "invalid_request" }); - }); - - it("an identity's frame and close go to the running guest: not_open, busy and not_found pass through", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "watcher"); - const guest = driver.guestOf(dot.id); - const identity = await scheduler.createIdentity(dot.id, { name: "Shop" }); - - await expect(scheduler.identityFrame(dot.id, identity.id)).rejects.toMatchObject({ status: 409, code: "not_open" }); - await expect(scheduler.identityFrame(dot.id, "missing-abc123")).rejects.toMatchObject({ status: 404, code: "not_found" }); - guest.launchIdentity(identity.id); - expect([...(await scheduler.identityFrame(dot.id, identity.id)).slice(0, 2)]).toEqual([0xff, 0xd8]); - guest.identityBusy = true; - await expect(scheduler.identityFrame(dot.id, identity.id)).rejects.toMatchObject({ status: 503, code: "busy" }); - guest.identityBusy = false; - - await scheduler.closeIdentity(dot.id, identity.id); - expect((await scheduler.getIdentity(dot.id, identity.id)).status).toBe("available"); - await expect(scheduler.identityFrame(dot.id, identity.id)).rejects.toMatchObject({ status: 409, code: "not_open" }); - // Closing a closed identity is not an error; an unknown one is a 404. - await scheduler.closeIdentity(dot.id, identity.id); - await expect(scheduler.closeIdentity(dot.id, "missing-abc123")).rejects.toMatchObject({ status: 404, code: "not_found" }); - - await scheduler.lifecycle.stop(dot.id, "user"); - await expect(scheduler.identityFrame(dot.id, identity.id)).rejects.toMatchObject({ status: 409, code: "computer_stopped" }); - await expect(scheduler.closeIdentity(dot.id, identity.id)).rejects.toMatchObject({ status: 409, code: "computer_stopped" }); - }); - - it("every coded answer of the engine's identity routes that is not a 4xx passes through, so the UI can tell a crashed browser from a silent computer", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "failing"); - const guest = driver.guestOf(dot.id); - const identity = await scheduler.createIdentity(dot.id, { name: "Shop" }); - guest.launchIdentity(identity.id); - - const browserAnswers = Object.entries(IDENTITY_ERROR_STATUS).filter(([, status]) => status >= 500); - expect(browserAnswers.map(([code]) => code).sort()).toEqual(["busy", "crashed", "frame_failed"]); - for (const [code, status] of browserAnswers) { - guest.identityFault = code as IdentityErrorCode; - await expect(scheduler.identityFrame(dot.id, identity.id)).rejects.toMatchObject({ status, code }); - } - guest.identityFault = null; - expect([...(await scheduler.identityFrame(dot.id, identity.id)).slice(0, 2)]).toEqual([0xff, 0xd8]); - }); - - it("a guest that does not answer a frame is a 502, not a pass-through", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "silent"); - const identity = await scheduler.createIdentity(dot.id, { name: "Shop" }); - driver.guestOf(dot.id).launchIdentity(identity.id); - driver.guestOf(dot.id).powerOff(); - - await expect(scheduler.identityFrame(dot.id, identity.id)).rejects.toMatchObject({ status: 502, code: "guest_unavailable" }); - }); - - it("PATCH pushes the new config to a READY guest and refuses a shrinking disk", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "patchy"); - const updated = await scheduler.updateDot("patchy", `${yaml("patchy")}instructions: be brief\n`); - expect(updated.config.instructions).toBe("be brief"); - expect(driver.guestOf(dot.id).config?.instructions).toBe("be brief"); - await expect( - scheduler.updateDot(dot.id, yaml("patchy").replace("computer:\n", "computer:\n disk: 20gb\n")), - ).rejects.toMatchObject({ status: 409, code: "disk_shrink" }); - await expect(scheduler.updateDot(dot.id, "name: Bad Name")).rejects.toMatchObject({ status: 400, code: "invalid_config" }); - }); - - it("PATCH made from an old read is a 409 dot_changed: it cannot undo an always-allow answered since, and nothing is saved", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "stale-form"); - const guest = driver.guestOf(dot.id); - // The approval is asked (the Dot's status moves to WAITING_APPROVAL) BEFORE the form reads the Dot: what makes the - // save stale below is the always-allow alone. - const approvalId = guest.requestApproval(undefined); - await waitFor(async () => (await db.approvals.get(approvalId)) !== null && (await db.dots.get(dot.id))?.status === "WAITING_APPROVAL", "stored"); - const loaded = await scheduler.requireDot(dot.id); - expect(loaded.config.permissions?.["browser.identity.delete"]).toBeUndefined(); - - // Another view answers "Always allow" while the form is open. - await scheduler.resolveApproval(approvalId, "approve", { always: true }); - expect((await db.dots.get(dot.id))?.config.permissions["browser.identity.delete"]).toBe("allow"); - - const save = `${yaml("stale-form")}instructions: be brief\n`; - await expect(scheduler.updateDot(dot.id, save, loaded.config_version)).rejects.toMatchObject({ status: 409, code: "dot_changed" }); - const stored = (await db.dots.get(dot.id))!; - expect(stored.config.permissions["browser.identity.delete"]).toBe("allow"); - expect(stored.config.instructions).toBeUndefined(); - expect(guest.config?.instructions).toBeUndefined(); - - // Read again, the save goes through, once; a save with no precondition is as it always was; a malformed one is a 400. - const fresh = await scheduler.requireDot(dot.id); - expect((await scheduler.updateDot(dot.id, save, fresh.config_version)).config.instructions).toBe("be brief"); - await expect(scheduler.updateDot(dot.id, save, fresh.config_version)).rejects.toMatchObject({ status: 409, code: "dot_changed" }); - expect((await scheduler.updateDot(dot.id, `${yaml("stale-form")}instructions: no precondition\n`)).config.instructions).toBe("no precondition"); - await expect(scheduler.updateDot(dot.id, save, "2")).rejects.toMatchObject({ status: 400, code: "invalid_request" }); - await expect(scheduler.updateDot(dot.id, save, 0)).rejects.toMatchObject({ status: 400, code: "invalid_request" }); - await expect(scheduler.updateDot("missing-dot", save, fresh.config_version)).rejects.toMatchObject({ status: 404 }); - }); - - it("PATCH made from a read before a status change is saved: only a change of the config makes it a 409", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "status-moves"); - const loaded = await scheduler.requireDot(dot.id); - // A turn of the Dot moves its status (and updated_at) while the form is open; its config is as it was read. - driver.guestOf(dot.id).emit("agent.state", { state: "EXECUTING" }); - await waitFor(async () => (await db.dots.get(dot.id))?.status === "RUNNING", "the status to move"); - const moved = (await db.dots.get(dot.id))!; - expect(Date.parse(moved.updated_at)).toBeGreaterThan(Date.parse(loaded.updated_at)); - expect(moved.config_version).toBe(loaded.config_version); - - const saved = await scheduler.updateDot(dot.id, `${yaml("status-moves")}instructions: after a turn\n`, loaded.config_version); - expect(saved.config.instructions).toBe("after a turn"); - expect(saved.config_version).toBe(loaded.config_version + 1); - }); - - it("reboot waits for the new boot and pushes the key again", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "rebooter"); - const guest = driver.guestOf(dot.id); - await scheduler.lifecycle.reboot(dot.id); - expect(guest.boots).toBe(2); - expect(guest.openrouterKey).toBe("sk-or-test"); - expect(scheduler.lifecycle.isReady(dot.id)).toBe(true); - }); - - it("recovery: a VM whose disk is gone is ERROR; a running VM's port and pid come from QEMU, not the row", async () => { - const driver = new FakeDriver(); - const first = make(driver).scheduler; - const lost = await readyDot(first, "disk-lost"); - const unrecorded = await readyDot(first, "port-lost"); - await first.close(); - open.splice(open.indexOf(first), 1); - - driver.vms.delete(lost.id); - // As if the control plane stopped between spawning QEMU and recording its port. - await db.computers.setProcess(unrecorded.id, null); - - const { scheduler } = make(driver, { dispatchIntervalMs: 60_000, idleCheckIntervalMs: 60_000 }); - await scheduler.start(); - await scheduler.settle(); - expect(await db.dots.get(lost.id)).toMatchObject({ status: "ERROR", error: "the VM disk is missing" }); - expect((await db.computers.get(lost.id))?.state).toBe("ERROR"); - expect(scheduler.lifecycle.isReady(unrecorded.id)).toBe(true); - expect(await db.computers.get(unrecorded.id)).toMatchObject({ - guest_port: driver.vms.get(unrecorded.id)?.guestPort, - pid: driver.vms.get(unrecorded.id)?.pid, - }); - // Adopted, not restarted. - expect(driver.guestOf(unrecorded.id).boots).toBe(1); - }); - - it("delete destroys the VM and removes the rows but keeps the history", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "goner"); - await scheduler.deleteDot("goner"); - await scheduler.settle(); - expect(driver.vms.has(dot.id)).toBe(false); - expect(await db.dots.get(dot.id)).toBeNull(); - const history = await scheduler.listEvents(dot.id); - expect(history.at(-1)?.type).toBe("dot.deleted"); - await expect(scheduler.requireDot("goner")).rejects.toMatchObject({ status: 404 }); - }); - - it("events: filters by type and task, refuses a type no event has, and filters a deleted Dot's history by id", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "filtered"); - const task = await scheduler.createTask(dot.id, { description: "look" }); - await waitFor(async () => (await db.tasks.get(task.id))?.status === "COMPLETED", "task COMPLETED"); - await scheduler.sendMessage(dot.id, "hello"); - driver.guestOf(dot.id).emit("browser.identity.created", { identity_id: "bi_a", name: "a" }); - await waitFor(async () => (await scheduler.listEvents(dot.id, { types: ["browser.identity.created"] })).length === 1, "identity stored"); - - expect(types(await scheduler.listEvents(dot.id, { types: ["user.message", "browser.identity.created"] }))).toEqual(["user.message", "browser.identity.created"]); - expect(types(await scheduler.listEvents(dot.id, { taskId: task.id }))).toEqual( - expect.arrayContaining(["task.created", "task.started", "task.completed"]), - ); - expect((await scheduler.listEvents(dot.id, { taskId: task.id, types: ["task.completed"] })).map((e) => e.data.task_id)).toEqual([task.id]); - // An empty list is no filter; a name that is no event type is an error rather than a quiet Dot. - expect((await scheduler.listEvents(dot.id, { types: [] })).length).toBeGreaterThan(5); - await expect(scheduler.listEvents(dot.id, { types: ["task.completed", "task.done"] })).rejects.toMatchObject({ status: 400, code: "invalid_request" }); - // The Dot keeps its notes itself: a note written is no event type. - await expect(scheduler.listEvents(dot.id, { types: ["memory.written"] })).rejects.toMatchObject({ status: 400, code: "invalid_request" }); - await expect(scheduler.listEvents(dot.id, { taskId: "" })).rejects.toMatchObject({ status: 400 }); - - await scheduler.deleteDot("filtered"); - await scheduler.settle(); - expect(types(await scheduler.listEvents(dot.id, { taskId: task.id, types: ["task.completed"] }))).toEqual(["task.completed"]); - }); - - it("files: the path rule comes before the guest, the guest's refusals pass through, and a stopped computer is 409", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "filer"); - const guest = driver.guestOf(dot.id); - guest.putFile("/home/dot/memory/a.md", "alpha"); - const calls = guest.calls.length; - - expect(await scheduler.listFiles(dot.id)).toMatchObject({ path: "/home/dot", entries: [{ name: "memory", type: "dir" }] }); - expect(await scheduler.listFiles("filer", "memory")).toMatchObject({ path: "/home/dot/memory", entries: [{ name: "a.md", size: 5 }] }); - const read = await scheduler.readFile(dot.id, "~/memory/a.md"); - expect(read.path).toBe("/home/dot/memory/a.md"); - expect(new TextDecoder().decode(read.content)).toBe("alpha"); - expect(guest.calls.slice(calls)).toEqual(["listFiles", "listFiles", "readFile"]); - - const before = guest.calls.length; - await expect(scheduler.readFile(dot.id, "/etc/passwd")).rejects.toMatchObject({ status: 400, code: "invalid_path" }); - await expect(scheduler.listFiles(dot.id, "memory/../..")).rejects.toMatchObject({ status: 400, code: "invalid_path" }); - await expect(scheduler.readFile(dot.id, undefined)).rejects.toMatchObject({ status: 400, code: "invalid_path" }); - expect(guest.calls.length).toBe(before); - - await expect(scheduler.readFile(dot.id, "memory/none.md")).rejects.toMatchObject({ status: 404, code: "not_found" }); - guest.putFile("/home/dot/big", new Uint8Array(16 * 1024 * 1024 + 1)); - await expect(scheduler.readFile(dot.id, "big")).rejects.toMatchObject({ status: 413, code: "file_too_large" }); - // A guest that cannot be reached is the Dot's computer not answering, not the caller's fault. - guest.powerOff(); - await expect(scheduler.readFile(dot.id, "memory/a.md")).rejects.toMatchObject({ status: 502, code: "guest_unavailable" }); - }); - - it("files: a symbolic link under home that leads out of it is the guest's 403 outside_home, and one that stays is read", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "linker"); - const guest = driver.guestOf(dot.id); - guest.putFile("/home/dot/documents/note.txt", "kept"); - guest.putFile("/etc/invisible-dots/config.json", "token"); - // The lexical rule lets all of these through (they name a path under home); the guest's real-path rule is what refuses. - guest.link("/home/dot/environ", "/proc/4242/environ"); - guest.link("/home/dot/config", "/etc/invisible-dots/config.json"); - guest.link("/home/dot/etc", "/etc"); - guest.link("/home/dot/docs", "/home/dot/documents"); - - for (const path of ["environ", "config", "etc/invisible-dots/config.json", "/home/dot/environ"]) { - await expect(scheduler.readFile(dot.id, path), path).rejects.toMatchObject({ status: 403, code: "outside_home" }); - } - await expect(scheduler.listFiles(dot.id, "etc")).rejects.toMatchObject({ status: 403, code: "outside_home" }); - expect(new TextDecoder().decode((await scheduler.readFile(dot.id, "docs/note.txt")).content)).toBe("kept"); - expect(await scheduler.listFiles(dot.id, "docs")).toMatchObject({ entries: [{ name: "note.txt", size: 4 }] }); - }); - - describe("MCP servers", () => { - const withServers = (name: string, servers: string, permissions = "") => - `${yaml(name)}${permissions ? `permissions:\n${permissions}` : ""}mcp_servers:\n${servers}`; - const TIME = " time:\n command: uvx\n args: [mcp-server-time]\n secrets: [TIME_TOKEN]\n"; - const WEB = " web:\n url: https://web.example/mcp\n secrets: [Authorization]\n"; - - it("a secret is set write-only, pushed with the key, and the answer says only which are set", async () => { - const { scheduler, driver } = make(); - const dot = await scheduler.createDot(withServers("mcp-push", TIME + WEB)); - await waitUntilSettledReady(scheduler, driver, dot.id, "mcp-push"); - const guest = driver.guestOf(dot.id); - expect(guest.mcpSecrets).toEqual({}); - - const answer = await scheduler.setMcpSecret(dot.id, "web", "Authorization", " Bearer w-1 "); - expect(answer.secrets).toEqual([ - { server: "time", name: "TIME_TOKEN", set: false }, - { server: "web", name: "Authorization", set: true }, - ]); - expect(JSON.stringify(answer)).not.toContain("w-1"); - expect(guest.mcpSecrets).toEqual({ web: { Authorization: "Bearer w-1" } }); - - await scheduler.setMcpSecret(dot.id, "web", "Authorization", null); - expect(guest.mcpSecrets).toEqual({}); - // The guest restarts with nothing in memory: the next READY pushes what is stored again. - await scheduler.setMcpSecret(dot.id, "time", "TIME_TOKEN", "tt-1"); - guest.restartAgent(); - await waitFor(() => guest.mcpSecrets.time?.TIME_TOKEN === "tt-1", "the secrets pushed again after the restart"); - }); - - it("refuses a secret the config does not name, and a value that cannot travel, without storing anything", async () => { - const { scheduler, driver } = make(); - const dot = await scheduler.createDot(withServers("mcp-refuse", TIME)); - await waitUntilSettledReady(scheduler, driver, dot.id, "mcp-refuse"); - - await expect(scheduler.setMcpSecret(dot.id, "ghost", "TIME_TOKEN", "x")).rejects.toMatchObject({ status: 404 }); - await expect(scheduler.setMcpSecret(dot.id, "time", "OTHER", "x")).rejects.toMatchObject({ status: 404 }); - const refused = scheduler.setMcpSecret(dot.id, "time", "TIME_TOKEN", "tok-SECRET\nX"); - await expect(refused).rejects.toMatchObject({ status: 400, code: "invalid_request" }); - await refused.catch((error: Error) => expect(error.message).not.toContain("SECRET")); - expect((await scheduler.mcpSecrets(dot.id)).secrets).toEqual([{ server: "time", name: "TIME_TOKEN", set: false }]); - }); - - it("a config that drops a server, or a name of its secrets, drops their values with it", async () => { - const { scheduler, driver } = make(); - const dot = await scheduler.createDot(withServers("mcp-drop", TIME + WEB)); - await waitUntilSettledReady(scheduler, driver, dot.id, "mcp-drop"); - await scheduler.setMcpSecret(dot.id, "time", "TIME_TOKEN", "tt-1"); - await scheduler.setMcpSecret(dot.id, "web", "Authorization", "Bearer w-1"); - - await scheduler.updateDot(dot.id, withServers("mcp-drop", TIME)); - expect(await db.secrets.get(dot.id, "mcp/web/Authorization")).toBeNull(); - expect(driver.guestOf(dot.id).mcpSecrets).toEqual({ time: { TIME_TOKEN: "tt-1" } }); - // Declared again, it starts with nothing set. - await scheduler.updateDot(dot.id, withServers("mcp-drop", TIME + WEB)); - expect((await scheduler.mcpSecrets(dot.id)).secrets.find((s) => s.server === "web")?.set).toBe(false); - }); - - it("the tool table says where each declared server is, and its tools are under its permission", async () => { - const { scheduler, driver } = make(); - const dot = await scheduler.createDot(withServers("mcp-table", TIME + WEB, ' mcp.time: allow\n')); - await waitUntilSettledReady(scheduler, driver, dot.id, "mcp-table"); - const guest = driver.guestOf(dot.id); - guest.mcpTools.set("time", [{ name: "mcp_time_now", description: "Now." }]); - guest.mcpFailures.set("web", "its secret Authorization is not set"); - - const { tools, mcp_servers } = await scheduler.listTools(dot.id); - expect(tools.find((t) => t.name === "mcp_time_now")).toEqual({ name: "mcp_time_now", permission: "mcp.time", offered: true, description: "Now." }); - expect(mcp_servers).toEqual([ - { name: "time", state: "connected", error: null, tools: 1 }, - { name: "web", state: "failed", error: "its secret Authorization is not set", tools: 0 }, - ]); - }); - - describe("a server the Dot adds itself (add_mcp_server)", () => { - const ARGS = { name: "clock", command: "uvx", args: ["mcp-server-time"], reason: "to tell the time" }; - const refusals = async (dotId: string) => (await db.events.list({ dotId })).filter((e) => e.type === "mcp.server.refused").map((e) => e.data); - - it("is written into the config when the person approved exactly these arguments, with its permission left to ask, and pushed", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "mcp-self"); - const guest = driver.guestOf(dot.id); - // The same arguments in another order: the person approved this entry, however the keys were written. - const id = guest.requestApproval(undefined, "add_mcp_server", "mcp.servers.add", { reason: "to tell the time", args: ["mcp-server-time"], command: "uvx", name: "clock" }); - await waitFor(async () => (await db.approvals.get(id)) !== null, "the approval stored"); - await scheduler.resolveApproval(id, "approve"); - - guest.emit("mcp.server.added", { arguments: ARGS }); - await waitFor(async () => "clock" in ((await db.dots.get(dot.id))?.config.mcp_servers ?? {}), "the server in the config"); - const config = (await db.dots.get(dot.id))!.config; - expect(config.mcp_servers.clock).toEqual({ command: "uvx", args: ["mcp-server-time"], env: {}, secrets: [], timeout_s: 120, startup_timeout_s: 60 }); - expect(config.permissions).not.toHaveProperty("mcp.clock"); - await waitFor(() => guest.config?.mcp_servers.clock !== undefined, "the config pushed to the guest"); - expect((await db.events.list({ dotId: dot.id })).some((e) => e.type === "dot.updated" && e.data.mcp_server_added === "clock" && e.data.approval_id === id)).toBe(true); - }); - - it("uses an approval up: it adds one server once, and a call refused for its entry keeps it", async () => { - const { scheduler, driver } = make(); - const dot = await scheduler.createDot(`${yaml("mcp-once")}mcp_servers:\n clock:\n command: npx\n`); - await waitUntilSettledReady(scheduler, driver, dot.id, "mcp-once"); - const guest = driver.guestOf(dot.id); - const id = guest.requestApproval(undefined, "add_mcp_server", "mcp.servers.add", ARGS); - await waitFor(async () => (await db.approvals.get(id)) !== null, "the approval stored"); - await scheduler.resolveApproval(id, "approve"); - - // "clock" is taken: refused before the approval is touched. - guest.emit("mcp.server.added", { arguments: ARGS }); - await waitFor(async () => (await refusals(dot.id)).length === 1, "refused: the name is taken"); - expect((await refusals(dot.id))[0]).toEqual({ name: "clock", reason: 'the Dot\'s config already declares an MCP server "clock"' }); - - // The person frees the name: the same approval adds it now. - await scheduler.updateDot(dot.id, yaml("mcp-once")); - guest.emit("mcp.server.added", { arguments: ARGS }); - await waitFor(async () => { - const clock = (await db.dots.get(dot.id))?.config.mcp_servers.clock; - return clock !== undefined && "command" in clock && clock.command === "uvx"; - }, "added with the approval"); - - // Removed again, the same call cannot bring it back on the approval it used. - await scheduler.updateDot(dot.id, yaml("mcp-once")); - guest.emit("mcp.server.added", { arguments: ARGS }); - await waitFor(async () => (await refusals(dot.id)).length === 2, "refused: the approval is used"); - expect((await refusals(dot.id))[1]).toEqual({ name: "clock", reason: "mcp.servers.add asks, and no unused approval of exactly these arguments was given" }); - expect((await db.dots.get(dot.id))?.config.mcp_servers).toEqual({}); - }); - - it("is written without an approval when the person allowed mcp.servers.add", async () => { - const { scheduler, driver } = make(); - const dot = await scheduler.createDot(`${yaml("mcp-allowed")}permissions:\n mcp.servers.add: allow\n`); - await waitUntilSettledReady(scheduler, driver, dot.id, "mcp-allowed"); - driver.guestOf(dot.id).emit("mcp.server.added", { arguments: { name: "web", url: "https://web.example/mcp", secrets: ["Authorization"], reason: "r" } }); - await waitFor(async () => "web" in ((await db.dots.get(dot.id))?.config.mcp_servers ?? {}), "the server in the config"); - expect((await scheduler.mcpSecrets(dot.id)).secrets).toEqual([{ server: "web", name: "Authorization", set: false }]); - }); - - it("is refused, and said, when nobody approved it, the permission is denied, the name is taken or the entry is not one", async () => { - const { scheduler, driver } = make(); - const dot = await scheduler.createDot(`${yaml("mcp-refused")}mcp_servers:\n taken:\n command: uvx\n`); - await waitUntilSettledReady(scheduler, driver, dot.id, "mcp-refused"); - const guest = driver.guestOf(dot.id); - guest.emit("mcp.server.added", { arguments: ARGS }); - await waitFor(async () => (await refusals(dot.id)).length === 1, "the first refusal"); - expect((await refusals(dot.id))[0]).toEqual({ name: "clock", reason: "mcp.servers.add asks, and no unused approval of exactly these arguments was given" }); - - await scheduler.updateDot(dot.id, `${yaml("mcp-refused")}permissions:\n mcp.servers.add: allow\nmcp_servers:\n taken:\n command: uvx\n`); - guest.emit("mcp.server.added", { arguments: { name: "taken", command: "npx" } }); - guest.emit("mcp.server.added", { arguments: { name: "both", command: "uvx", url: "https://a.example/mcp" } }); - await waitFor(async () => (await refusals(dot.id)).length === 3, "two more refusals"); - const [, taken, both] = await refusals(dot.id); - expect(taken).toEqual({ name: "taken", reason: 'the Dot\'s config already declares an MCP server "taken"' }); - expect((both as { reason: string }).reason).toMatch(/MCP server is either/); - - await scheduler.updateDot(dot.id, `${yaml("mcp-refused")}permissions:\n mcp.servers.add: deny\nmcp_servers:\n taken:\n command: uvx\n`); - guest.emit("mcp.server.added", { arguments: ARGS }); - await waitFor(async () => (await refusals(dot.id)).length === 4, "the denied one"); - expect((await refusals(dot.id))[3]).toEqual({ name: "clock", reason: "the Dot's config denies mcp.servers.add" }); - expect(Object.keys((await db.dots.get(dot.id))!.config.mcp_servers)).toEqual(["taken"]); - }); - }); - - it("always allowing the permission of a server the config no longer declares is refused, and nothing changes", async () => { - const { scheduler, driver } = make(); - const dot = await scheduler.createDot(withServers("mcp-always", TIME)); - await waitUntilSettledReady(scheduler, driver, dot.id, "mcp-always"); - const id = driver.guestOf(dot.id).requestApproval(undefined, "mcp_time_now", "mcp.time"); - await waitFor(async () => (await scheduler.listApprovals("pending")).some((a) => a.id === id), "the approval"); - await scheduler.updateDot(dot.id, yaml("mcp-always")); - - await expect(scheduler.resolveApproval(id, "approve", { always: true })).rejects.toMatchObject({ status: 409, code: "permission_gone" }); - expect((await db.approvals.get(id))?.status).toBe("pending"); - expect((await db.dots.get(dot.id))?.config.permissions).toEqual({}); - }); - }); - - it("tools: the engine's table with what is offered now; an unreachable computer is not an empty table", async () => { - const { scheduler, driver } = make(); - const dot = await readyDot(scheduler, "toolbox"); - const guest = driver.guestOf(dot.id); - - const { tools, mcp_servers } = await scheduler.listTools("toolbox"); - expect(mcp_servers).toEqual([]); - expect(tools.map((t) => [t.name, t.permission])).toEqual([ - ["exec", "computer.exec"], - ["read_file", "files.read"], - ["write_file", "files.write"], - ["cron", "automations"], - ["browser_identity_list", "browser.identity.list"], - ["browser_identity_create", "browser.identity.create"], - ]); - // The config the host pushed has its defaults filled in: nothing is denied, so the model is offered every tool. - expect(tools.every((t) => t.offered)).toBe(true); - await scheduler.updateDot(dot.id, `${yaml("toolbox")}permissions: - automations: deny -`); - await waitFor(async () => (await scheduler.listTools(dot.id)).tools.some((t) => !t.offered), "config pushed"); - expect((await scheduler.listTools(dot.id)).tools.filter((t) => !t.offered).map((t) => t.name)).toEqual(["cron"]); - - guest.powerOff(); - await expect(scheduler.listTools(dot.id)).rejects.toMatchObject({ status: 502, code: "guest_unavailable" }); - }); - - it("the tools need a running computer", async () => { - const { scheduler } = make(); - const dot = await readyDot(scheduler, "tool-sleeper"); - await scheduler.stopComputer(dot.id); - await scheduler.settle(); - - await expect(scheduler.listTools(dot.id)).rejects.toMatchObject({ status: 409, code: "computer_stopped" }); - await expect(scheduler.listSkills(dot.id)).rejects.toMatchObject({ status: 409, code: "computer_stopped" }); - }); - - it("delete removes the Dot's own OpenRouter key and keeps the global one", async () => { - const { scheduler } = make(); - const dot = await readyDot(scheduler, "keyed"); - await scheduler.setOpenRouterKey("sk-or-own", dot.id); - expect(await db.secrets.get(dot.id, "openrouter_api_key")).toBe("sk-or-own"); - await scheduler.deleteDot("keyed"); - await scheduler.settle(); - expect(await db.secrets.get(dot.id, "openrouter_api_key")).toBeNull(); - expect(await db.secrets.get("global", "openrouter_api_key")).toBe("sk-or-test"); - }); - - it("recovery: a stop that was interrupted is finished for the reason it was asked for", async () => { - const driver = new FakeDriver(); - const first = make(driver).scheduler; - const byIdle = await readyDot(first, "cut-idle"); - const byPerson = await readyDot(first, "cut-person"); - const offByIdle = await readyDot(first, "cut-off"); - await first.close(); - open.splice(open.indexOf(first), 1); - - // The control plane went down between STOPPING and the end of the stop: two VMs still run, one is already off. - await db.computers.setState(byIdle.id, "STOPPING", undefined, "idle"); - await db.computers.setState(byPerson.id, "STOPPING", undefined, "user"); - await db.computers.setState(offByIdle.id, "STOPPING", undefined, "idle"); - driver.crash(offByIdle.id); - - const { scheduler } = make(driver, { dispatchIntervalMs: 60_000, idleCheckIntervalMs: 60_000 }); - await scheduler.start(); - await scheduler.settle(); - expect(await db.computers.get(byIdle.id)).toMatchObject({ state: "STOPPED", stop_reason: "idle" }); - expect(await db.computers.get(byPerson.id)).toMatchObject({ state: "STOPPED", stop_reason: "user" }); - expect(await db.computers.get(offByIdle.id)).toMatchObject({ state: "STOPPED", stop_reason: "idle" }); - }); - - it("recovery: an idle stop that was interrupted is finished also when work came due meanwhile, and the computer is woken for it", async () => { - const driver = new FakeDriver(); - const lifecycle = { healthPollMs: 5, readyTimeoutMs: 3_000, pumpRetryMs: 10, pumpMaxRetryMs: 50, automationWakeLeadMs: 60_000 }; - const first = make(driver, { lifecycle }).scheduler; - const withAutomation = await readyDot(first, "cut-automation"); - const withTask = await readyDot(first, "cut-task"); - await first.close(); - open.splice(open.indexOf(first), 1); - - // Both were being put to sleep when the control plane went down; while it was down an automation came due for - // one and a task for the other. - await db.computers.setState(withAutomation.id, "STOPPING", undefined, "idle"); - await db.computers.setState(withTask.id, "STOPPING", undefined, "idle"); - const { scheduler, clock } = make(driver, { dispatchIntervalMs: 60_000, idleCheckIntervalMs: 60_000, lifecycle }); - await db.computers.setNextAutomation(withAutomation.id, clock.now().getTime() - 1_000); - const task = await db.tasks.insert({ id: `task_cut${Date.now()}`, dotId: withTask.id, description: "came due while away" }); - - await scheduler.start(); - await waitFor(async () => (await db.tasks.get(task.id))?.status === "COMPLETED", "the task ran"); - await scheduler.settle(); - // The pass that start() runs came before the stop was over; the next one wakes the computer for its automation. - await scheduler.pass(); - await scheduler.settle(); - // The stop was finished, not called off: a computer that runs is a computer that is READY and pumped. - for (const dot of [withAutomation, withTask]) { - expect(driver.calls.filter((c) => c === `stop:${dot.id}`)).toHaveLength(1); - expect(driver.calls.filter((c) => c === `start:${dot.id}`)).toHaveLength(2); - expect(await db.computers.get(dot.id)).toMatchObject({ state: "RUNNING", stop_reason: null }); - // settle() is the scheduler's own work: the automation the woken guest runs may still be RUNNING the Dot (seen on - // a loaded PostgreSQL), and it is READY once that is done. - await waitFor(async () => (await db.dots.get(dot.id))?.status === "READY", `${dot.name} READY`, 15_000); - expect(scheduler.lifecycle.isReady(dot.id)).toBe(true); - } - }); - - it("recovery: reattaches to running VMs, marks powered-off ones STOPPED and delivers undelivered tasks", async () => { - const driver = new FakeDriver(); - const first = make(driver).scheduler; - const alive = await readyDot(first, "survivor"); - const dead = await readyDot(first, "casualty"); - await first.close(); - open.splice(open.indexOf(first), 1); - - // While the control plane was down: one VM powered off, one task claimed but never delivered. - driver.crash(dead.id); - const orphan = await db.tasks.insert({ id: `task_orphan${Date.now()}`, dotId: alive.id, description: "orphan" }); - await db.transaction((tx) => tx.tasks.claimNext()); - driver.guestOf(alive.id).emit("browser.identity.created", { identity_id: "bi_away", name: "while-away" }); - - const { scheduler } = make(driver, { dispatchIntervalMs: 60_000, idleCheckIntervalMs: 60_000 }); - await scheduler.start(); - await waitFor(async () => (await db.tasks.get(orphan.id))?.status === "COMPLETED", "orphan task completed"); - // Reattached through the recorded port: no second QEMU was spawned for the survivor. - expect(driver.calls.filter((c) => c === `start:${alive.id}`)).toHaveLength(1); - expect(scheduler.lifecycle.isReady(alive.id)).toBe(true); - expect(await db.computers.get(dead.id)).toMatchObject({ state: "STOPPED", guest_port: null, pid: null, stop_reason: "exited" }); - expect((await db.dots.get(dead.id))?.status).toBe("IDLE"); - // Claimed before the restart, sent after it from the outbox: one run, one task.created. - const runs = await db.tasks.runs(orphan.id); - expect(runs.map((r) => r.outcome)).toEqual(["completed"]); - expect(driver.guestOf(alive.id).inbound.filter((e) => e.type === "task.created" && e.data.task_id === orphan.id)).toHaveLength(1); - const away = await db.events.list({ dotId: alive.id, types: ["browser.identity.created"] }); - expect(away.map((e) => e.data.name)).toEqual(["while-away"]); - }); -}); diff --git a/apps/scheduler/test/upgrade.test.ts b/apps/scheduler/test/upgrade.test.ts deleted file mode 100644 index ceb465ae..00000000 --- a/apps/scheduler/test/upgrade.test.ts +++ /dev/null @@ -1,102 +0,0 @@ -/** - * A release's database under this control plane: what a person's install holds must open, migrate and - * go on working after an upgrade. Each directory of test/fixtures/upgrade holds the database one release - * left (host.sql, made by make-host-state.ts from that release's checkout) and the test master key its - * secrets were sealed with. - */ -import { mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { fileURLToPath } from "node:url"; -import { Database, loadMigrations, OPENROUTER_KEY_NAME, PgliteDb } from "@invisible-dots/database"; -import { createScratchPostgres, testAdapters } from "@invisible-dots/database/testing"; -import pg from "pg"; -import { afterEach, describe, expect, it } from "vitest"; -import { Scheduler } from "../src/index.js"; -import { FakeDriver, ManualClock } from "../src/testing.js"; - -const FIXTURES = fileURLToPath(new URL("./fixtures/upgrade/", import.meta.url)); -const RELEASES = readdirSync(FIXTURES, { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name); - -describe.each(testAdapters())("a release's database under this control plane (%s)", (kind) => { - const cleanup: (() => Promise)[] = []; - afterEach(async () => { - for (const step of cleanup.splice(0).reverse()) await step(); - }); - - /** The release's database loaded as it was left, then opened and migrated by this build. */ - async function upgraded(release: string): Promise { - const dump = readFileSync(join(FIXTURES, release, "host.sql"), "utf8"); - const masterKey = Buffer.from(readFileSync(join(FIXTURES, release, "master.key"), "utf8").trim(), "hex"); - let db: Database; - if (kind === "pglite") { - const dataDir = mkdtempSync(join(tmpdir(), "idots-upgrade-")); - cleanup.push(async () => rmSync(dataDir, { recursive: true, force: true })); - const loader = await PgliteDb.open(dataDir); - await loader.exec(dump); - await loader.close(); - db = await Database.open({ target: { kind: "pglite", dataDir }, masterKey }); - } else { - const { url, drop } = await createScratchPostgres(); - cleanup.push(drop); - const loader = new pg.Client({ connectionString: url }); - await loader.connect(); - await loader.query(dump); - await loader.end(); - db = await Database.open({ target: { kind: "pg", url }, masterKey, poolSize: 3 }); - } - cleanup.push(() => db.close()); - await db.migrate(); - return db; - } - - it("has a release to upgrade from", () => { - expect(RELEASES.length).toBeGreaterThan(0); - }); - - it.each(RELEASES)("%s: migrates to every migration of this build, and its Dots, configs, tasks and secrets read back", async (release) => { - const db = await upgraded(release); - const applied = (await db.query<{ version: string }>("SELECT version FROM schema_migrations ORDER BY version")).rows.map((r) => r.version); - expect(applied).toEqual((await loadMigrations()).map((m) => m.version)); - - const dots = await db.dots.list(); - expect(dots.map((d) => d.name).sort()).toEqual(["fare-watch", "minimal-dot"]); - const watcher = dots.find((d) => d.name === "fare-watch")!; - expect(watcher.config.model.id).toBe("z-ai/glm-5.3-flash"); - expect(watcher.config.permissions?.["files.write"]).toBe("ask"); - expect((await db.computers.get(dots.find((d) => d.name === "minimal-dot")!.id))?.state).toBe("STOPPED"); - - const tasks = await db.tasks.listByDot(watcher.id, { limit: 50 }); - expect(tasks.map((t) => t.status).sort()).toEqual(["CANCELLED", "COMPLETED", "COMPLETED", "WAITING_APPROVAL"]); - - // The secrets open with the key they were sealed with. - expect(await db.secrets.get("global", OPENROUTER_KEY_NAME)).toBe("sk-or-v1-upgrade-fixture-not-a-real-key"); - expect(await db.secrets.get(watcher.id, "telegram_bot_token")).toBe("123456:upgrade-fixture-not-a-real-token"); - const [binding] = await db.channels.listBindings(watcher.id); - expect(binding).toMatchObject({ kind: "telegram", account: "fare_watch_bot" }); - expect((await db.channels.peers(binding!.id)).map((p) => [p.peer_id, p.role])).toEqual([["4242", "owner"]]); - }); - - it.each(RELEASES)("%s: the approval it left pending is answered, new work is queued, and new events follow the old ones", async (release) => { - const db = await upgraded(release); - const scheduler = new Scheduler({ db, driver: new FakeDriver(), clock: new ManualClock(new Date("2026-11-01T09:00:00.000Z")) }); - cleanup.push(() => scheduler.close()); - const watcher = await scheduler.requireDot("fare-watch"); - const lastEvent = (await db.query<{ id: string }>("SELECT max(id)::text AS id FROM events")).rows[0]!.id; - - const [pending] = await scheduler.listApprovals("pending"); - expect(pending).toMatchObject({ dot_id: watcher.id, status: "pending", tool: "browser_identity_delete" }); - const resolved = await scheduler.resolveApproval(pending!.id, "approve", { note: "after the upgrade" }); - expect(resolved.status).toBe("approved"); - const queued = (await db.query<{ type: string; data: { approval_id?: string } }>("SELECT type, data FROM inbound_events WHERE dot_id = $1 AND sent_at IS NULL", [watcher.id])).rows; - expect(queued.some((row) => row.type === "approval.received" && row.data.approval_id === pending!.id)).toBe(true); - - const task = await scheduler.createTask("fare-watch", { description: "Check the fares after the upgrade." }); - expect(task.status).toBe("PENDING"); - const conversation = await scheduler.conversation("fare-watch"); - expect(JSON.stringify(conversation)).toContain("Remember: my favourite colour is teal."); - - const events = await scheduler.listEvents(watcher.id, { after: Number(lastEvent) }); - expect(events.map((e) => e.type)).toEqual(expect.arrayContaining(["approval.resolved", "task.created"])); - }); -}); diff --git a/apps/scheduler/tsconfig.json b/apps/scheduler/tsconfig.json deleted file mode 100644 index 31a061f4..00000000 --- a/apps/scheduler/tsconfig.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "include": [ - "src/**/*.ts", - "test/**/*.ts" - ] -} diff --git a/apps/vm-manager/package.json b/apps/vm-manager/package.json deleted file mode 100644 index 7b1ff6ed..00000000 --- a/apps/vm-manager/package.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "name": "@invisible-dots/vm-manager", - "version": "0.0.0", - "private": true, - "description": "QEMU driver: overlays, seed ISOs, the QEMU command line, port forwards, the guest client and the host's process runner.", - "license": "MIT", - "type": "module", - "exports": { - ".": "./src/index.ts" - }, - "scripts": { - "typecheck": "tsc -p tsconfig.json" - }, - "dependencies": { - "@invisible-dots/iso": "0.0.0", - "@invisible-dots/shared": "0.0.0" - }, - "devDependencies": { - "yaml": "^2.9.1" - } -} diff --git a/apps/vm-manager/src/accelerator-access.ts b/apps/vm-manager/src/accelerator-access.ts deleted file mode 100644 index 5e65e2da..00000000 --- a/apps/vm-manager/src/accelerator-access.ts +++ /dev/null @@ -1,102 +0,0 @@ -/** - * The host side of the accelerator (architecture section 1.1), read before - * QEMU is asked to use it: /dev/kvm opened read-write on Linux, the - * HypervisorPlatform optional feature on Windows. One of the two places that - * ask which operating system this is (the other is `accelerator()` in - * host.ts); `doctor` reports what it finds and `invisible-dots setup` - * enables what it finds missing. - */ -import { open } from "node:fs/promises"; -import type { DoctorCheck } from "@invisible-dots/shared"; -import type { Runner } from "./doctor.js"; -import { firstLine, runProcess } from "./runner.js"; -import { HYPERVISOR_PLATFORM_FEATURE, HYPERVISOR_PLATFORM_STATE_SCRIPT, powershellArgs, powershellPath } from "./powershell.js"; - -const KVM_DEVICE = "/dev/kvm"; -const USERMOD_COMMAND = "sudo usermod -aG kvm $USER"; -const FEATURE_QUERY_TIMEOUT_MS = 60_000; -export const UNSUPPORTED_FIX = "run invisible_dots on Linux or Windows (x86-64)"; - -export interface AccessDeps { - platform: NodeJS.Platform; - env: Record; - run: Runner; - /** Opens a path read-write and closes it again; rejects with the errno error. */ - openReadWrite(path: string): Promise; -} - -/** This machine: its platform, the real process runner and the real /dev/kvm. */ -export function hostAccessDeps(env: Record): AccessDeps { - return { - platform: process.platform, - env, - run: runProcess, - openReadWrite: async (path) => { - await (await open(path, "r+")).close(); - }, - }; -} - -/** The host side of the accelerator, before QEMU is asked to use it (doctor's "accelerator" check). */ -export async function checkAcceleratorAccess(deps: AccessDeps): Promise { - if (deps.platform === "linux") return checkKvmDevice(deps); - if (deps.platform === "win32") return checkHypervisorPlatform(deps); - return { - id: "accelerator", - label: "accelerator", - status: "failed", - detail: `${deps.platform} hosts are not supported`, - fix: UNSUPPORTED_FIX, - }; -} - -export async function checkKvmDevice(deps: AccessDeps): Promise { - const base = { id: "accelerator", label: "accelerator" } as const; - try { - await deps.openReadWrite(KVM_DEVICE); - return { ...base, status: "ok", detail: `${KVM_DEVICE} opens read-write` }; - } catch (error) { - const code = (error as NodeJS.ErrnoException).code; - if (code === "ENOENT") { - return { - ...base, - status: "missing", - detail: `${KVM_DEVICE} does not exist`, - fix: "enable hardware virtualization (Intel VT-x or AMD-V) in the firmware settings, then: sudo modprobe kvm_intel (or kvm_amd)", - }; - } - if (code === "EACCES" || code === "EPERM") { - return { - ...base, - status: "missing", - detail: `${KVM_DEVICE} exists but this user cannot open it read-write`, - fix: `${USERMOD_COMMAND}, then log out and in again (a new login is needed for the group to apply)`, - }; - } - return { ...base, status: "failed", detail: `opening ${KVM_DEVICE} failed: ${(error as Error).message}`, fix: "check the kvm kernel module (dmesg | grep -i kvm)" }; - } -} - -async function checkHypervisorPlatform(deps: AccessDeps): Promise { - const base = { id: "accelerator", label: "accelerator" } as const; - const answer = await deps.run(powershellPath(deps.env), powershellArgs(HYPERVISOR_PLATFORM_STATE_SCRIPT), { timeoutMs: FEATURE_QUERY_TIMEOUT_MS }); - if (answer.code !== 0) { - const why = answer.timedOut ? "timed out" : (answer.startError?.message ?? (firstLine(answer.stderr) || `exit code ${answer.code}`)); - return { ...base, status: "failed", detail: `cannot read the ${HYPERVISOR_PLATFORM_FEATURE} feature state: ${why}` }; - } - const state = answer.stdout.trim(); - if (state === "1") return { ...base, status: "ok", detail: `the ${HYPERVISOR_PLATFORM_FEATURE} feature is enabled` }; - if (state === "2") { - return { - ...base, - status: "missing", - detail: `the ${HYPERVISOR_PLATFORM_FEATURE} feature is disabled`, - fix: "invisible-dots setup (enables it with one administrator prompt; a restart follows)", - }; - } - const detail = - state === "" - ? `this Windows does not list the ${HYPERVISOR_PLATFORM_FEATURE} feature` - : `the ${HYPERVISOR_PLATFORM_FEATURE} feature is in state ${state} (3 means its files were removed from this Windows)`; - return { ...base, status: "failed", detail, fix: "use a Windows 10 or 11 edition that offers Windows Hypervisor Platform" }; -} diff --git a/apps/vm-manager/src/doctor.ts b/apps/vm-manager/src/doctor.ts deleted file mode 100644 index d05f4706..00000000 --- a/apps/vm-manager/src/doctor.ts +++ /dev/null @@ -1,291 +0,0 @@ -/** - * The host report (architecture section 11.1): every check, in the - * contract's order, each answering ok / missing / failed and naming the - * command that fixes it. It runs in the API process (`GET /api/doctor`) and - * in `invisible-dots doctor` alike. Doctor only reads: it never creates a - * directory, never opens the database and never changes a setting. - * - * Status meanings, so every check uses them the same way (`DoctorStatus`): - * - ok: present and working. - * - missing: absent; the fix line installs or creates it. - * - failed: present but not working, or the check itself could not tell. - */ -import { ENV, type DoctorAnswer, type DoctorCheck, type DoctorCheckId, type DoctorStatus } from "@invisible-dots/shared"; -import { SETUP_COMMAND, STORE_OPENROUTER_KEY } from "./errors.js"; -import { isSupportedQemuVersion, MIN_QEMU_VERSION_TEXT, parseQemuVersion, type Accelerator, type QemuVersion } from "./host.js"; -import { CPU_MODEL, qemuPathProblem } from "./qemu-args.js"; -import { firstLine, type RunOptions, type RunResult } from "./runner.js"; - -/** `npm run build --workspace @invisible-dots/web`: what leaves the web client where `invisible-dots server` serves it from. */ -export const WEB_BUILD_COMMAND = "npm run build --workspace @invisible-dots/web"; - -export interface WebBuild { - /** The server script `node` runs. */ - entry: string; - /** The first file or directory of the build that does not exist; undefined when the build is complete. */ - missing: string | undefined; -} - -/** Where the QEMU programs were found, each searched separately (section 3.1). */ -export interface FoundQemu { - system?: string; - img?: string; - /** The places searched, in order, for the message when something is not found. */ - searched: string[]; - /** INVISIBLE_DOTS_QEMU_DIR is set, so it was the only place searched. */ - configured?: boolean; -} - -export type Runner = (command: string, args: readonly string[], options?: RunOptions) => Promise; - -export interface DoctorDeps { - /** `process.versions.node`. */ - nodeVersion: string; - /** The vm-manager's choice for this host (section 1.1); throws on a host invisible_dots does not run on. */ - accelerator(): Accelerator; - findQemu(): Promise; - run: Runner; - /** The host side of the accelerator: /dev/kvm or the HypervisorPlatform feature (accelerator-access.ts). */ - acceleratorAccess(): Promise; - /** INVISIBLE_DOTS_HOME, as the server and the image builder will use it. */ - home: string; - /** Free space for INVISIBLE_DOTS_HOME, measured on the nearest directory that exists. */ - freeSpace(): Promise<{ path: string; bytes: number }>; - /** The golden image and the runtime ISO against their manifests, one result each. */ - images(): Promise; - /** Where the built web client is, and the first file of it that is missing. Only a process that serves the web client (`invisible-dots server`) has one: without it the report has no web row. */ - webBuild?(): Promise; - /** Whether an OpenRouter key is stored. */ - openRouterKey(): Promise; -} - -export const MIN_NODE_MAJOR = 24; -/** - * One golden image (a few GiB once provisioned) plus the overlays of a few - * Dots, which grow with what each Dot stores. Below this a first `image - * build` or a busy Dot runs the disk full. - */ -export const MIN_FREE_BYTES = 20 * 1024 ** 3; -/** - * The probe boots the empty machine's firmware, which takes a fraction of a - * second with a working accelerator (0.2 s measured with QEMU 8.2 and KVM); - * a QEMU still running after this long has a virtual CPU that does not run. - */ -export const PROBE_TIMEOUT_MS = 30_000; -const VERSION_TIMEOUT_MS = 15_000; - -/** - * Starts QEMU with the accelerator and the CPU model of the Dots (section - * 3.4) on their machine type with no devices at all, and lets the virtual - * CPU run: the firmware finds nothing to boot, `-boot reboot-timeout=0` - * makes it reset at once, and `-no-reboot` turns that reset into QEMU - * exiting with code 0. So exit code 0 means the accelerator opened AND ran - * guest code, which a QEMU held before its first instruction could never - * show; there is no monitor to talk to, the same as for a Dot. `-accel` is - * given alone, so QEMU cannot fall back to software emulation. - * - * Measured with QEMU 8.2.2 and KVM: exit 0 after 0.2 s, and the firmware's - * debug output reads "No bootable device. Retrying in 0 seconds. Rebooting." - * Without `-no-reboot`, or without the reboot timeout, QEMU never exits. - * - * Not `-machine none`: measured with QEMU 11.1 on Windows, WHPX with - * `-machine none` aborts with "X86_MACHINE: Object ... is not an instance of - * type x86-machine" (exit 3) even where WHPX works, and it has no CPU to run. - */ -export function acceleratorProbeArgs(accelerator: Accelerator): string[] { - return [ - "-nodefaults", - "-no-user-config", - "-machine", - "q35", - "-accel", - accelerator, - "-cpu", - CPU_MODEL, - "-display", - "none", - "-no-reboot", - "-boot", - "reboot-timeout=0", - ]; -} - -/** - * PATH alone can hold dozens of directories, so the line names the search - * order of section 3.1 and how many places it covered rather than all of them. - */ -function notFound(found: FoundQemu): string { - const count = found.searched.length; - if (count === 1 && found.searched[0]!.startsWith("(")) return `not found ${found.searched[0]}`; - if (found.configured) return `not found in ${ENV.QEMU_DIR} (${found.searched[0] ?? ""}), the only place searched while it is set`; - return `not found in the official installer's directory or on PATH (${count} ${count === 1 ? "directory" : "directories"} searched)`; -} - -function versionText(version: QemuVersion): string { - return `${version.major}.${version.minor}.${version.micro}`; -} - -export function formatBytes(bytes: number): string { - return `${(bytes / 1024 ** 3).toFixed(1)} GiB`; -} - -function result(id: DoctorCheckId, label: string, status: DoctorStatus, detail: string, fix?: string): DoctorCheck { - return fix === undefined || status === "ok" ? { id, label, status, detail } : { id, label, status, detail, fix }; -} - -export function checkNode(version: string): DoctorCheck { - const major = Number(version.split(".")[0]); - if (Number.isInteger(major) && major >= MIN_NODE_MAJOR) return result("node", "Node", "ok", version); - return result("node", "Node", "failed", `${version} is older than ${MIN_NODE_MAJOR}`, `install Node ${MIN_NODE_MAJOR} or newer from https://nodejs.org`); -} - -async function checkQemuSystem(found: FoundQemu, run: Runner): Promise { - const label = "QEMU"; - const minimum = MIN_QEMU_VERSION_TEXT; - if (!found.system) { - return result("qemu", label, "missing", `qemu-system-x86_64 ${notFound(found)}`, SETUP_COMMAND); - } - const answer = await run(found.system, ["--version"], { timeoutMs: VERSION_TIMEOUT_MS }); - const version = answer.code === 0 ? parseQemuVersion(answer.stdout) : undefined; - if (!version) { - const why = answer.startError?.message ?? (firstLine(answer.stderr) || `exit code ${answer.code}`); - return result("qemu", label, "failed", `${found.system} --version did not report a version (${why})`, SETUP_COMMAND); - } - const text = versionText(version); - if (!isSupportedQemuVersion(version)) { - return result( - "qemu", - label, - "failed", - `${found.system} is version ${text}, older than ${minimum}`, - `${SETUP_COMMAND}, or set INVISIBLE_DOTS_QEMU_DIR to a QEMU ${minimum} or newer`, - ); - } - return result("qemu", label, "ok", `${text} at ${found.system}`); -} - -async function checkQemuImg(found: FoundQemu, run: Runner): Promise { - if (!found.img) return result("qemu-img", "qemu-img", "missing", `qemu-img ${notFound(found)}`, SETUP_COMMAND); - const answer = await run(found.img, ["--version"], { timeoutMs: VERSION_TIMEOUT_MS }); - const version = answer.code === 0 ? parseQemuVersion(answer.stdout) : undefined; - if (!version) { - const why = answer.startError?.message ?? (firstLine(answer.stderr) || `exit code ${answer.code}`); - return result("qemu-img", "qemu-img", "failed", `${found.img} --version did not report a version (${why})`, SETUP_COMMAND); - } - return result("qemu-img", "qemu-img", "ok", `${versionText(version)} at ${found.img}`); -} - -/** - * Runs whenever QEMU is ready, also when the host-side check said the - * accelerator is missing: that check reads a precondition, the probe reads - * the result. Measured on Windows 11 with VirtualMachinePlatform enabled - * (WSL 2): HypervisorPlatform reports disabled and WHPX still works. - */ -async function probeAccelerator(deps: DoctorDeps, qemu: DoctorCheck, access: DoctorCheck, system: string | undefined): Promise { - const label = "accelerator probe"; - if (qemu.status !== "ok" || !system) { - return result("accelerator-probe", label, qemu.status, `not run: QEMU is not ready`, qemu.fix); - } - const accelerator = deps.accelerator(); - const command = `qemu-system-x86_64 -accel ${accelerator} -cpu ${CPU_MODEL} -machine q35`; - const answer = await deps.run(system, acceleratorProbeArgs(accelerator), { timeoutMs: PROBE_TIMEOUT_MS }); - if (answer.code === 0 && !answer.timedOut) return result("accelerator-probe", label, "ok", `${command} ran the firmware and exited`); - const why = answer.timedOut - ? `did not exit within ${PROBE_TIMEOUT_MS / 1000} s, so the virtual CPU does not run${firstLine(answer.stderr) ? ` (QEMU said: ${firstLine(answer.stderr)})` : ""}` - : (answer.startError?.message ?? (firstLine(answer.stderr) || `exit code ${answer.code}`)); - const fix = - access.status !== "ok" && access.fix - ? access.fix - : "enable hardware virtualization (Intel VT-x or AMD-V) in the firmware settings; after enabling the accelerator, restart the computer"; - return result("accelerator-probe", label, "failed", `${command} failed: ${why}`, fix); -} - -/** A host-side "missing" that the probe contradicts is reported as what it is: QEMU uses the accelerator. */ -function confirmedByProbe(access: DoctorCheck, probe: DoctorCheck): DoctorCheck { - if (access.status !== "missing" || probe.status !== "ok") return access; - return result("accelerator", access.label, "ok", `${access.detail}, but the probe below shows QEMU uses the accelerator`); -} - -const DATA_DIRECTORY = "data directory"; - -/** - * INVISIBLE_DOTS_HOME: a path QEMU can be given (the same rule a Dot's start - * applies, qemuPathProblem), with enough free space. A home under an account - * name with an accent fails here, before an image build or a Dot does. - */ -export function checkDataDirectory(home: string, space: { path: string; bytes: number }): DoctorCheck { - const problem = qemuPathProblem(home); - if (problem) { - return result("disk", DATA_DIRECTORY, "failed", `${home} ${problem}`, `set ${ENV.HOME} to a directory whose path is plain ASCII without commas`); - } - const detail = `${formatBytes(space.bytes)} free at ${space.path}`; - if (space.bytes >= MIN_FREE_BYTES) return result("disk", DATA_DIRECTORY, "ok", detail); - return result( - "disk", - DATA_DIRECTORY, - "failed", - `${detail}, less than ${formatBytes(MIN_FREE_BYTES)}`, - `free some space, or set ${ENV.HOME} to a directory on a larger disk`, - ); -} - -/** The web client `invisible-dots server` serves: built where the server looks for it. */ -export function checkWebBuild(build: WebBuild): DoctorCheck { - if (!build.missing) return result("web", "web client", "ok", `built at ${build.entry}`); - return result("web", "web client", "missing", `not built: ${build.missing} does not exist`, WEB_BUILD_COMMAND); -} - -/** Runs one check; a check that throws is reported as failed instead of ending the report. */ -async function guarded(id: DoctorCheckId, label: string, check: () => Promise): Promise { - try { - return await check(); - } catch (error) { - return result(id, label, "failed", `the check itself failed: ${(error as Error).message}`); - } -} - -/** Every check of section 11.1, in its order. */ -export async function runDoctor(deps: DoctorDeps): Promise { - const results: DoctorCheck[] = [checkNode(deps.nodeVersion)]; - let found: FoundQemu = { searched: [] }; - try { - found = await deps.findQemu(); - } catch (error) { - found = { searched: [`(the search failed: ${(error as Error).message})`] }; - } - const qemu = await guarded("qemu", "QEMU", () => checkQemuSystem(found, deps.run)); - results.push(qemu); - results.push(await guarded("qemu-img", "qemu-img", () => checkQemuImg(found, deps.run))); - const access = await guarded("accelerator", "accelerator", () => deps.acceleratorAccess()); - const probe = await guarded("accelerator-probe", "accelerator probe", () => probeAccelerator(deps, qemu, access, found.system)); - results.push(confirmedByProbe(access, probe), probe); - results.push(await guarded("disk", DATA_DIRECTORY, async () => checkDataDirectory(deps.home, await deps.freeSpace()))); - try { - results.push(...(await deps.images())); - } catch (error) { - const detail = `the check itself failed: ${(error as Error).message}`; - results.push(result("golden-image", "golden image", "failed", detail), result("runtime-image", "runtime ISO", "failed", detail)); - } - const webBuild = deps.webBuild; - if (webBuild) results.push(await guarded("web", "web client", async () => checkWebBuild(await webBuild()))); - results.push(await guarded("openrouter", "OpenRouter key", () => deps.openRouterKey())); - return results; -} - -export function allOk(results: readonly DoctorCheck[]): boolean { - return results.every((r) => r.status === "ok"); -} - -/** The report as the API and `doctor --json` send it. */ -export function doctorAnswer(results: readonly DoctorCheck[]): DoctorAnswer { - return { ok: allOk(results), checks: [...results] }; -} - -/** - * The OpenRouter key row from what the control plane knows (`openrouter_configured` of its health), asked of - * the scheduler in the API and of the running server by the CLI. - */ -export function openRouterCheck(configured: boolean): DoctorCheck { - const base = { id: "openrouter", label: "OpenRouter key" } as const; - return configured ? { ...base, status: "ok", detail: "stored" } : { ...base, status: "missing", detail: "no key stored", fix: STORE_OPENROUTER_KEY }; -} diff --git a/apps/vm-manager/src/errors.ts b/apps/vm-manager/src/errors.ts deleted file mode 100644 index c1689fc8..00000000 --- a/apps/vm-manager/src/errors.ts +++ /dev/null @@ -1,174 +0,0 @@ -/** Errors raised by the vm-manager. Each one says which command or VM it is about, and what fixes it when something can. */ - -/** The commands that prepare a host (architecture section 11); errors and the doctor report about the host name them. */ -export const DOCTOR_COMMAND = "invisible-dots doctor"; -export const SETUP_COMMAND = "invisible-dots setup"; -/** Stores the OpenRouter key: in a terminal it asks for it, one line; piped, it reads standard input. */ -export const STORE_OPENROUTER_KEY = "invisible-dots secret openrouter"; - -/** A host command exited non-zero, timed out or could not be started. */ -export class CommandError extends Error { - readonly command: string; - readonly args: readonly string[]; - readonly exitCode: number | null; - readonly stdout: string; - readonly stderr: string; - readonly timedOut: boolean; - /** The errno code when the command could not be started at all (ENOENT when it is not installed). */ - readonly code: string | undefined; - - constructor(init: { - command: string; - args: readonly string[]; - exitCode: number | null; - stdout?: string; - stderr?: string; - timedOut?: boolean; - code?: string; - message?: string; - cause?: unknown; - }) { - const line = [init.command, ...init.args].join(" "); - const stderr = (init.stderr ?? "").trim(); - const reason = - init.message ?? - (init.timedOut - ? "timed out" - : init.code === "ENOENT" - ? `command not found (run "${DOCTOR_COMMAND}")` - : `exited with code ${init.exitCode}`); - super(`${line}: ${reason}${stderr ? `: ${stderr}` : ""}`, init.cause === undefined ? undefined : { cause: init.cause }); - this.name = "CommandError"; - this.command = init.command; - this.args = init.args; - this.exitCode = init.exitCode; - this.stdout = init.stdout ?? ""; - this.stderr = init.stderr ?? ""; - this.timedOut = init.timedOut ?? false; - this.code = init.code; - } - - /** Whether the command itself is missing from the host. */ - get notFound(): boolean { - return this.code === "ENOENT"; - } -} - -/** A feature the contract places out of scope for this version (architecture section 10). */ -export class NotImplementedError extends Error { - constructor(feature: string) { - super(`${feature} is out of scope for this version (architecture section 10)`); - this.name = "NotImplementedError"; - } -} - -/** The operation needs the VM in another state, e.g. a disk resize on a running VM. */ -export class VmStateError extends Error { - readonly dotId: string; - constructor(dotId: string, message: string) { - super(`dot ${dotId}: ${message}`); - this.name = "VmStateError"; - this.dotId = dotId; - } -} - -/** Invalid input to the vm-manager (a bad dot id, a shrinking disk, a missing image). */ -export class VmManagerError extends Error { - constructor(message: string, options?: { cause?: unknown }) { - super(message, options); - this.name = "VmManagerError"; - } -} - -/** QEMU (qemu-system-x86_64 or qemu-img) is not installed where the vm-manager looks (section 3.1). */ -export class QemuNotFoundError extends Error { - readonly binary: string; - readonly searched: readonly string[]; - constructor(binary: string, searched: readonly string[]) { - super( - `${binary} was not found (looked in: ${searched.length > 0 ? searched.join(", ") : "nothing"}). ` + - `Run "${SETUP_COMMAND}" to install QEMU, or set INVISIBLE_DOTS_QEMU_DIR to the directory that holds it; ` + - `"${DOCTOR_COMMAND}" checks the result.`, - ); - this.name = "QemuNotFoundError"; - this.binary = binary; - this.searched = searched; - } -} - -/** - * QEMU could not use the hardware accelerator. There is no fallback to - * software emulation (section 1.1): the start fails and says what fixes it. - */ -export class AcceleratorUnavailableError extends Error { - readonly accelerator: string; - readonly qemuOutput: string; - constructor(accelerator: string, qemuOutput: string) { - super( - `QEMU could not use the ${accelerator} accelerator, and invisible_dots never falls back to software emulation. ` + - `Run "${DOCTOR_COMMAND}" to see what is missing and "${SETUP_COMMAND}" to fix it. QEMU said: ${qemuOutput.trim() || "nothing"}`, - ); - this.name = "AcceleratorUnavailableError"; - this.accelerator = accelerator; - this.qemuOutput = qemuOutput; - } -} - -/** - * The accelerator does not run the Dots' CPU model (`CPU_MODEL`, section - * 3.4). Section 3.4 forbids replacing it with a guessed one, so this is an - * error to report, not to work around. - */ -export class CpuModelError extends Error { - readonly accelerator: string; - readonly qemuOutput: string; - constructor(accelerator: string, cpuModel: string, qemuOutput: string) { - super( - `QEMU rejected "-cpu ${cpuModel}" with the ${accelerator} accelerator. invisible_dots does not substitute another CPU model ` + - `(architecture section 3.4); run "${DOCTOR_COMMAND}". QEMU said: ${qemuOutput.trim() || "nothing"}`, - ); - this.name = "CpuModelError"; - this.accelerator = accelerator; - this.qemuOutput = qemuOutput; - } -} - -/** - * QEMU exited during start, never set up its port forward, or started but - * never brought the guest up. `qemuOutput` is what QEMU wrote to its log and - * `serialOutput` the end of the guest's serial console: together they are - * what says why, since nothing else watches a VM's insides (section 3.4). - */ -export class VmStartError extends Error { - readonly dotId: string; - readonly qemuOutput: string; - readonly serialOutput: string; - constructor(dotId: string, message: string, qemuOutput = "", options?: { cause?: unknown; serialOutput?: string }) { - const output = qemuOutput.trim(); - const serial = (options?.serialOutput ?? "").trim(); - super( - `dot ${dotId}: ${message}${output ? `. QEMU said: ${output}` : ""}${serial ? `. The serial console ends with: ${serial}` : ""}`, - options?.cause === undefined ? undefined : { cause: options.cause }, - ); - this.name = "VmStartError"; - this.dotId = dotId; - this.qemuOutput = qemuOutput; - this.serialOutput = options?.serialOutput ?? ""; - } -} - -/** A guest route answered with an error status, or with something that is not the expected shape. */ -export class GuestRequestError extends Error { - readonly status: number; - /** The `error` code of an `{ error, message }` body, when the guest sent one. */ - readonly code: string | undefined; - readonly route: string; - - constructor(route: string, status: number, message: string, code?: string) { - super(`${route}: ${status === 0 ? "" : `HTTP ${status}: `}${message}`); - this.name = "GuestRequestError"; - this.route = route; - this.status = status; - this.code = code; - } -} diff --git a/apps/vm-manager/src/guest-client.ts b/apps/vm-manager/src/guest-client.ts deleted file mode 100644 index 834eb1f3..00000000 --- a/apps/vm-manager/src/guest-client.ts +++ /dev/null @@ -1,495 +0,0 @@ -/** - * The host's client for one Dot: every dot-agentd route (architecture 5.2) and - * every agent route behind `/v1/agent` (5.3), over HTTP/1.1 to - * 127.0.0.1:, which QEMU forwards to dot-agentd's TCP port 1024 - * (5.1). Every request carries the Dot's bearer token: the port is not a - * credential, any local process and any guest can reach it. - */ -import { createHmac, randomBytes, timingSafeEqual } from "node:crypto"; -import { request, type IncomingMessage, type OutgoingHttpHeaders } from "node:http"; -import { - abortableSleep, - AGENTD_ROUTES, - AGENT_ROUTES, - FILE_TOO_LARGE, - GUEST_PROOF_CONTEXT, - GUEST_UNPROVEN, - GuestHealthTimeoutError, - isFatalGuestError, - parseOutboundEvent, - pollGuestHealth, - PREPARE_SLEEP_TIMEOUT_MS, - REFUSED_PROBLEM_MAX, - SseParser, - type AgentHealthAnswer, - type AgentStateAnswer, - type BrowserIdentity, - type BrowserIdentityListAnswer, - type CreateBrowserIdentityRequest, - type DotRuntimeConfig, - type ExecAnswer, - type ExecRequest, - type FileListAnswer, - type HealthAnswer, - type InboundEvent, - type OutboundEvent, - type PollGuestHealthOptions, - type PostEventAnswer, - type ProofAnswer, - type RefusedEvent, - type SystemAnswer, - type SecretsRequest, - type SkillListAnswer, - type ToolListAnswer, - silentLogger, - type Logger, -} from "@invisible-dots/shared"; -import { GuestRequestError } from "./errors.js"; -import { GUEST_PORT_HOST } from "./ports.js"; - -export interface GuestClientOptions { - /** Per-request timeout. Default 30 s; exec adds its own timeout on top. */ - timeoutMs?: number; - logger?: Logger; - /** Default 127.0.0.1, where QEMU binds the forward. */ - host?: string; -} - -export interface EventStreamOptions { - /** Resume after this outbound `seq`; the stream sends only newer events. Default 0. */ - after?: number; - /** - * Called, in order with the events, for a message the host refused, and awaited before the stream goes on. A refused - * message with a `seq` is passed once (the stream resumes after it); one without cannot be told apart on a reconnect. - */ - onRefused?: (info: RefusedEvent) => void | Promise; - signal?: AbortSignal; - /** First reconnect delay; doubles up to `maxReconnectDelayMs`. Default 500 ms. */ - reconnectDelayMs?: number; - maxReconnectDelayMs?: number; - /** Called before each reconnect, e.g. to log it. */ - onReconnect?: (info: { after: number; attempt: number; error: Error }) => void; -} - -/** The proof a guest holding `token` gives for `nonce` (`GET /v1/proof`, section 5.1). */ -export function guestProof(token: string, nonce: string): string { - return createHmac("sha256", token).update(`${GUEST_PROOF_CONTEXT}${nonce}`).digest("hex"); -} - -interface Call { - /** Sent without the token; only the proof route is. */ - unauthenticated?: boolean; - method?: string; - path: string; - body?: unknown; - rawBody?: Uint8Array; - contentType?: string; - timeoutMs?: number; - signal?: AbortSignal; - /** The most the answer may hold; a larger one ends the call with `FILE_TOO_LARGE` and is not buffered. */ - maxBytes?: number; -} - -interface Answer { - status: number; - body: Buffer; - /** The media type of the answer, without parameters, lower case; "" when it sent none. */ - contentType: string; -} - -function filesQuery(path: string): string { - return `?path=${encodeURIComponent(path)}`; -} - -function parseError(route: string, answer: Pick): GuestRequestError { - const text = answer.body.toString("utf8"); - try { - const parsed = JSON.parse(text) as { error?: unknown; message?: unknown }; - if (typeof parsed.error === "string") { - return new GuestRequestError( - route, - answer.status, - typeof parsed.message === "string" ? `${parsed.error}: ${parsed.message}` : parsed.error, - parsed.error, - ); - } - } catch { - // Not JSON: the raw text is the most useful message. - } - const hint = answer.status === 401 ? " (the Dot token was refused)" : ""; - return new GuestRequestError(route, answer.status, `${text.trim().slice(0, 500) || "no body"}${hint}`); -} - -export class GuestClient { - readonly host: string; - readonly port: number; - private readonly token: string; - private readonly timeoutMs: number; - private readonly logger: Logger; - /** Settles once whatever listens on the port proved it holds the token; until then the token is never sent. */ - private proven: Promise | undefined; - - constructor(port: number, token: string, options: GuestClientOptions = {}) { - if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error(`GuestClient needs a TCP port, got ${port}`); - if (!token) throw new Error("GuestClient needs the Dot token"); - this.host = options.host ?? GUEST_PORT_HOST; - this.port = port; - this.token = token; - this.timeoutMs = options.timeoutMs ?? 30_000; - this.logger = options.logger ?? silentLogger; - } - - /** Where this client sends requests, for logs and errors. */ - get address(): string { - return `${this.host}:${this.port}`; - } - - private open(method: string, path: string, headers: OutgoingHttpHeaders, signal: AbortSignal | undefined) { - // No keep-alive agent: after a VM restart the same port may lead to a new guest, and a pooled socket would not. - return request({ host: this.host, port: this.port, method, path, headers, agent: false, signal }); - } - - /** - * Make whatever listens on the port prove it holds the Dot token before - * the token is sent there (section 5.1). A port is not a credential: after - * a host restart or a QEMU that died, another process can listen on a - * port the host still has on record. A failed proof is GUEST_UNPROVEN, - * which no caller retries; a guest that is not up yet is retried as usual. - */ - async prove(options: { timeoutMs?: number; signal?: AbortSignal } = {}): Promise { - const nonce = randomBytes(16).toString("hex"); - const path = `${AGENTD_ROUTES.proof}?nonce=${nonce}`; - const answer = await this.json({ path, unauthenticated: true, ...options }); - const expected = Buffer.from(guestProof(this.token, nonce), "utf8"); - const given = Buffer.from(typeof answer?.proof === "string" ? answer.proof : "", "utf8"); - if (given.length !== expected.length || !timingSafeEqual(given, expected)) { - throw new GuestRequestError( - `GET ${AGENTD_ROUTES.proof}`, - 0, - `${this.address} is not this Dot's guest: it could not prove it holds the Dot token, so the token was not sent to it`, - GUEST_UNPROVEN, - ); - } - } - - private ensureProven(signal: AbortSignal | undefined): Promise { - this.proven ??= this.prove({ signal }).catch((error: unknown) => { - // A guest still booting fails the same way; the next request asks again. - this.proven = undefined; - throw error; - }); - return this.proven; - } - - private async send(call: Call): Promise { - if (!call.unauthenticated) await this.ensureProven(call.signal); - return this.sendRaw(call); - } - - private sendRaw(call: Call): Promise { - const method = call.method ?? "GET"; - const route = `${method} ${call.path}`; - const headers: OutgoingHttpHeaders = call.unauthenticated ? {} : { authorization: `Bearer ${this.token}` }; - let body: Buffer | undefined; - if (call.rawBody !== undefined) { - body = Buffer.from(call.rawBody); - headers["content-type"] = call.contentType ?? "application/octet-stream"; - } else if (call.body !== undefined) { - body = Buffer.from(JSON.stringify(call.body), "utf8"); - headers["content-type"] = "application/json"; - } - if (body !== undefined) headers["content-length"] = body.length; - const timeoutMs = call.timeoutMs ?? this.timeoutMs; - - return new Promise((resolve, reject) => { - let settled = false; - const finish = (error: Error | undefined, answer?: Answer) => { - if (settled) return; - settled = true; - clearTimeout(timer); - if (error) { - req.destroy(); - reject(error); - } else if (answer!.status < 200 || answer!.status >= 300) { - reject(parseError(route, answer!)); - } else { - resolve(answer!); - } - }; - const req = this.open(method, call.path, headers, call.signal); - const timer = setTimeout(() => finish(new GuestRequestError(route, 0, `no answer from ${this.address} within ${timeoutMs} ms`)), timeoutMs); - req.on("response", (res: IncomingMessage) => { - const chunks: Buffer[] = []; - let received = 0; - const tooLarge = () => new GuestRequestError(route, 413, `the answer is larger than ${call.maxBytes} bytes`, FILE_TOO_LARGE); - // An error answer is small and its text is the reason; only a successful body is held to the limit. - const limited = call.maxBytes !== undefined && (res.statusCode ?? 0) >= 200 && (res.statusCode ?? 0) < 300; - if (limited && Number(res.headers["content-length"]) > call.maxBytes!) return finish(tooLarge()); - res.on("data", (chunk: Buffer) => { - received += chunk.length; - if (limited && received > call.maxBytes!) return finish(tooLarge()); - chunks.push(chunk); - }); - res.on("error", (error) => finish(new GuestRequestError(route, 0, `answer interrupted: ${error.message}`))); - res.on("end", () => finish(undefined, { status: res.statusCode ?? 0, body: Buffer.concat(chunks), contentType: String(res.headers["content-type"] ?? "").split(";")[0]!.trim().toLowerCase() })); - }); - req.on("error", (error: NodeJS.ErrnoException) => { - const hint = error.code === "ECONNREFUSED" ? " (nothing listens on the guest port: is the VM running?)" : ""; - finish(new GuestRequestError(route, 0, `${this.address}: ${error.message}${hint}`, error.code)); - }); - req.end(body); - }); - } - - private async json(call: Call): Promise { - const answer = await this.send(call); - const text = answer.body.toString("utf8"); - try { - return JSON.parse(text) as T; - } catch { - throw new GuestRequestError(`${call.method ?? "GET"} ${call.path}`, answer.status, `answer is not JSON: ${text.slice(0, 200)}`); - } - } - - private async noContent(call: Call): Promise { - await this.send(call); - } - - private agentPath(path: string): string { - return `${AGENTD_ROUTES.agent}${path}`; - } - - // dot-agentd (5.2) - - health(options: { timeoutMs?: number; signal?: AbortSignal } = {}): Promise { - return this.json({ path: AGENTD_ROUTES.health, ...options }); - } - - system(): Promise { - return this.json({ path: AGENTD_ROUTES.system }); - } - - exec(body: ExecRequest): Promise { - // The command may legitimately run for timeout_ms; give the transport that long plus a margin. - const timeoutMs = body.timeout_ms === undefined ? undefined : body.timeout_ms + this.timeoutMs; - return this.json({ method: "POST", path: AGENTD_ROUTES.exec, body, timeoutMs }); - } - - /** The file's bytes; with `maxBytes`, a larger file is refused (413 `FILE_TOO_LARGE`) without being read in full. */ - async readFile(path: string, options: { maxBytes?: number } = {}): Promise { - return (await this.send({ path: `${AGENTD_ROUTES.files}${filesQuery(path)}`, maxBytes: options.maxBytes })).body; - } - - writeFile(path: string, content: string | Uint8Array): Promise { - const rawBody = typeof content === "string" ? Buffer.from(content, "utf8") : content; - return this.noContent({ method: "PUT", path: `${AGENTD_ROUTES.files}${filesQuery(path)}`, rawBody }); - } - - listFiles(path: string): Promise { - return this.json({ path: `${AGENTD_ROUTES.filesList}${filesQuery(path)}` }); - } - - /** - * Ask the guest to power itself off (`/usr/bin/sudo -n /usr/bin/systemctl poweroff`). The 202 - * comes back before the shutdown begins; QEMU exits when the guest is off, - * which is what the vm-manager then waits for. - */ - powerOff(timeoutMs?: number): Promise { - return this.noContent({ method: "POST", path: AGENTD_ROUTES.poweroff, timeoutMs }); - } - - /** PNG bytes of display :0. */ - async screenshot(): Promise { - return (await this.send({ path: AGENTD_ROUTES.screenshot })).body; - } - - // invisible-dots-agent (5.3), through /v1/agent - - agentHealth(): Promise { - return this.json({ path: this.agentPath(AGENT_ROUTES.health) }); - } - - pushSecrets(secrets: SecretsRequest): Promise { - return this.noContent({ method: "POST", path: this.agentPath(AGENT_ROUTES.secrets), body: secrets }); - } - - putConfig(config: DotRuntimeConfig): Promise { - return this.noContent({ method: "PUT", path: this.agentPath(AGENT_ROUTES.config), body: config }); - } - - postEvent(event: InboundEvent): Promise { - return this.json({ method: "POST", path: this.agentPath(AGENT_ROUTES.events), body: event }); - } - - state(): Promise { - return this.json({ path: this.agentPath(AGENT_ROUTES.state) }); - } - - listBrowserIdentities(): Promise { - return this.json({ path: this.agentPath(AGENT_ROUTES.browserIdentities) }); - } - - createBrowserIdentity(body: CreateBrowserIdentityRequest): Promise { - return this.json({ method: "POST", path: this.agentPath(AGENT_ROUTES.browserIdentities), body }); - } - - getBrowserIdentity(id: string): Promise { - return this.json({ path: this.agentPath(AGENT_ROUTES.browserIdentity(id)) }); - } - - deleteBrowserIdentity(id: string): Promise { - return this.noContent({ method: "DELETE", path: this.agentPath(AGENT_ROUTES.browserIdentity(id)) }); - } - - /** - * JPEG bytes of the open identity's window. The route's contract is a JPEG (architecture section 5.3), and the engine - * is its one owner: it answers `frame_failed` for any other media type, so the host labels the bytes `image/jpeg` - * all the way to the browser without looking again. - */ - async getBrowserIdentityFrame(id: string): Promise { - return (await this.send({ path: this.agentPath(AGENT_ROUTES.browserIdentityFrame(id)) })).body; - } - - closeBrowserIdentity(id: string): Promise { - return this.noContent({ method: "POST", path: this.agentPath(AGENT_ROUTES.browserIdentityClose(id)) }); - } - - /** The Dot's tools and whether the model is offered each now. */ - listTools(): Promise { - return this.json({ path: this.agentPath(AGENT_ROUTES.tools) }); - } - - /** The Dot's skills, the built-in ones and its own, each with its whole file. */ - listSkills(): Promise { - return this.json({ path: this.agentPath(AGENT_ROUTES.skills) }); - } - - /** Flushes state and closes browser sessions; can take a while with several browsers open. */ - prepareSleep(timeoutMs = PREPARE_SLEEP_TIMEOUT_MS): Promise { - return this.noContent({ method: "POST", path: this.agentPath(AGENT_ROUTES.prepareSleep), timeoutMs }); - } - - /** - * The outbound event stream, as an async iterator that survives disconnects: - * on a network error or end of stream it reconnects with `?after=`, so an event is neither lost nor delivered twice. It ends when - * `signal` aborts or the consumer stops iterating. Not retried: a 401 or - * 404 (a wrong token or a wrong guest does not get better by waiting), a - * failed proof, and a refused connection: QEMU listens on the guest port - * for as long as it runs, so nothing listening means the VM is gone and - * the caller has to look at the VM, not at this port. - */ - async *events(options: EventStreamOptions = {}): AsyncGenerator { - let after = options.after ?? 0; - const firstDelay = options.reconnectDelayMs ?? 500; - const maxDelay = options.maxReconnectDelayMs ?? 15_000; - let attempt = 0; - const signal = options.signal; - while (!signal?.aborted) { - const stream = this.openStream(after, signal); - let lastError: Error | undefined; - try { - for await (const item of stream) { - attempt = 0; - if ("refused" in item) { - const { seq } = item.refused; - if (seq !== null && seq <= after) continue; - await options.onRefused?.(item.refused); - if (seq !== null) after = seq; - continue; - } - const event = item.event; - if (event.seq <= after) continue; - after = event.seq; - yield event; - } - lastError = new Error("event stream ended"); - } catch (error) { - if (error instanceof GuestRequestError && (error.status === 401 || error.status === 404)) throw error; - if (isFatalGuestError(error) || (error as { code?: unknown }).code === "ECONNREFUSED") throw error; - lastError = error as Error; - } finally { - await stream.return(undefined); - } - if (signal?.aborted) return; - attempt++; - options.onReconnect?.({ after, attempt, error: lastError }); - this.logger.warn("guest event stream reconnecting", { guest: this.address, after, attempt, reason: lastError.message }); - await abortableSleep(Math.min(firstDelay * 2 ** (attempt - 1), maxDelay), signal); - } - } - - /** One connection to `/events/stream`, parsed into events and the messages that are none; ends when the connection ends. */ - private async *openStream( - after: number, - signal: AbortSignal | undefined, - ): AsyncGenerator<{ event: OutboundEvent } | { refused: RefusedEvent }, void, undefined> { - const path = `${this.agentPath(AGENT_ROUTES.eventsStream)}?after=${after}`; - const route = `GET ${path}`; - await this.ensureProven(signal); - const req = this.open("GET", path, { authorization: `Bearer ${this.token}`, accept: "text/event-stream" }, signal); - try { - const response = await new Promise((resolve, reject) => { - req.once("response", resolve); - req.once("error", (error: NodeJS.ErrnoException) => - reject(new GuestRequestError(route, 0, `${this.address}: ${error.message}`, error.code)), - ); - req.end(); - }); - if ((response.statusCode ?? 0) !== 200) { - const chunks: Buffer[] = []; - for await (const chunk of response) chunks.push(chunk as Buffer); - throw parseError(route, { status: response.statusCode ?? 0, body: Buffer.concat(chunks) }); - } - response.setEncoding("utf8"); - const parser = new SseParser(); - for await (const chunk of response) { - for (const message of parser.feed(chunk as string)) { - if (message.data === "") continue; - let event: OutboundEvent; - try { - event = parseOutboundEvent(JSON.parse(message.data)); - } catch (error) { - // One malformed message must not stall the stream forever; it is logged, handed to the consumer to be - // recorded, and skipped. - this.logger.error("guest sent an invalid event, skipped", { id: message.id, error: (error as Error).message }); - yield { refused: refusedOf(message.data, error as Error) }; - continue; - } - yield { event }; - } - } - } finally { - req.destroy(); - } - } -} - -/** What a refused message is called: the `seq` and `type` it carried, when it was JSON that carried them, and why it was refused. */ -function refusedOf(data: string, error: Error): RefusedEvent { - let seq: number | null = null; - let type: string | null = null; - try { - const raw = JSON.parse(data) as { seq?: unknown; type?: unknown }; - if (typeof raw.seq === "number" && Number.isSafeInteger(raw.seq) && raw.seq > 0) seq = raw.seq; - if (typeof raw.type === "string") type = raw.type.slice(0, 64); - } catch { - // Not JSON: nothing of it can be named. - } - const problem = error.message.length > REFUSED_PROBLEM_MAX ? `${error.message.slice(0, REFUSED_PROBLEM_MAX - 3)}...` : error.message; - return { seq, type, problem }; -} - -export type WaitForGuestHealthOptions = PollGuestHealthOptions; - -export { GuestHealthTimeoutError }; - -/** - * Poll `GET /v1/health` until the guest is up: the shared rule and loop of - * `pollGuestHealth`, over this client. The first request of a client asks - * the listener for its proof, so a port taken over by another process fails - * at once, without the token ever being sent to it. - */ -export function waitForGuestHealth(client: GuestClient, options: WaitForGuestHealthOptions = {}): Promise { - return pollGuestHealth(client, options); -} diff --git a/apps/vm-manager/src/host-doctor.ts b/apps/vm-manager/src/host-doctor.ts deleted file mode 100644 index 78c76880..00000000 --- a/apps/vm-manager/src/host-doctor.ts +++ /dev/null @@ -1,78 +0,0 @@ -/** - * The doctor's dependencies on the real machine (architecture section 11.1): - * QEMU discovery, the accelerator, the free disk space and the process - * runner. What needs more than the vm-manager knows, the images' manifests - * and the OpenRouter key, is handed in by the caller: the API process asks - * its own Scheduler, `invisible-dots doctor` asks the running server. - */ -import { stat, statfs } from "node:fs/promises"; -import { basename, dirname } from "node:path"; -import { ENV, type DoctorCheck } from "@invisible-dots/shared"; -import { checkAcceleratorAccess, hostAccessDeps } from "./accelerator-access.js"; -import type { DoctorDeps, FoundQemu } from "./doctor.js"; -import { QemuNotFoundError } from "./errors.js"; -import { accelerator, findQemu, isExecutableFile, QEMU_SYSTEM_NAMES, qemuSearchDirs } from "./host.js"; -import { runProcess } from "./runner.js"; - -/** - * The vm-manager's discovery, with each program reported separately for - * doctor. findQemu stops at the first program it cannot find; the probe - * passed as `exists` remembers qemu-system-x86_64 when it was found and only - * qemu-img is missing, so doctor can still check its version. - */ -export async function locateQemu(env: Record): Promise { - const found = new Set(); - const exists = async (path: string) => { - const ok = await isExecutableFile(path); - if (ok) found.add(path); - return ok; - }; - const configured = Boolean(env[ENV.QEMU_DIR]?.trim()); - try { - const programs = await findQemu(env, exists); - return { ...programs, searched: qemuSearchDirs(env), configured }; - } catch (error) { - if (!(error instanceof QemuNotFoundError)) throw error; - const system = [...found].find((path) => (QEMU_SYSTEM_NAMES as readonly string[]).includes(basename(path))); - return { ...(system ? { system } : {}), searched: [...error.searched], configured }; - } -} - -/** Free space where INVISIBLE_DOTS_HOME is or will be: its nearest existing directory, since doctor creates nothing. */ -export async function freeSpace(home: string): Promise<{ path: string; bytes: number }> { - let path = home; - for (;;) { - const info = await stat(path).catch(() => undefined); - if (info?.isDirectory()) break; - const parent = dirname(path); - if (parent === path) break; - path = parent; - } - const fs = await statfs(path); - return { path, bytes: fs.bavail * fs.bsize }; -} - -export interface HostDoctorOptions { - env: Record; - /** INVISIBLE_DOTS_HOME, as `hostPaths` resolved it. */ - home: string; - /** The golden image and the runtime ISO against their manifests, one row each. */ - images(): Promise; - /** The OpenRouter key row (`openRouterCheck` for a known answer). */ - openRouterKey(): Promise; -} - -export function hostDoctorDeps(options: HostDoctorOptions): DoctorDeps { - return { - nodeVersion: process.versions.node, - // Asked when a check needs it: on an unsupported host it throws, and that check reports it. - accelerator: () => accelerator(), - findQemu: () => locateQemu(options.env), - run: runProcess, - acceleratorAccess: () => checkAcceleratorAccess(hostAccessDeps(options.env)), - home: options.home, - freeSpace: () => freeSpace(options.home), - images: options.images, - openRouterKey: options.openRouterKey, - }; -} diff --git a/apps/vm-manager/src/host.ts b/apps/vm-manager/src/host.ts deleted file mode 100644 index d8a2bc5a..00000000 --- a/apps/vm-manager/src/host.ts +++ /dev/null @@ -1,160 +0,0 @@ -/** - * What the vm-manager needs from the host: where QEMU is and which - * accelerator it uses. `accelerator()` is the only function in the vm-manager - * whose answer depends on the operating system (architecture section 1.1); - * QEMU discovery is the same search on every host over a list of directories. - */ -import { constants } from "node:fs"; -import { access, stat } from "node:fs/promises"; -import { delimiter, isAbsolute, join } from "node:path"; -import { ENV } from "@invisible-dots/shared"; -import { QemuNotFoundError, SETUP_COMMAND } from "./errors.js"; - -export type Accelerator = "kvm" | "whpx"; - -/** - * The QEMU accelerator of this host (section 1.1): KVM on Linux, the Windows - * Hypervisor Platform on Windows. Any other host is unsupported rather than - * given software emulation, which the contract never uses. - */ -export function accelerator(platform: NodeJS.Platform = process.platform): Accelerator { - switch (platform) { - case "linux": - return "kvm"; - case "win32": - return "whpx"; - default: - throw new Error( - `invisible_dots does not run on ${platform}: it needs Linux (KVM) or Windows (Windows Hypervisor Platform) on x86-64 ` + - `(architecture section 1.1). Run "${SETUP_COMMAND}" on a supported host.`, - ); - } -} - -/** - * The official Windows installer installs into `$PROGRAMFILES64\qemu` (its - * InstallDir). virtualization/qemu/windows.json records the same - * subdirectory and a test keeps the two equal. - */ -export const OFFICIAL_QEMU_SUBDIR = "qemu"; - -/** - * Where the official Windows installer puts QEMU on this host by default: - * `%ProgramW6432%\qemu` (`%ProgramFiles%\qemu` for a 64-bit process, which - * Node is), so a Program Files on another drive is found too. Read from the - * variables Windows sets, which Linux does not have: there is no such - * directory there and nothing to check the platform for. - */ -export function officialQemuDir(env: Record = process.env): string | undefined { - const programFiles = (env.ProgramW6432 ?? env.PROGRAMW6432 ?? env.ProgramFiles ?? env.PROGRAMFILES)?.trim(); - return programFiles ? join(programFiles, OFFICIAL_QEMU_SUBDIR) : undefined; -} - -/** - * Executable names tried in every directory. Windows executables carry - * ".exe" and Linux ones do not; trying both everywhere keeps the search - * identical on both hosts. - */ -export const QEMU_SYSTEM_NAMES = ["qemu-system-x86_64", "qemu-system-x86_64.exe"] as const; -export const QEMU_IMG_NAMES = ["qemu-img", "qemu-img.exe"] as const; - -/** - * The directories searched for QEMU, in the order of section 3.1: when - * INVISIBLE_DOTS_QEMU_DIR is set, that directory alone (mixing a configured - * QEMU with another one would run a qemu-img of a different version on the - * same disks); otherwise the official installer's directory, then every - * PATH entry. The installer's directory comes first so the QEMU that `setup` - * just installed wins over an older one somewhere on PATH. Relative entries - * are dropped, because QEMU is always invoked by absolute path. - */ -export function qemuSearchDirs(env: Record = process.env): string[] { - const configured = env[ENV.QEMU_DIR]?.trim(); - if (configured) return [configured]; - const dirs: string[] = []; - const official = officialQemuDir(env); - if (official) dirs.push(official); - // Windows spells the variable "Path"; Node's process.env hides that, a plain object does not. - const pathValue = env.PATH ?? env.Path ?? ""; - for (const entry of pathValue.split(delimiter)) { - // cmd.exe users sometimes quote PATH entries that contain spaces. - const dir = entry.trim().replace(/^"(.*)"$/, "$1"); - if (dir) dirs.push(dir); - } - const seen = new Set(); - return dirs.filter((dir) => { - if (!isAbsolute(dir) || seen.has(dir)) return false; - seen.add(dir); - return true; - }); -} - -export interface QemuInstallation { - /** Absolute path of qemu-system-x86_64. */ - system: string; - /** Absolute path of qemu-img. */ - img: string; -} - -/** Whether `path` is a file this user may execute. Doctor's own search uses it too. */ -export async function isExecutableFile(path: string): Promise { - try { - // X_OK is checked as existence on Windows, where executability is the extension. - await access(path, constants.X_OK); - // A directory passes X_OK on Linux; it is not a program. - return (await stat(path)).isFile(); - } catch { - return false; - } -} - -async function findIn(dirs: readonly string[], names: readonly string[], exists: (path: string) => Promise): Promise { - for (const dir of dirs) { - for (const name of names) { - const candidate = join(dir, name); - if (await exists(candidate)) return candidate; - } - } - return undefined; -} - -/** Find qemu-system-x86_64 and qemu-img in the directories of `qemuSearchDirs`. */ -export async function findQemu( - env: Record = process.env, - exists: (path: string) => Promise = isExecutableFile, -): Promise { - const configured = env[ENV.QEMU_DIR]?.trim(); - const dirs = qemuSearchDirs(env); - if (configured && !isAbsolute(configured)) { - throw new QemuNotFoundError(`qemu-system-x86_64 (${ENV.QEMU_DIR}="${configured}" is not an absolute path)`, []); - } - const system = await findIn(dirs, QEMU_SYSTEM_NAMES, exists); - if (!system) throw new QemuNotFoundError("qemu-system-x86_64", dirs); - const img = await findIn(dirs, QEMU_IMG_NAMES, exists); - if (!img) throw new QemuNotFoundError("qemu-img", dirs); - return { system, img }; -} - -/** - * The oldest QEMU the contract supports (section 3.1): 8.2, the version - * Ubuntu 24.04 packages, measured with the argv of section 3.4 and doctor's - * probe. This is the one place the number lives; messages use the text below. - */ -export const MIN_QEMU_VERSION = { major: 8, minor: 2 } as const; -export const MIN_QEMU_VERSION_TEXT = `${MIN_QEMU_VERSION.major}.${MIN_QEMU_VERSION.minor}`; - -export interface QemuVersion { - major: number; - minor: number; - micro: number; -} - -/** Parse `qemu-system-x86_64 --version` or `qemu-img --version` output. */ -export function parseQemuVersion(output: string): QemuVersion | undefined { - const match = /version (\d+)\.(\d+)(?:\.(\d+))?/.exec(output); - if (!match) return undefined; - return { major: Number(match[1]), minor: Number(match[2]), micro: Number(match[3] ?? 0) }; -} - -export function isSupportedQemuVersion(version: QemuVersion): boolean { - return version.major > MIN_QEMU_VERSION.major || (version.major === MIN_QEMU_VERSION.major && version.minor >= MIN_QEMU_VERSION.minor); -} diff --git a/apps/vm-manager/src/index.ts b/apps/vm-manager/src/index.ts deleted file mode 100644 index c130d831..00000000 --- a/apps/vm-manager/src/index.ts +++ /dev/null @@ -1,85 +0,0 @@ -/** - * @invisible-dots/vm-manager: the QEMU driver of the control plane - * (architecture sections 3, 5.1, 9.4, 9.5). - * - * The control plane uses one VmManager per process: - * - * const vm = new VmManager({ env, logger, virtualizationDir }); - * - * Every call that needs more than a dot id takes a VmSpec (dotId, token in - * clear, absolute goldenImage and runtimeImage paths, cpus, memoryMiB, - * diskBytes); `vmSpecFromConfig(dotId, config, stored)` builds one. The - * manager stores nothing itself: a VM is its directory under - * INVISIBLE_DOTS_HOME/vms// plus the QEMU process. - * - * Lifecycle - * create(spec) qcow2 overlay on the golden image + seed.iso. Keeps an - * existing disk, so a retried create is safe. - * -> { vmDir, diskPath, seedPath, diskCreated, instanceId } - * start(spec) rewrites the seed, picks a free guest port, spawns QEMU - * detached with the argv of `qemuArgs()`, writes the pid - * file, returns when QEMU listens on the guest port and - * still runs a moment later. Retries on a taken port. - * -> { pid, guestPort, alreadyRunning }; store pid and - * guestPort in `computers`. - * waitForGuestHealth(dotId, guestPort, token, options?) - * polls GET /v1/health until dot-agentd and the agent - * report ok; fails early if QEMU exits, and with the - * end of QEMU's log and the serial console when the - * guest never comes up. - * stop(dotId, token, { timeoutMs? }) - * POST /v1/system/poweroff through the guest channel, - * waits for QEMU to exit, kills it after 60 s. - * -> { forced, wasRunning }. Call the agent's - * prepare-sleep first when sleeping a Dot (9.5). - * reboot(spec) stop then start, so new cpus, memory and runtime ISO - * apply. -> StartVmResult. - * destroy(dotId) kills QEMU and removes the VM directory and its log. - * state(dotId) -> { state: VmState, pid, guestPort, detail? } from the - * pid file and the process; STOPPED with no live QEMU. - * This is also the reconciliation after a control plane - * restart: the scheduler asks it for every Dot, and a - * pid file whose QEMU is gone is removed right there. - * resize(spec) cpus and memory apply at the next start; the disk is - * grown (never shrunk) while stopped. - * -> { restartRequired, diskResized } - * resizeDisk(dotId, bytes) - * guestClient(guestPort, token) - * a GuestClient for every route of sections 5.2 and 5.3, - * including the resumable event stream `events({ after })`. - * createSnapshot / listSnapshots / restoreSnapshot / deleteSnapshot - * reject with NotImplementedError (section 10). - * - * A process is killed only when it is proven to be the Dot's QEMU: this - * process spawned it and has not seen it exit, or its pid is alive, runs as - * this user, was spawned in this boot of the host, and its guest port - * listens (see `processIsOurs` in vm-manager.ts). - * - * Errors name their fix: QemuNotFoundError and AcceleratorUnavailableError - * point at "invisible-dots setup" and "invisible-dots doctor"; CpuModelError - * is QEMU refusing the Dots' CPU model (CPU_MODEL); VmStartError carries QEMU's own output (and the - * serial console when the guest never came up); VmStateError is an operation - * the VM's current state does not allow. - * - * Host helpers for doctor and setup: findQemu, qemuSearchDirs, - * parseQemuVersion, isSupportedQemuVersion, MIN_QEMU_VERSION, accelerator - * (the one platform function of section 1.1) and officialQemuDir. - * The doctor report (section 11.1) is `runDoctor(deps)`, with the real - * machine's deps from `hostDoctorDeps` and the accelerator's host side read - * by `checkAcceleratorAccess` (the other platform function). - * The process runner every host command goes through: runProcess, - * startProcess, NodeCommandRunner and NodeProcessControl (runner.ts). - */ -export * from "./accelerator-access.js"; -export * from "./doctor.js"; -export * from "./errors.js"; -export * from "./guest-client.js"; -export * from "./host-doctor.js"; -export * from "./host.js"; -export * from "./ports.js"; -export * from "./powershell.js"; -export * from "./process-file.js"; -export * from "./qemu-args.js"; -export * from "./runner.js"; -export * from "./seed.js"; -export * from "./vm-manager.js"; diff --git a/apps/vm-manager/src/ports.ts b/apps/vm-manager/src/ports.ts deleted file mode 100644 index 1d36bfee..00000000 --- a/apps/vm-manager/src/ports.ts +++ /dev/null @@ -1,47 +0,0 @@ -/** - * The guest port of section 3.5: bind port 0 on 127.0.0.1, read the port the - * kernel chose, release it and hand it to QEMU. Another process can take it in - * between, which is why the start retries when QEMU reports the forward could - * not be set up. Asking the kernel also keeps clear of ports Windows reserves - * (Hyper-V's excluded ranges), which a fixed or random choice could hit. - */ -import { connect, createServer } from "node:net"; - -export const GUEST_PORT_HOST = "127.0.0.1"; - -export function pickFreePort(host: string = GUEST_PORT_HOST): Promise { - return new Promise((resolve, reject) => { - const server = createServer(); - server.unref(); - server.once("error", reject); - server.listen({ host, port: 0, exclusive: true }, () => { - const address = server.address(); - const port = typeof address === "object" && address ? address.port : 0; - server.close(() => (port > 0 ? resolve(port) : reject(new Error(`could not read the port bound on ${host}`)))); - }); - }); -} - -/** - * Whether something accepts TCP connections on 127.0.0.1:`port`. QEMU's user - * networking listens on a Dot's guest port from the moment its netdev is set - * up until the process exits, whether or not the guest inside is up, and the - * kernel completes the handshake for a listening socket. So "the recorded pid - * is alive AND its recorded guest port listens" is what tells our QEMU from a - * process that inherited a recycled pid, the same way on both hosts. - */ -export function portListens(port: number, host: string = GUEST_PORT_HOST, timeoutMs = 2_000): Promise { - return new Promise((resolve) => { - const socket = connect({ host, port }); - let settled = false; - const done = (listens: boolean) => { - if (settled) return; - settled = true; - socket.destroy(); - resolve(listens); - }; - socket.setTimeout(timeoutMs, () => done(false)); - socket.once("connect", () => done(true)); - socket.once("error", () => done(false)); - }); -} diff --git a/apps/vm-manager/src/powershell.ts b/apps/vm-manager/src/powershell.ts deleted file mode 100644 index 498639fa..00000000 --- a/apps/vm-manager/src/powershell.ts +++ /dev/null @@ -1,33 +0,0 @@ -/** - * The Windows PowerShell the host runs to read what only Windows can say - * (architecture section 1.1): the state of the HypervisorPlatform feature, - * and the one program that runs it. Scripts travel as -EncodedCommand - * (base64 of UTF-16LE), never as a file: nothing on disk can be swapped - * between generating a script and running it, and no argument needs quoting - * rules beyond PowerShell's own. `invisible-dots setup` builds its elevated - * scripts on the same two functions. - */ - -/** The Windows optional feature WHPX needs. */ -export const HYPERVISOR_PLATFORM_FEATURE = "HypervisorPlatform"; - -/** - * Read without administrator rights, unlike Get-WindowsOptionalFeature. - * InstallState: 1 enabled, 2 disabled, 3 absent; no output when Windows does - * not list the feature at all. - */ -export const HYPERVISOR_PLATFORM_STATE_SCRIPT = `(Get-CimInstance -ClassName Win32_OptionalFeature -Filter "Name='${HYPERVISOR_PLATFORM_FEATURE}'").InstallState`; - -export function encodeCommand(script: string): string { - return Buffer.from(script, "utf16le").toString("base64"); -} - -/** Windows PowerShell 5.1 by absolute path: it ships with every Windows, and PATH could point anywhere. */ -export function powershellPath(env: Record): string { - const systemRoot = env.SystemRoot ?? env.SYSTEMROOT ?? "C:\\Windows"; - return `${systemRoot}\\System32\\WindowsPowerShell\\v1.0\\powershell.exe`; -} - -export function powershellArgs(script: string): string[] { - return ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-EncodedCommand", encodeCommand(script)]; -} diff --git a/apps/vm-manager/src/process-file.ts b/apps/vm-manager/src/process-file.ts deleted file mode 100644 index c6f53690..00000000 --- a/apps/vm-manager/src/process-file.ts +++ /dev/null @@ -1,51 +0,0 @@ -/** - * The pid file of a Dot's VM (architecture section 3.2): what lets a control plane that restarted find the QEMU it - * left running. The VmManager decides from it whether that QEMU still runs and may be killed (`processIsOurs`). - */ -import { readFile, writeFile } from "node:fs/promises"; -import { replaceFile } from "@invisible-dots/shared"; - -/** - * The QEMU process of a Dot, the guest port it forwards, and the host's uptime when it was spawned. A host uptime - * lower than that one means the host restarted since, so the process is gone whatever now runs under that pid. - */ -export interface ProcessFile { - pid: number; - guest_port: number; - host_uptime_s?: number; -} - -export function parseProcessFile(text: string): ProcessFile | undefined { - try { - const value = JSON.parse(text) as Partial; - const pid = value.pid; - const port = value.guest_port; - if (Number.isSafeInteger(pid) && pid! > 0 && Number.isInteger(port) && port! >= 1 && port! <= 65535) { - const record: ProcessFile = { pid: pid!, guest_port: port! }; - if (typeof value.host_uptime_s === "number" && Number.isFinite(value.host_uptime_s)) record.host_uptime_s = value.host_uptime_s; - return record; - } - } catch { - // Falls through: not JSON is as unreadable as JSON of the wrong shape. - } - return undefined; -} - -/** The pid file at `path`: undefined when there is none, "unreadable" when it is not one this code wrote. */ -export async function readProcessFile(path: string): Promise { - let text: string; - try { - text = await readFile(path, "utf8"); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return undefined; - throw error; - } - return parseProcessFile(text) ?? "unreadable"; -} - -/** Written whole or not at all (a temporary file renamed over it), so a reader never sees half a pid file. */ -export async function writeProcessFile(path: string, record: ProcessFile): Promise { - const temporary = `${path}.tmp`; - await writeFile(temporary, `${JSON.stringify(record)}\n`, { mode: 0o600 }); - await replaceFile(temporary, path); -} diff --git a/apps/vm-manager/src/qemu-args.ts b/apps/vm-manager/src/qemu-args.ts deleted file mode 100644 index f81caf30..00000000 --- a/apps/vm-manager/src/qemu-args.ts +++ /dev/null @@ -1,169 +0,0 @@ -/** - * The QEMU command line of a Dot's VM (architecture section 3.4). This is the - * one function that builds it, on every host; only the accelerator differs, - * and it arrives here as a value from host.ts. - */ -import { CONFIG_BOUNDS, GUEST_PORT, parseSizeMiB, vmName } from "@invisible-dots/shared"; -import { VmManagerError } from "./errors.js"; -import type { Accelerator } from "./host.js"; - -export interface QemuArgsSpec { - dotId: string; - accelerator: Accelerator; - cpus: number; - memoryMiB: number; - /** All paths absolute: QEMU runs detached and resolves nothing against the control plane's directory. */ - diskPath: string; - seedPath: string; - runtimeIsoPath: string; - /** The free TCP port on 127.0.0.1 forwarded to dot-agentd (section 3.5). */ - guestPort: number; - serialLogPath: string; -} - -/** - * Why QEMU cannot be given this path, or undefined when it can. The same - * rule on every host: - * - QEMU's option syntax gives "," a meaning, and the escape for it (",,") - * applies to -drive but not to -serial file:, so no one spelling of a - * path with a comma is right everywhere; - * - a character outside printable ASCII: measured with the pinned Windows - * build (QEMU 11.1), qemu-system-x86_64 cannot open a -drive file whose - * path holds any (a Latin-1 "é" as much as Cyrillic), and qemu-img cannot - * create one outside the system code page. Linux would take UTF-8, but a - * home that works on one host and not on the other is the divergence - * section 1.1 rules out, and the default home sits under the account name. - */ -export function qemuPathProblem(path: string): string | undefined { - if (path.includes("\n") || path.includes("\r")) return "contains a line break"; - if (path.includes(",")) return "contains a comma, which QEMU's command line cannot carry"; - if (/[^\x20-\x7e]/.test(path)) return "contains a character outside plain ASCII, which QEMU on Windows cannot open"; - return undefined; -} - -/** A host path for QEMU's command line, refused with the fix when qemuPathProblem names one. */ -export function qemuPathArg(label: string, path: string): string { - const problem = qemuPathProblem(path); - if (problem) { - throw new VmManagerError(`${label} ${JSON.stringify(path)} ${problem}; set INVISIBLE_DOTS_HOME to a directory whose path is plain ASCII without commas`); - } - return path; -} - -function integerInRange(label: string, value: number, min: number, max: number): number { - if (!Number.isInteger(value) || value < min || value > max) { - throw new VmManagerError(`${label} must be an integer from ${min} to ${max}, got ${value}`); - } - return value; -} - -/** - * The cpus and memory every VM of this project may have, Dots and the image builder's VM alike: the range a Dot's - * config allows (CONFIG_BOUNDS of packages/shared, which owns it), in the units QEMU takes. - */ -export const VM_RESOURCE_BOUNDS = { - cpus: { min: CONFIG_BOUNDS.cpu.min, max: CONFIG_BOUNDS.cpu.max }, - memoryMiB: { min: parseSizeMiB(CONFIG_BOUNDS.memory.min), max: parseSizeMiB(CONFIG_BOUNDS.memory.max) }, -} as const; - -/** Refuses cpus or memory outside VM_RESOURCE_BOUNDS, naming the value. */ -export function checkVmResources(spec: { cpus: number; memoryMiB: number }): void { - integerInRange("cpus", spec.cpus, VM_RESOURCE_BOUNDS.cpus.min, VM_RESOURCE_BOUNDS.cpus.max); - integerInRange("memoryMiB", spec.memoryMiB, VM_RESOURCE_BOUNDS.memoryMiB.min, VM_RESOURCE_BOUNDS.memoryMiB.max); -} - -/** - * The CPU model of every VM of this project (section 3.4): the host's own - * CPU, without the virtualization extensions. A Dot never runs a hypervisor, - * and under WHPX a guest that sees VMX stops: measured with QEMU 11.1 on - * Windows 11 (Intel Core Ultra 7 255H, the hypervisor running), `-cpu host` - * and `-cpu max` pause the VM at its first firmware instructions with - * `WHPX: Unexpected VP exit code 4` (WHvRunVpExitReasonUnrecoverableException), - * while `-cpu host,-vmx` boots the Ubuntu cloud image to its login prompt and - * so does every named model. No other feature of the host CPU matters - * (bisected). Public reports give the same remedy (`vmx=off`). `-svm` is the - * AMD spelling of the same extension; QEMU accepts both names on every host. - * One value on every accelerator, because section 1.1 keeps one command line. - */ -export const CPU_MODEL = "host,-vmx,-svm"; - -/** - * The machine every VM of this project runs on, Dots and the image - * builder's VM alike, so a golden image is provisioned on the machine type, - * accelerator and CPU model a Dot later boots it on: changing one here - * changes both. - */ -export function machineArgs(spec: { accelerator: Accelerator; cpus: number; memoryMiB: number }): string[] { - checkVmResources(spec); - return [ - "-machine", - "q35", - "-accel", - spec.accelerator, - "-cpu", - CPU_MODEL, - "-smp", - String(spec.cpus), - "-m", - String(spec.memoryMiB), - ]; -} - -/** The `-drive` value of a VM's qcow2 system disk. */ -export function diskDriveArg(path: string): string { - return `if=virtio,file=${qemuPathArg("disk", path)},format=qcow2,discard=unmap`; -} - -/** - * The `-drive` value of a read-only ISO (a seed or the runtime ISO), as a virtio disk: every Ubuntu cloud kernel has - * virtio-blk built in, where a CD-ROM needs the SATA controller's and the CD's modules, which the minimal cloud image - * does not load (measured: its builder VM never saw the seed, so cloud-init never ran). The guest finds both by their - * labels (cidata, IDOTS-RT), whatever device they are. - * format=raw: an ISO is raw, and naming it stops QEMU probing the format. - */ -export function isoDriveArg(label: string, path: string): string { - return `if=virtio,file=${qemuPathArg(label, path)},format=raw,readonly=on`; -} - -/** - * User-mode networking (`netdev` says whether a port is forwarded), the - * random number device, the serial console to a file and no display: the - * devices every VM of this project has. - */ -export function deviceArgs(netdev: string, serialLogPath: string): string[] { - return [ - "-netdev", - netdev, - "-device", - "virtio-net-pci,netdev=net0", - "-device", - "virtio-rng-pci", - "-serial", - `file:${qemuPathArg("serial log", serialLogPath)}`, - "-display", - "none", - ]; -} - -/** - * The argument array after the executable, exactly as section 3.4 lists it. - * There is no monitor of any kind (no -qmp, no -monitor) and nothing that - * holds the VM before its first instruction (no -S): the control plane - * reaches a VM only through the guest channel (section 5.1) and sees its - * QEMU only as a process. - */ -export function qemuArgs(spec: QemuArgsSpec): string[] { - const port = integerInRange("guestPort", spec.guestPort, 1, 65535); - return [ - "-name", - vmName(spec.dotId), - ...machineArgs(spec), - "-drive", - diskDriveArg(spec.diskPath), - "-drive", - isoDriveArg("seed ISO", spec.seedPath), - "-drive", - isoDriveArg("runtime ISO", spec.runtimeIsoPath), - ...deviceArgs(`user,id=net0,hostfwd=tcp:127.0.0.1:${port}-:${GUEST_PORT}`, spec.serialLogPath), - ]; -} diff --git a/apps/vm-manager/src/runner.ts b/apps/vm-manager/src/runner.ts deleted file mode 100644 index 7cbe2323..00000000 --- a/apps/vm-manager/src/runner.ts +++ /dev/null @@ -1,222 +0,0 @@ -/** - * The process runners of the host side. Every program the control plane - * and the CLI start goes through this file or the `runProcess` it is built - * on (packages/shared, so the secret files' ACL can use it too): short - * commands (qemu-img, QEMU's --version, PowerShell, sudo apt-get) through - * `runProcess`, a long-running child the caller watches (the image builder's - * VM) through `startProcess`, and a Dot's QEMU, which must outlive the - * control plane, through ProcessControl. All take argument arrays, never a shell, so a dot id or a - * path can never be read as shell syntax, and tests swap the interfaces - * (CommandRunner, ProcessControl) for fakes that record argv. - */ -import { spawn, type ChildProcess } from "node:child_process"; -import { closeSync, openSync } from "node:fs"; -import { stat } from "node:fs/promises"; -import { processPresence, runProcess, type ProcessPresence, type RunOptions, type RunResult } from "@invisible-dots/shared"; -import { CommandError } from "./errors.js"; - -export { runProcess, type RunOptions, type RunResult }; - -export interface CommandResult { - stdout: string; - stderr: string; -} - -export interface CommandRunner { - /** Run a command to completion. Rejects with CommandError unless it exits 0. */ - run(command: string, args: readonly string[], options?: RunOptions): Promise; -} - -export const DEFAULT_COMMAND_TIMEOUT_MS = 60_000; - -/** The real CommandRunner: `runProcess`, with a default timeout and a non-zero exit turned into a CommandError. */ -export class NodeCommandRunner implements CommandRunner { - async run(command: string, args: readonly string[], options: RunOptions = {}): Promise { - const timeoutMs = options.timeoutMs ?? DEFAULT_COMMAND_TIMEOUT_MS; - const result = await runProcess(command, args, { ...options, timeoutMs }); - if (result.code === 0 && !result.timedOut) return { stdout: result.stdout, stderr: result.stderr }; - throw new CommandError({ - command, - args, - exitCode: result.code, - stdout: result.stdout, - stderr: result.stderr, - timedOut: result.timedOut, - code: result.startError?.code, - message: result.timedOut ? `timed out after ${timeoutMs} ms` : undefined, - cause: result.startError, - }); - } -} - -/** A long-running child the caller watches: its exit is reported once, its stderr kept as a short tail. */ -export interface StartedProcess { - readonly pid: number | undefined; - /** The last few KiB of stderr, for error messages when the process dies. */ - stderrTail(): string; - /** Called once when the process exits or fails to start; immediately when that already happened. */ - onExit(listener: (code: number | null, signal: NodeJS.Signals | null, error?: Error) => void): void; - kill(signal?: NodeJS.Signals): void; -} - -const STDERR_TAIL_CHARS = 4096; - -export interface StartProcessOptions { - /** - * The child's working directory. Always given: an inherited one is held - * open by the child for its whole life, and on Windows that blocks moving - * or deleting whatever directory the parent happened to start in. - */ - cwd: string; - /** The child's whole environment (an allowlist, see allowlistedEnvironment). */ - env: Record; -} - -export function startProcess(command: string, args: readonly string[], options: StartProcessOptions): StartedProcess { - let tail = ""; - let exit: { code: number | null; signal: NodeJS.Signals | null; error?: Error } | undefined; - const listeners: ((code: number | null, signal: NodeJS.Signals | null, error?: Error) => void)[] = []; - const settle = (code: number | null, signal: NodeJS.Signals | null, error?: Error) => { - if (exit) return; - exit = error ? { code, signal, error } : { code, signal }; - for (const listener of listeners.splice(0)) listener(code, signal, error); - }; - let child: ChildProcess; - try { - child = spawn(command, [...args], { stdio: ["ignore", "ignore", "pipe"], windowsHide: true, cwd: options.cwd, env: options.env }); - } catch (error) { - settle(null, null, error as Error); - return { pid: undefined, stderrTail: () => "", onExit: (l) => l(null, null, error as Error), kill: () => undefined }; - } - child.stderr?.on("data", (chunk: Buffer) => { - tail = (tail + chunk.toString()).slice(-STDERR_TAIL_CHARS); - }); - child.on("error", (error) => settle(null, null, error)); - child.on("close", (code, signal) => settle(code, signal)); - return { - get pid() { - return child.pid; - }, - stderrTail: () => tail, - onExit(listener) { - if (exit) listener(exit.code, exit.signal, exit.error); - else listeners.push(listener); - }, - kill(signal: NodeJS.Signals = "SIGTERM") { - if (!exit) child.kill(signal); - }, - }; -} - -/** The first non-empty line of a program's error output, for one-line reports. */ -export function firstLine(text: string): string { - return ( - text - .split(/\r?\n/) - .map((line) => line.trim()) - .find((line) => line.length > 0) ?? "" - ); -} - -export interface ProcessExit { - code: number | null; - signal: NodeJS.Signals | null; -} - -/** A process started detached from the control plane. */ -export interface DetachedProcess { - readonly pid: number; - /** - * Settles when the process exits while this control plane still runs. It - * never rejects; after a control plane restart nobody holds this promise - * and liveness is read from the pid instead. - */ - readonly exited: Promise; -} - -export interface SpawnDetachedOptions { - /** stdout and stderr are appended to this file, so QEMU's messages survive the control plane. */ - logPath: string; - /** - * The working directory. A detached QEMU outlives the server, and on - * Windows a process's working directory cannot be moved or deleted while - * it runs, so it is never the server's own (which may be a checkout or a - * removable drive): the caller passes one of its own directories. - */ - cwd: string; - /** The whole environment of the child: an allowlist, never the server's own (it may hold the API token or DATABASE_URL). */ - env: Record; -} - -export interface ProcessControl { - /** Start a process in its own session, not tied to this process's lifetime. Rejects when it cannot be started at all. */ - spawnDetached(command: string, args: readonly string[], options: SpawnDetachedOptions): Promise; - /** - * What `process.kill(pid, 0)` says about the pid, the same on both hosts: - * gone, ours (this user may signal it) or foreign (it exists and belongs - * to another user). A QEMU this control plane spawned is always ours. - */ - presence(pid: number): ProcessPresence; - /** Kill the process outright (SIGKILL; TerminateProcess on Windows). A missing process is not an error. */ - kill(pid: number): void; -} - -/** - * The real ProcessControl. `detached: true` matters on both hosts: on Linux - * it puts QEMU in a new session, so a Ctrl-C to the server does not reach it; - * on Windows it keeps libuv from placing QEMU in the job object that is - * closed, and its processes killed, when the server exits. - */ -export class NodeProcessControl implements ProcessControl { - async spawnDetached(command: string, args: readonly string[], options: SpawnDetachedOptions): Promise { - const fd = openSync(options.logPath, "a", 0o600); - try { - const child = spawn(command, [...args], { - detached: true, - stdio: ["ignore", fd, fd], - windowsHide: true, - cwd: options.cwd, - env: options.env, - }); - const exited = new Promise((resolve) => { - child.once("exit", (code, signal) => resolve({ code, signal })); - child.once("error", () => resolve({ code: null, signal: null })); - }); - await new Promise((resolve, reject) => { - child.once("spawn", resolve); - child.once("error", (error: NodeJS.ErrnoException) => - reject(new CommandError({ command, args, exitCode: null, code: error.code, cause: error })), - ); - }); - // The control plane must be able to exit while QEMU keeps running. - child.unref(); - return { pid: child.pid!, exited }; - } finally { - closeSync(fd); - } - } - - presence(pid: number): ProcessPresence { - return processPresence(pid); - } - - kill(pid: number): void { - try { - process.kill(pid, "SIGKILL"); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; - } - } -} - -export { retryWhileInUse } from "@invisible-dots/shared"; - -/** Size of a file, 0 when it does not exist: where this start's output begins in an appended log. */ -export async function fileSize(path: string): Promise { - try { - return (await stat(path)).size; - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return 0; - throw error; - } -} diff --git a/apps/vm-manager/src/seed.ts b/apps/vm-manager/src/seed.ts deleted file mode 100644 index 39db6758..00000000 --- a/apps/vm-manager/src/seed.ts +++ /dev/null @@ -1,131 +0,0 @@ -/** - * The NoCloud seed of a Dot (architecture sections 3.2 and 4.2): user-data and - * meta-data rendered from virtualization/cloud-init/ and written as an ISO - * labelled "cidata" by packages/iso. The templates on disk are the single - * source of the content; this module fills them in, quoting every value as a - * YAML scalar. - */ -import { createHash } from "node:crypto"; -import { readFile } from "node:fs/promises"; -import { dirname, join, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; -import { writeIso } from "@invisible-dots/iso"; -import type { GuestBootConfig } from "@invisible-dots/shared"; - -/** `virtualization/` of this repository, found relative to this file. */ -export const DEFAULT_VIRTUALIZATION_DIR = resolve(dirname(fileURLToPath(import.meta.url)), "../../../virtualization"); - -/** cloud-init's NoCloud datasource looks for a filesystem with this label. */ -export const SEED_VOLUME_ID = "cidata"; - -/** Volume label of the runtime ISO; the image builder writes it, the seed mounts by it. */ -export const RUNTIME_ISO_LABEL = "IDOTS-RT"; - -/** - * Every date inside the seed image. A fixed value makes the image a pure - * function of its content, so writing the same seed twice gives the same bytes. - */ -export const SEED_TIMESTAMP = new Date("2026-01-01T00:00:00Z"); - -/** - * A YAML double-quoted scalar. JSON string syntax is a subset of YAML's - * double-quoted style, so JSON.stringify is a correct encoder here. - */ -export function yamlScalar(value: string): string { - return JSON.stringify(value); -} - -/** - * Replace each `{{name}}` with `encode(values[name])`. A placeholder without a - * value is an error: an empty value in a seed is a broken VM that cloud-init - * would accept. - */ -export function renderTemplate(template: string, values: Record, encode: (value: string) => string): string { - return template.replace(/\{\{\s*([A-Za-z][A-Za-z0-9_]*)\s*\}\}/g, (_match, name: string) => { - const value = values[name]; - if (value === undefined) throw new Error(`template placeholder "${name}" has no value`); - return encode(String(value)); - }); -} - -/** The guest hostname. Dot ids contain "_", which is not valid in a hostname, so it becomes "-". */ -export function guestHostname(dotId: string): string { - return `invisible-dot-${dotId.replace(/_/g, "-")}`; -} - -export interface SeedTemplates { - userData: string; - metaData: string; -} - -export interface SeedFiles { - userData: string; - metaData: string; - instanceId: string; - /** The digest of the rendered content, which the next seed is compared with. */ - digest: string; -} - -/** The seed written last for a Dot: its content's digest and its instance-id (seed-instance.json beside seed.iso). */ -export interface PreviousSeed { - digest: string; - instanceId: string; -} - -/** - * Render the seed of one Dot. - * - * cloud-init runs its per-instance modules (users, write_files, mounts...) - * once per instance-id it has not run them for. Rewriting the seed the last - * one was (a retried create, a restart) keeps that seed's id and re-runs - * nothing. A seed that differs from the last one (a new token, a VM proxy set - * or cleared, new templates) gets an id cloud-init has never seen: one derived - * from the content alone would come back with the content (a proxy cleared - * gives the first boot's seed again, whose modules ran long ago), and the old - * config.json would stay. So a changed seed's id also covers the id before it. - */ -export function renderSeed(templates: SeedTemplates, dotId: string, token: string, proxy?: string, previous?: PreviousSeed): SeedFiles { - const bootConfig: GuestBootConfig = { dotId, token, ...(proxy ? { proxy } : {}) }; - const hostname = guestHostname(dotId); - const userData = renderTemplate( - templates.userData, - { - hostname, - bootConfigJson: `${JSON.stringify(bootConfig)}\n`, - runtimeDevice: `LABEL=${RUNTIME_ISO_LABEL}`, - }, - yamlScalar, - ); - // The digest also covers the meta-data template, so changing it re-runs cloud-init as well. - const digest = createHash("sha256").update(userData).update("\0").update(templates.metaData).update("\0").update(hostname).digest("hex"); - const instanceId = - previous === undefined - ? `iid-${dotId}-${digest.slice(0, 16)}` - : previous.digest === digest - ? previous.instanceId - : `iid-${dotId}-${digest.slice(0, 16)}-${createHash("sha256").update(previous.instanceId).digest("hex").slice(0, 8)}`; - const metaData = renderTemplate(templates.metaData, { instanceId, hostname }, yamlScalar); - return { userData, metaData, instanceId, digest }; -} - -/** Read the two cloud-init templates from a `virtualization/` directory. */ -export async function loadSeedTemplates(virtualizationDir: string = DEFAULT_VIRTUALIZATION_DIR): Promise { - const read = (path: string) => - readFile(join(virtualizationDir, path), "utf8").catch((error: unknown) => { - throw new Error(`cannot read template ${join(virtualizationDir, path)}: ${(error as Error).message}`, { cause: error }); - }); - const [userData, metaData] = await Promise.all([read("cloud-init/user-data.yaml.tmpl"), read("cloud-init/meta-data.yaml.tmpl")]); - return { userData, metaData }; -} - -/** Write seed.iso (mode 0600: it carries the Dot's token). The file is replaced atomically. */ -export async function writeSeedIso(path: string, seed: SeedFiles): Promise { - await writeIso( - path, - [ - { path: "user-data", data: seed.userData }, - { path: "meta-data", data: seed.metaData }, - ], - { volumeId: SEED_VOLUME_ID, timestamp: SEED_TIMESTAMP, mode: 0o600 }, - ); -} diff --git a/apps/vm-manager/src/vm-manager.ts b/apps/vm-manager/src/vm-manager.ts deleted file mode 100644 index a973fb51..00000000 --- a/apps/vm-manager/src/vm-manager.ts +++ /dev/null @@ -1,863 +0,0 @@ -/** - * The QEMU driver (architecture sections 3, 9.4, 9.5). What a VM is lives in - * its directory (disk, seed, pid file) and in the QEMU process; the control - * plane passes the rest (token, images, resources) on every call that needs - * it. That is what lets the control plane restart while VMs keep running. - * - * There is no monitor: QEMU is seen only as a process (alive or not, and the - * guest port its user networking listens on), and the guest only through the - * guest channel (section 5.1). Both work the same way on Linux and Windows, - * which is why there is nothing else (section 1.1). - * - * The one thing kept in memory is the ChildProcess of every QEMU this - * process spawned: while it is held, its exit is known exactly. After a - * restart the pid file stands in for it, checked as `processIsOurs` says. - */ -import { randomBytes } from "node:crypto"; -import { appendFile, mkdir, open, readFile, rm, writeFile } from "node:fs/promises"; -import { isAbsolute, join, relative, resolve } from "node:path"; -import { - allowlistedEnvironment, - computerResources, - hostPaths, - hostRestartedSince, - hostUptimeSeconds, - isFatalGuestError, - pathExists, - prefixedStderrLogger, - replaceFile, - type DotConfig, - type HostPaths, - type Logger, - type VmState, -} from "@invisible-dots/shared"; -import { - AcceleratorUnavailableError, - CommandError, - CpuModelError, - GuestRequestError, - NotImplementedError, - VmManagerError, - VmStartError, - VmStateError, -} from "./errors.js"; -import { GuestClient, waitForGuestHealth, type GuestClientOptions, type WaitForGuestHealthOptions } from "./guest-client.js"; -import { accelerator as hostAccelerator, findQemu, type Accelerator, type QemuInstallation } from "./host.js"; -import { pickFreePort, portListens } from "./ports.js"; -import { readProcessFile, writeProcessFile, type ProcessFile } from "./process-file.js"; -import { checkVmResources, CPU_MODEL, qemuArgs, qemuPathArg } from "./qemu-args.js"; -import { - fileSize, - NodeCommandRunner, - NodeProcessControl, - retryWhileInUse, - type CommandRunner, - type DetachedProcess, - type ProcessControl, - type ProcessExit, -} from "./runner.js"; -import { DEFAULT_VIRTUALIZATION_DIR, loadSeedTemplates, renderSeed, writeSeedIso, type PreviousSeed, type SeedTemplates } from "./seed.js"; - -/** Everything the vm-manager needs to know about one Dot's computer. */ -export interface VmSpec { - dotId: string; - /** The Dot token, in clear (the control plane decrypts `token_enc`). It goes into the seed. */ - token: string; - /** The Dot's VM proxy in clear, or absent for a direct exit. It goes into the seed. */ - proxy?: string; - /** Absolute path of the golden image the overlay is backed by. */ - goldenImage: string; - /** Absolute path of the runtime ISO attached on every start. */ - runtimeImage: string; - cpus: number; - memoryMiB: number; - /** Size of the overlay at create; resizeDisk grows it later. */ - diskBytes: number; -} - -/** Build a VmSpec from a parsed Dot config plus what the control plane stored. */ -export function vmSpecFromConfig( - dotId: string, - config: Pick, - stored: { token: string; goldenImage: string; runtimeImage: string }, -): VmSpec { - const resources = computerResources(config); - return { dotId, ...stored, cpus: resources.cpus, memoryMiB: resources.memoryMiB, diskBytes: resources.diskBytes }; -} - -/** A new Dot token: 32 random bytes, base64url. It only authorizes requests to that Dot's VM. */ -export function generateToken(): string { - return randomBytes(32).toString("base64url"); -} - -export interface VmStateInfo { - /** RUNNING while this Dot's QEMU process runs, STOPPED when it does not, ERROR when a live pid cannot be identified. */ - state: VmState; - /** The QEMU pid from the pid file, null when no QEMU runs. */ - pid: number | null; - /** The host port forwarded to dot-agentd, from the pid file; null when no QEMU runs. */ - guestPort: number | null; - /** Why the state is ERROR, in words. */ - detail?: string; -} - -export interface CreateVmResult { - vmDir: string; - diskPath: string; - seedPath: string; - /** False when the overlay already existed and was kept (a retried create never wipes a disk). */ - diskCreated: boolean; - /** The cloud-init instance-id written into the seed. */ - instanceId: string; -} - -export interface StartVmResult { - /** QEMU's pid; store it as `computers.pid`. */ - pid: number; - /** The host port forwarded to dot-agentd; store it as `computers.guest_port`. */ - guestPort: number; - /** True when QEMU was already running and nothing was started. */ - alreadyRunning: boolean; -} - -export interface StopVmResult { - /** True when the guest did not power off by itself in time and QEMU was killed. */ - forced: boolean; - /** False when there was no running VM to stop. */ - wasRunning: boolean; -} - -export interface ResizeResult { - /** True when the VM is running and new cpus or memory apply from its next start. */ - restartRequired: boolean; - /** True when the overlay was grown. */ - diskResized: boolean; -} - -export interface VmManagerOptions { - /** Environment for INVISIBLE_DOTS_HOME and INVISIBLE_DOTS_QEMU_DIR. Default process.env. */ - env?: Record; - /** Host paths; default `hostPaths(env)`. */ - paths?: HostPaths; - logger?: Logger; - /** Where the cloud-init templates live. Default the repository's `virtualization/`. */ - virtualizationDir?: string; - /** QEMU binaries; default found by `findQemu(env)` on first use. */ - qemu?: QemuInstallation; - /** The accelerator; default `accelerator()` of this host. */ - accelerator?: Accelerator; - /** Runs qemu-img. */ - runner?: CommandRunner; - /** Starts, probes and kills QEMU. */ - processes?: ProcessControl; - /** Picks the guest port; default a kernel-chosen free port on 127.0.0.1. */ - pickPort?: () => Promise; - /** How long a stop waits for the guest to power off before QEMU is killed (sections 3.4 and 9.5). Default 60 s. */ - shutdownTimeoutMs?: number; - /** How long QEMU gets to disappear after it was killed. Default 10 s. */ - killTimeoutMs?: number; - /** Timeout of the poweroff request itself. Default 10 s. */ - powerOffRequestTimeoutMs?: number; - /** How long a start waits for QEMU to set up the guest port forward. Default 30 s. */ - startTimeoutMs?: number; - /** - * Once the forward is up, QEMU must still be running this much later: it - * builds the machine (and checks the CPU model) after it sets up networking, - * so a refused CPU model ends the process just after the port listens. - * Default 1 s. - */ - startSettleMs?: number; - /** Ports tried when QEMU cannot bind the forward (section 3.5). Default 5. */ - maxPortAttempts?: number; - /** How often liveness is polled while waiting. Default 500 ms. */ - pollIntervalMs?: number; - /** Timeout of qemu-img commands. Default 5 minutes. */ - qemuImgTimeoutMs?: number; - sleep?: (ms: number) => Promise; -} - -/** Dot ids become directory names, QEMU `-name` values and parts of hostnames. */ -const DOT_ID_PATTERN = /^[a-z0-9][a-z0-9_-]{0,63}$/; - -function assertDotId(dotId: string): void { - if (!DOT_ID_PATTERN.test(dotId)) { - throw new VmManagerError(`invalid dot id "${dotId}": expected lowercase letters, digits, "_" and "-", at most 64 characters`); - } -} - -/** QEMU wrote this when it could not bind the host side of the port forward. */ -const PORT_TAKEN = /Could not set up host forwarding rule/i; - -/** What QEMU prints when the accelerator itself is unusable, on either host. */ -const ACCELERATOR_FAILURES = [ - /Could not access KVM kernel module/i, - /failed to initialize kvm/i, - /\/dev\/kvm/i, - /WHPX: No accelerator found/i, - /failed to initialize whpx/i, - /WHPX: Failed to/i, - /invalid accelerator/i, - /no accelerator found/i, - /accelerator .* not (?:found|available|supported)/i, -]; - -const CPU_MODEL_FAILURES = [/CPU model 'host' requires/i, /unable to find CPU model 'host'/i]; - -/** How much of the end of a log goes into an error: enough for QEMU's last words and the guest's last boot lines. */ -const LOG_TAIL_BYTES = 4 * 1024; - -/** seed-instance.json, or undefined for one this code did not write: the next seed then takes the content's own id. */ -function parsePreviousSeed(text: string): PreviousSeed | undefined { - try { - const value = JSON.parse(text) as Partial; - if (typeof value.digest === "string" && typeof value.instanceId === "string") return { digest: value.digest, instanceId: value.instanceId }; - } catch { - // Not JSON: as if there were none. - } - return undefined; -} - -/** A QEMU this process spawned: its exit is set the moment Node reports it. */ -interface OwnedProcess { - pid: number; - exited: Promise; - exit?: ProcessExit; -} - -interface Probe extends VmStateInfo { - /** The pid file, when there is one that can be read. */ - record?: ProcessFile; - /** Whether the live process is proven to be this Dot's QEMU, so it may be killed. */ - verified: boolean; -} - -const STOPPED: Probe = { state: "STOPPED", pid: null, guestPort: null, verified: false }; - -export class VmManager { - readonly paths: HostPaths; - private readonly env: Record; - private readonly logger: Logger; - private readonly options: VmManagerOptions; - private readonly runner: CommandRunner; - private readonly processes: ProcessControl; - private readonly sleep: (ms: number) => Promise; - private qemuPromise: Promise | undefined; - private templates: Promise | undefined; - private readonly locks = new Map>(); - private readonly owned = new Map(); - - constructor(options: VmManagerOptions = {}) { - this.options = options; - this.env = options.env ?? process.env; - this.paths = options.paths ?? hostPaths(this.env); - this.logger = options.logger ?? prefixedStderrLogger("vm-manager"); - this.runner = options.runner ?? new NodeCommandRunner(); - this.processes = options.processes ?? new NodeProcessControl(); - this.sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))); - if (options.qemu) this.qemuPromise = Promise.resolve(options.qemu); - } - - /** The QEMU binaries this manager runs; throws QemuNotFoundError naming the fix. */ - qemu(): Promise { - this.qemuPromise ??= findQemu(this.env).catch((error: unknown) => { - // Not cached: `invisible-dots setup` may install QEMU while the server runs. - this.qemuPromise = undefined; - throw error; - }); - return this.qemuPromise; - } - - /** The accelerator every VM of this host starts with. */ - get accelerator(): Accelerator { - return this.options.accelerator ?? hostAccelerator(); - } - - generateToken(): string { - return generateToken(); - } - - /** A client for every guest route of a running Dot. */ - guestClient(guestPort: number, token: string, options: GuestClientOptions = {}): GuestClient { - return new GuestClient(guestPort, token, { logger: this.logger, ...options }); - } - - /** - * Wait until the guest answers `GET /v1/health` (section 9.4). Fails early - * when QEMU exits meanwhile instead of waiting out the whole timeout. A - * guest that never comes up is reported with the end of QEMU's log and of - * the serial console, because a VM that QEMU runs but that never boots - * (a vCPU the accelerator stopped, a guest stuck in its firmware) shows - * nothing anywhere else. A refused token (401) is passed on as it is. - */ - async waitForGuestHealth(dotId: string, guestPort: number, token: string, options: WaitForGuestHealthOptions = {}) { - assertDotId(dotId); - const client = this.guestClient(guestPort, token); - try { - return await waitForGuestHealth(client, { - ...options, - check: async () => { - await options.check?.(); - if (!(await this.qemuRuns(dotId))) throw new VmStateError(dotId, "QEMU is not running any more"); - }, - }); - } catch (error) { - if (isFatalGuestError(error) || options.signal?.aborted) throw error; - throw await this.guestNotUp(dotId, error as Error); - } - } - - private async guestNotUp(dotId: string, cause: Error): Promise { - const qemuLog = this.paths.qemuLogPath(dotId); - const serialLog = this.paths.serialLogPath(dotId); - const [qemuOutput, serialOutput] = await Promise.all([this.readTail(qemuLog), this.readTail(serialLog)]); - return new VmStartError(dotId, `the guest did not come up (${cause.message}); QEMU's log is ${qemuLog}, the serial console ${serialLog}`, qemuOutput, { - cause, - serialOutput, - }); - } - - private loadTemplates(): Promise { - this.templates ??= loadSeedTemplates(this.options.virtualizationDir ?? DEFAULT_VIRTUALIZATION_DIR).catch((error: unknown) => { - this.templates = undefined; - throw error; - }); - return this.templates; - } - - /** Run `fn` with no other operation on the same Dot in flight in this process. */ - private async withLock(dotId: string, fn: () => Promise): Promise { - const previous = this.locks.get(dotId) ?? Promise.resolve(); - const run = previous.catch(() => undefined).then(fn); - const tail = run.catch(() => undefined); - this.locks.set(dotId, tail); - try { - return await run; - } finally { - if (this.locks.get(dotId) === tail) this.locks.delete(dotId); - } - } - - private vmDir(dotId: string): string { - const dir = resolve(this.paths.vmDir(dotId)); - const rel = relative(resolve(this.paths.vmsDir), dir); - // Belt and braces next to assertDotId: this directory is removed recursively on destroy. - if (rel === "" || rel.startsWith("..") || isAbsolute(rel)) { - throw new VmManagerError(`refusing VM directory ${dir}: it is not inside ${this.paths.vmsDir}`); - } - return dir; - } - - private validateSpec(spec: VmSpec): void { - assertDotId(spec.dotId); - checkVmResources(spec); - if (!Number.isSafeInteger(spec.diskBytes) || spec.diskBytes <= 0) { - throw new VmManagerError(`diskBytes must be a positive integer, got ${spec.diskBytes}`); - } - if (!spec.token) throw new VmManagerError("the Dot token is empty"); - if (!isAbsolute(spec.goldenImage) || !isAbsolute(spec.runtimeImage)) { - throw new VmManagerError("goldenImage and runtimeImage must be absolute paths: QEMU runs detached and qemu-img records the backing path"); - } - } - - /** Both images of the spec exist: a VM is never created or started on a path that leads nowhere. */ - private async assertImages(spec: VmSpec): Promise { - for (const [label, path] of [ - ["golden image", spec.goldenImage], - ["runtime ISO", spec.runtimeImage], - ] as const) { - if (!(await pathExists(path))) throw new VmManagerError(`${label} ${path} does not exist: build it with "invisible-dots image build" first`); - } - } - - private async qemuImg(args: string[]) { - const { img } = await this.qemu(); - return this.runner.run(img, args, { timeoutMs: this.options.qemuImgTimeoutMs ?? 300_000 }); - } - - private async writeSeed(spec: VmSpec): Promise { - // The seed written last decides the id of this one (renderSeed): the same content keeps it, any other gets a new one. - const record = join(this.paths.vmDir(spec.dotId), "seed-instance.json"); - const previous = await readFile(record, "utf8").then(parsePreviousSeed, () => undefined); - const seed = renderSeed(await this.loadTemplates(), spec.dotId, spec.token, spec.proxy, previous); - // Replaces a seed.iso a QEMU killed a moment ago may still hold open (Windows). - await retryWhileInUse(() => writeSeedIso(this.paths.seedPath(spec.dotId), seed)); - const temporary = `${record}.tmp`; - await writeFile(temporary, `${JSON.stringify({ digest: seed.digest, instanceId: seed.instanceId })}\n`, { mode: 0o600 }); - await replaceFile(temporary, record); - return seed.instanceId; - } - - /** - * Provision a Dot's computer (section 9.4, up to the seed): the overlay - * backed by the golden image and the NoCloud seed. Safe to call again after - * a failure: an existing disk is kept and the seed is rewritten. - */ - async create(spec: VmSpec): Promise { - this.validateSpec(spec); - return this.withLock(spec.dotId, async () => { - await this.assertImages(spec); - const current = await this.probe(spec.dotId); - if (current.state !== "STOPPED") { - throw new VmStateError(spec.dotId, `the VM is ${current.state}; stop it before creating its disk and seed again`); - } - const dir = this.vmDir(spec.dotId); - await mkdir(dir, { recursive: true, mode: 0o700 }); - const diskPath = this.paths.diskPath(spec.dotId); - let diskCreated = false; - if (await pathExists(diskPath)) { - this.logger.warn("overlay already exists, keeping it", { dotId: spec.dotId, disk: diskPath }); - } else { - await this.qemuImg(["create", "-f", "qcow2", "-F", "qcow2", "-b", qemuPathArg("golden image", spec.goldenImage), qemuPathArg("disk", diskPath), String(spec.diskBytes)]); - diskCreated = true; - } - const instanceId = await this.writeSeed(spec); - this.logger.info("vm created", { dotId: spec.dotId, diskCreated, instanceId }); - return { vmDir: dir, diskPath, seedPath: this.paths.seedPath(spec.dotId), diskCreated, instanceId }; - }); - } - - /** Forget a QEMU that is gone: its pid file, and its ChildProcess if this process held it. */ - private async forget(dotId: string): Promise { - this.owned.delete(dotId); - await rm(this.paths.processFilePath(dotId), { force: true }); - } - - /** - * Whether the QEMU of the pid file still runs. Its exit when this process - * spawned it; otherwise its pid, unless the host restarted since the file - * was written. A pid that exists for another user counts as running here, - * so such a VM is reported ERROR rather than STOPPED and forgotten. - */ - private runs(dotId: string, record: ProcessFile): boolean { - const owned = this.owned.get(dotId); - if (owned && owned.pid === record.pid) return owned.exit === undefined; - if (hostRestartedSince(record.host_uptime_s)) return false; - return this.processes.presence(record.pid) !== "gone"; - } - - private async qemuRuns(dotId: string): Promise { - const record = await readProcessFile(this.paths.processFilePath(dotId)); - return record !== undefined && record !== "unreadable" && this.runs(dotId, record); - } - - /** - * The rule that decides whether a process may be killed or treated as a - * Dot's QEMU: either this process spawned it and Node has not reported its - * exit (Node holds the process, so its pid cannot be recycled meanwhile), - * or its pid from the pid file is alive and runs as this user (a QEMU this - * control plane spawned always does; EPERM means another user's process) - * AND the guest port from the same file accepts connections. QEMU listens - * there for as long as it runs; a process that got a recycled pid after a - * crash or a host restart does not listen on that one port. Nothing else - * is ever killed. A listener is still not trusted with the Dot token: the - * guest client asks it for its proof first (section 5.1). - */ - private async processIsOurs(dotId: string, record: ProcessFile): Promise { - const owned = this.owned.get(dotId); - if (owned && owned.pid === record.pid) return owned.exit === undefined; - if (hostRestartedSince(record.host_uptime_s)) return false; - return this.processes.presence(record.pid) === "ours" && (await portListens(record.guest_port)); - } - - /** Read a VM's state from its pid file and its process (section 3.4). */ - private async probe(dotId: string): Promise { - const record = await readProcessFile(this.paths.processFilePath(dotId)); - if (record === undefined) return STOPPED; - const file = this.paths.processFilePath(dotId); - if (record === "unreadable") { - return { - state: "ERROR", - pid: null, - guestPort: null, - verified: false, - detail: `${file} is not a pid file this version wrote, so whether a QEMU runs on this disk is unknown; make sure none does, then remove the file`, - }; - } - if (!this.runs(dotId, record)) return { ...STOPPED, record }; - if (await this.processIsOurs(dotId, record)) { - return { state: "RUNNING", pid: record.pid, guestPort: record.guest_port, verified: true, record }; - } - return { - state: "ERROR", - pid: record.pid, - guestPort: null, - verified: false, - record, - detail: - `process ${record.pid} from ${file} is alive but nothing listens on its guest port ${record.guest_port}: ` + - `either the pid now belongs to another process or that QEMU hangs. It is never killed by invisible_dots; ` + - `check the process, then remove ${file}`, - }; - } - - /** - * The VM's state (section 3.4): RUNNING while its QEMU runs, STOPPED when it - * does not. Whether the guest is up is guest health (section 9.3). This is - * the reconciliation of section 3.4 too, the one place it happens: a pid - * file whose process is gone is removed the moment it is read, under the - * Dot's lock, so a pid recycled later can never make it look alive. - */ - async state(dotId: string): Promise { - assertDotId(dotId); - return this.withLock(dotId, async () => { - const { verified: _verified, record, ...info } = await this.probe(dotId); - if (info.state === "STOPPED" && record) { - this.logger.info("QEMU of the pid file is gone, removing the file", { dotId, pid: record.pid }); - await this.forget(dotId); - } - return info; - }); - } - - /** - * Start the VM (section 9.4 from "pick a guest port" to "spawn QEMU"). The - * seed is rewritten from `spec` first, so the token on record is the one - * the guest sees; cpus, memory and the runtime ISO come from `spec` too, so - * new values apply on this start. Returns once QEMU listens on the guest - * port and is still running a moment later; waiting for the guest is - * `waitForGuestHealth`. - */ - async start(spec: VmSpec): Promise { - this.validateSpec(spec); - return this.withLock(spec.dotId, () => this.startLocked(spec)); - } - - private async startLocked(spec: VmSpec): Promise { - const { dotId } = spec; - const current = await this.probe(dotId); - if (current.state === "RUNNING" && current.record) { - return { pid: current.record.pid, guestPort: current.record.guest_port, alreadyRunning: true }; - } - if (current.state !== "STOPPED") { - throw new VmStateError(dotId, `cannot start a VM that is ${current.state}${current.detail ? ` (${current.detail})` : ""}; stop or destroy it first`); - } - await this.forget(dotId); - const diskPath = this.paths.diskPath(dotId); - if (!(await pathExists(diskPath))) throw new VmStateError(dotId, `${diskPath} does not exist; create the VM first`); - await this.assertImages(spec); - await this.writeSeed(spec); - const qemu = await this.qemu(); - const accel = this.accelerator; - await mkdir(this.paths.logsDir, { recursive: true, mode: 0o700 }); - const logPath = this.paths.qemuLogPath(dotId); - const attempts = this.options.maxPortAttempts ?? 5; - const pickPort = this.options.pickPort ?? (() => pickFreePort()); - - for (let attempt = 1; attempt <= attempts; attempt++) { - const guestPort = await pickPort(); - const args = qemuArgs({ - dotId, - accelerator: accel, - cpus: spec.cpus, - memoryMiB: spec.memoryMiB, - diskPath, - seedPath: this.paths.seedPath(dotId), - runtimeIsoPath: spec.runtimeImage, - guestPort, - serialLogPath: this.paths.serialLogPath(dotId), - }); - await appendFile(logPath, `--- ${new Date().toISOString()} start, guest port ${guestPort}\n`, { mode: 0o600 }); - const logOffset = await fileSize(logPath); - let child: DetachedProcess; - try { - child = await this.processes.spawnDetached(qemu.system, args, { - logPath, - cwd: this.paths.home, - env: allowlistedEnvironment(process.env), - }); - } catch (error) { - if (error instanceof CommandError && error.notFound) { - // Installed QEMU went away since it was found: look again next time. - this.qemuPromise = this.options.qemu ? this.qemuPromise : undefined; - } - throw new VmStartError(dotId, `cannot run ${qemu.system}: ${(error as Error).message}`, "", { cause: error }); - } - const owned: OwnedProcess = { pid: child.pid, exited: child.exited }; - void child.exited.then((exit) => { - owned.exit = exit; - }); - this.owned.set(dotId, owned); - const record: ProcessFile = { pid: child.pid, guest_port: guestPort, host_uptime_s: hostUptimeSeconds() }; - try { - // Before anything can fail: from here on a restarted control plane finds this QEMU. - await writeProcessFile(this.paths.processFilePath(dotId), record); - } catch (error) { - await this.killOurs(dotId, record); - await this.forget(dotId); - throw new VmStartError(dotId, `cannot write the pid file ${this.paths.processFilePath(dotId)}: ${(error as Error).message}`, "", { cause: error }); - } - this.logger.info("qemu spawned", { dotId, pid: child.pid, guestPort, attempt }); - - const outcome = await this.waitForForward(owned, guestPort); - if (outcome.kind === "exited") { - const output = await this.readLogFrom(logPath, logOffset); - await this.forget(dotId); - if (PORT_TAKEN.test(output) && attempt < attempts) { - this.logger.warn("guest port taken, trying another", { dotId, guestPort, attempt }); - continue; - } - throw this.startFailure(dotId, accel, output, outcome.exit); - } - if (outcome.kind === "timeout") { - await this.killOurs(dotId, record); - await this.forget(dotId); - const output = await this.readLogFrom(logPath, logOffset); - throw new VmStartError( - dotId, - `QEMU did not set up its port forward on 127.0.0.1:${guestPort} within ${this.options.startTimeoutMs ?? 30_000} ms and was killed`, - output, - ); - } - this.logger.info("vm started", { dotId, pid: child.pid, guestPort }); - return { pid: child.pid, guestPort, alreadyRunning: false }; - } - throw new VmStartError(dotId, `QEMU could not bind a guest port in ${attempts} attempts`); - } - - /** - * Wait until QEMU listens on the guest port and is still running - * `startSettleMs` later, until it exits, or until the start timeout. The - * listening port is the sign QEMU parsed its command line, opened the - * accelerator and set up networking; the settle catches what fails right - * after (the CPU model is checked when the machine is built). - */ - private async waitForForward(owned: OwnedProcess, guestPort: number): Promise<{ kind: "ready" } | { kind: "exited"; exit: ProcessExit } | { kind: "timeout" }> { - const deadline = Date.now() + (this.options.startTimeoutMs ?? 30_000); - const interval = this.options.pollIntervalMs ?? 500; - const exited = async () => { - // A settled `exited` is seen only after the microtask queue drains. - await Promise.resolve(); - if (owned.exit === undefined && this.processes.presence(owned.pid) !== "gone") return undefined; - // Let the exit event deliver the code when it is about to. - return owned.exit ?? (await Promise.race([owned.exited, this.sleep(interval).then(() => ({ code: null, signal: null }))])); - }; - for (;;) { - const exit = await exited(); - if (exit) return { kind: "exited", exit }; - if (await portListens(guestPort)) { - await this.sleep(this.options.startSettleMs ?? 1000); - const late = await exited(); - return late ? { kind: "exited", exit: late } : { kind: "ready" }; - } - if (Date.now() >= deadline) return { kind: "timeout" }; - await this.sleep(interval); - } - } - - private startFailure(dotId: string, accel: Accelerator, output: string, exit: ProcessExit): Error { - if (CPU_MODEL_FAILURES.some((pattern) => pattern.test(output))) return new CpuModelError(accel, CPU_MODEL, output); - if (ACCELERATOR_FAILURES.some((pattern) => pattern.test(output))) return new AcceleratorUnavailableError(accel, output); - const how = exit.signal ? `was killed by ${exit.signal}` : exit.code === null ? "exited" : `exited with code ${exit.code}`; - return new VmStartError(dotId, `QEMU ${how} during start`, output); - } - - /** What a log gained since `offset`, at most its last 64 KiB: the tail is what explains a failure. */ - private async readLogFrom(path: string, offset: number): Promise { - return this.readRange(path, offset, 64 * 1024); - } - - /** The last few KiB of a log, or "" when it does not exist. */ - private readTail(path: string): Promise { - return this.readRange(path, 0, LOG_TAIL_BYTES); - } - - private async readRange(path: string, offset: number, maxBytes: number): Promise { - try { - const handle = await open(path, "r"); - try { - const { size } = await handle.stat(); - // A runaway log must not be read whole. - const start = Math.max(offset, size - maxBytes); - const buffer = Buffer.alloc(Math.max(0, size - start)); - await handle.read(buffer, 0, buffer.length, start); - return buffer.toString("utf8"); - } finally { - await handle.close(); - } - } catch { - return ""; - } - } - - /** Wait until the QEMU of `record` is gone; false when it still runs after `timeoutMs`. */ - private async waitForExit(dotId: string, record: ProcessFile, timeoutMs: number): Promise { - const deadline = Date.now() + timeoutMs; - const interval = this.options.pollIntervalMs ?? 500; - for (;;) { - // A settled `exited` is seen only after the microtask queue drains. - await Promise.resolve(); - if (!this.runs(dotId, record)) return true; - if (Date.now() >= deadline) return false; - await this.sleep(interval); - } - } - - /** Kill the QEMU of `record`, only if `processIsOurs` still holds, and wait for it to go. */ - private async killOurs(dotId: string, record: ProcessFile): Promise { - if (!(await this.processIsOurs(dotId, record))) { - this.logger.info("QEMU is gone, nothing to kill", { dotId, pid: record.pid }); - return; - } - this.logger.warn("killing QEMU", { dotId, pid: record.pid }); - this.processes.kill(record.pid); - const timeoutMs = this.options.killTimeoutMs ?? 10_000; - if (!(await this.waitForExit(dotId, record, timeoutMs))) { - throw new VmStateError(dotId, `QEMU ${record.pid} is still running ${timeoutMs} ms after it was killed`); - } - } - - /** - * Graceful stop (sections 3.4 and 9.5): `POST /v1/system/poweroff` through - * the guest channel, then wait up to the shutdown timeout for QEMU to exit - * (the guest powers off and QEMU exits with it), then kill it. A guest - * that does not accept the poweroff (it is not up, or it refuses the - * token) cannot power itself off, so its QEMU is killed at once. Callers - * that want the guest's state flushed first call the agent's prepare-sleep - * before this. - */ - async stop(dotId: string, token: string, options: { timeoutMs?: number } = {}): Promise { - assertDotId(dotId); - return this.withLock(dotId, () => this.stopLocked(dotId, token, options.timeoutMs ?? this.options.shutdownTimeoutMs ?? 60_000)); - } - - private async stopLocked(dotId: string, token: string, timeoutMs: number): Promise { - const current = await this.probe(dotId); - if (current.state === "STOPPED") { - await this.forget(dotId); - return { forced: false, wasRunning: false }; - } - if (!current.verified || !current.record) { - throw new VmStateError(dotId, `cannot stop it safely: ${current.detail ?? "its QEMU cannot be identified"}`); - } - const record = current.record; - let accepted = false; - try { - await this.guestClient(record.guest_port, token).powerOff(this.options.powerOffRequestTimeoutMs ?? 10_000); - accepted = true; - } catch (error) { - this.logger.warn("the guest did not accept the poweroff, killing QEMU", { dotId, error: (error as Error).message }); - } - let forced = false; - if (!accepted || !(await this.waitForExit(dotId, record, timeoutMs))) { - if (accepted) this.logger.warn("guest did not power off in time, killing QEMU", { dotId, timeoutMs }); - await this.killOurs(dotId, record); - forced = true; - } - await this.forget(dotId); - this.logger.info("vm stopped", { dotId, forced }); - return { forced, wasRunning: true }; - } - - /** - * Reboot as a full stop and start, not a reset: a reset skips the guest's - * shutdown (the agent's SQLite state, browser profiles) and keeps the old - * QEMU process, so a new runtime ISO or new cpus and memory would not - * apply. The guest port changes; store the result. - */ - async reboot(spec: VmSpec, options: { timeoutMs?: number } = {}): Promise { - this.validateSpec(spec); - return this.withLock(spec.dotId, async () => { - await this.stopLocked(spec.dotId, spec.token, options.timeoutMs ?? this.options.shutdownTimeoutMs ?? 60_000); - return this.startLocked(spec); - }); - } - - /** Remove everything: the running QEMU (killed, no graceful shutdown: the disk goes too) and the VM directory. */ - async destroy(dotId: string): Promise { - assertDotId(dotId); - return this.withLock(dotId, async () => { - const dir = this.vmDir(dotId); - const current = await this.probe(dotId); - if (current.state !== "STOPPED") { - if (!current.verified || !current.record) { - throw new VmStateError(dotId, `cannot destroy it safely: ${current.detail ?? "its QEMU cannot be identified"}`); - } - await this.killOurs(dotId, current.record); - } - this.owned.delete(dotId); - await retryWhileInUse(() => rm(dir, { recursive: true, force: true })); - await retryWhileInUse(() => rm(this.paths.qemuLogPath(dotId), { force: true })); - this.logger.info("vm destroyed", { dotId, dir }); - }); - } - - /** - * Apply new resources. cpus and memory are not stored here: they are part - * of the VmSpec of every start, so a running VM gets them on its next - * start. The disk is grown with `qemu-img resize` and only while the VM is - * stopped; the guest's cloud-init grows the partition on the next boot. - */ - async resize(spec: VmSpec): Promise { - this.validateSpec(spec); - return this.withLock(spec.dotId, async () => { - const current = await this.probe(spec.dotId); - if (current.state === "STOPPED") { - const { resized } = await this.resizeDiskLocked(spec.dotId, spec.diskBytes); - return { restartRequired: false, diskResized: resized }; - } - const size = await this.diskSize(spec.dotId); - if (spec.diskBytes !== size) { - throw new VmStateError(spec.dotId, `the disk can only be resized while the VM is stopped (it is ${current.state})`); - } - return { restartRequired: true, diskResized: false }; - }); - } - - /** Grow the overlay; never shrink it, because that destroys the filesystem on it. */ - async resizeDisk(dotId: string, diskBytes: number): Promise<{ resized: boolean; previousBytes: number }> { - assertDotId(dotId); - return this.withLock(dotId, async () => { - const current = await this.probe(dotId); - if (current.state !== "STOPPED") { - throw new VmStateError(dotId, `the disk can only be resized while the VM is stopped (it is ${current.state})`); - } - return this.resizeDiskLocked(dotId, diskBytes); - }); - } - - private async diskSize(dotId: string): Promise { - const disk = this.paths.diskPath(dotId); - // -U: read the header even if a QEMU holds the image; the size is in the header and never torn. - const { stdout } = await this.qemuImg(["info", "--output=json", "-U", disk]); - try { - const size = Number((JSON.parse(stdout) as { "virtual-size": number })["virtual-size"]); - if (!Number.isSafeInteger(size)) throw new Error(`virtual-size is ${size}`); - return size; - } catch (error) { - throw new VmManagerError(`cannot read the size of ${disk} from qemu-img info`, { cause: error }); - } - } - - private async resizeDiskLocked(dotId: string, diskBytes: number): Promise<{ resized: boolean; previousBytes: number }> { - if (!Number.isSafeInteger(diskBytes) || diskBytes <= 0) { - throw new VmManagerError(`diskBytes must be a positive integer, got ${diskBytes}`); - } - const disk = this.paths.diskPath(dotId); - const previousBytes = await this.diskSize(dotId); - if (diskBytes < previousBytes) { - throw new VmManagerError(`dot ${dotId}: the disk is ${previousBytes} bytes; shrinking it to ${diskBytes} is not supported`); - } - if (diskBytes === previousBytes) return { resized: false, previousBytes }; - await this.qemuImg(["resize", "-f", "qcow2", disk, String(diskBytes)]); - this.logger.info("disk resized", { dotId, previousBytes, diskBytes }); - return { resized: true, previousBytes }; - } - - // Snapshots and rollback are out of scope (architecture section 10). - - createSnapshot(_dotId: string, _name: string): Promise { - return Promise.reject(new NotImplementedError("VM snapshots")); - } - - listSnapshots(_dotId: string): Promise { - return Promise.reject(new NotImplementedError("VM snapshots")); - } - - restoreSnapshot(_dotId: string, _name: string): Promise { - return Promise.reject(new NotImplementedError("VM snapshot rollback")); - } - - deleteSnapshot(_dotId: string, _name: string): Promise { - return Promise.reject(new NotImplementedError("VM snapshots")); - } -} diff --git a/apps/vm-manager/test/accelerator-access.test.ts b/apps/vm-manager/test/accelerator-access.test.ts deleted file mode 100644 index 8197b04c..00000000 --- a/apps/vm-manager/test/accelerator-access.test.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { - checkAcceleratorAccess, - encodeCommand, - HYPERVISOR_PLATFORM_STATE_SCRIPT, - powershellArgs, - powershellPath, - type AccessDeps, -} from "../src/index.js"; -import { fakeRunner, type Answer } from "./doctor-fakes.js"; - -const PS = "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe"; - -function errno(code: string): NodeJS.ErrnoException { - return Object.assign(new Error(`${code}: /dev/kvm`), { code }); -} - -function fakeHost(options: { platform: NodeJS.Platform; respond?: (command: string, args: readonly string[]) => Answer; kvm?: string }) { - const runner = fakeRunner((command, args) => options.respond?.(command, args) ?? {}); - const deps: AccessDeps = { - platform: options.platform, - env: { SystemRoot: "C:\\Windows" }, - run: runner.run, - openReadWrite: async () => { - if (options.kvm) throw errno(options.kvm); - }, - }; - return { deps, calls: runner.calls }; -} - -describe("accelerator access (the host side of section 1.1)", () => { - it("Linux: /dev/kvm read-write is ok; absent and not permitted are missing with their fixes", async () => { - expect(await checkAcceleratorAccess(fakeHost({ platform: "linux" }).deps)).toEqual({ - id: "accelerator", - label: "accelerator", - status: "ok", - detail: "/dev/kvm opens read-write", - }); - expect(await checkAcceleratorAccess(fakeHost({ platform: "linux", kvm: "EACCES" }).deps)).toEqual({ - id: "accelerator", - label: "accelerator", - status: "missing", - detail: "/dev/kvm exists but this user cannot open it read-write", - fix: "sudo usermod -aG kvm $USER, then log out and in again (a new login is needed for the group to apply)", - }); - expect(await checkAcceleratorAccess(fakeHost({ platform: "linux", kvm: "ENOENT" }).deps)).toMatchObject({ - status: "missing", - detail: "/dev/kvm does not exist", - }); - expect((await checkAcceleratorAccess(fakeHost({ platform: "linux", kvm: "EIO" }).deps)).status).toBe("failed"); - }); - - it("Windows: reads the HypervisorPlatform state through Get-CimInstance, without elevation", async () => { - const states: Record = { "1\r\n": "ok", "2\r\n": "missing", "3\r\n": "failed", "": "failed" }; - for (const [stdout, status] of Object.entries(states)) { - const host = fakeHost({ platform: "win32", respond: () => ({ stdout }) }); - expect((await checkAcceleratorAccess(host.deps)).status).toBe(status); - expect(host.calls).toEqual([{ command: PS, args: powershellArgs(HYPERVISOR_PLATFORM_STATE_SCRIPT), options: { timeoutMs: 60_000 } }]); - } - const disabled = await checkAcceleratorAccess(fakeHost({ platform: "win32", respond: () => ({ stdout: "2\r\n" }) }).deps); - expect(disabled).toEqual({ - id: "accelerator", - label: "accelerator", - status: "missing", - detail: "the HypervisorPlatform feature is disabled", - fix: "invisible-dots setup (enables it with one administrator prompt; a restart follows)", - }); - const broken = await checkAcceleratorAccess(fakeHost({ platform: "win32", respond: () => ({ code: 1, stderr: "Get-CimInstance : Access denied\r\n" }) }).deps); - expect(broken).toMatchObject({ status: "failed", detail: "cannot read the HypervisorPlatform feature state: Get-CimInstance : Access denied" }); - }); - - it("any other host is refused", async () => { - expect(await checkAcceleratorAccess(fakeHost({ platform: "darwin" }).deps)).toMatchObject({ status: "failed", detail: "darwin hosts are not supported" }); - }); -}); - -describe("the PowerShell the host runs", () => { - it("is Windows PowerShell by absolute path, given a script as an encoded command that reads back to the script", () => { - expect(Buffer.from(encodeCommand("Write-Output 1"), "base64").toString("utf16le")).toBe("Write-Output 1"); - expect(powershellArgs("x")).toEqual(["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-EncodedCommand", encodeCommand("x")]); - expect(powershellPath({ SystemRoot: "D:\\Win" })).toBe("D:\\Win\\System32\\WindowsPowerShell\\v1.0\\powershell.exe"); - expect(powershellPath({})).toBe("C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell.exe"); - expect(HYPERVISOR_PLATFORM_STATE_SCRIPT).toBe(`(Get-CimInstance -ClassName Win32_OptionalFeature -Filter "Name='HypervisorPlatform'").InstallState`); - }); -}); diff --git a/apps/vm-manager/test/agentd.linux.test.ts b/apps/vm-manager/test/agentd.linux.test.ts deleted file mode 100644 index 8881884d..00000000 --- a/apps/vm-manager/test/agentd.linux.test.ts +++ /dev/null @@ -1,179 +0,0 @@ -/** - * The host's GuestClient against the real dot-agentd, not a stand-in: the daemon built with the image builder's one - * build, run as its README's development mode describes (`--run-as=`: the model's work as the daemon's own user; - * `--listen 127.0.0.1:0`), with a guest config holding the Dot's token, and a stand-in for the engine on the unix - * socket the daemon proxies `/v1/agent` to. What the two sides each test alone with a fake of the other, this tests - * across the seam: the proof, the token, the routes, the bodies, and an event stream passed on as it is written. - * - * Linux only (vitest.config.ts leaves `*.linux.test.ts` out elsewhere): the daemon is a Linux program, its exec route - * runs /bin/bash and the engine's socket is a unix socket. It needs Go, which CI's node job sets up from go.mod. - */ -import { execFile, spawn, type ChildProcess } from "node:child_process"; -import { mkdir, mkdtemp, realpath, rm, writeFile } from "node:fs/promises"; -import { createServer, get, type Server, type ServerResponse } from "node:http"; -import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; -import { promisify } from "node:util"; -import { AGENTD_ROUTES, GUEST_UNPROVEN, type OutboundEvent } from "@invisible-dots/shared"; -import { agentdBuildCommand } from "@invisible-dots/image-builder"; -import { afterAll, beforeAll, describe, expect, it } from "vitest"; -import { GuestClient, GuestRequestError } from "../src/index.js"; - -const REPO = resolve(fileURLToPath(new URL(".", import.meta.url)), "../../.."); -const TOKEN = "integration-token-0123456789"; - -let dir: string; -let home: string; -let daemon: ChildProcess | undefined; -let agent: Server | undefined; -let port: number; -/** What reached the stand-in engine through the daemon's proxy, as `METHOD /path?query`. */ -const agentSeen: string[] = []; -/** The open event streams the stand-in holds, to write the next event into. */ -const streams: ServerResponse[] = []; - -function event(seq: number): OutboundEvent { - return { seq, id: `evt_${seq}`, type: "automation.next_run", ts: "2026-10-11T10:00:00.000Z", data: { next_run_at_ms: seq } }; -} - -function sse(res: ServerResponse, seq: number): void { - res.write(`id: ${seq}\ndata: ${JSON.stringify(event(seq))}\n\n`); -} - -/** The engine's side of the proxy: /health, and /events/stream, which sends one event and keeps the stream open. */ -function startAgent(socket: string): Promise { - const server = createServer((req, res) => { - const url = new URL(req.url!, "http://agent"); - agentSeen.push(`${req.method} ${url.pathname}${url.search}`); - if (url.pathname === "/health") { - res.writeHead(200, { "content-type": "application/json" }); - res.end(JSON.stringify({ status: "ok", state: "idle", openrouter_configured: true, browser: { identities: 0, open: 0 }, checks: {} })); - return; - } - if (url.pathname === "/events/stream") { - res.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); - sse(res, Number(url.searchParams.get("after") ?? 0) + 1); - streams.push(res); - return; - } - res.writeHead(404, { "content-type": "application/json" }).end(JSON.stringify({ error: "not_found" })); - }); - return new Promise((done, fail) => { - server.once("error", fail); - server.listen(socket, () => done(server)); - }); -} - -/** Starts the daemon and answers the TCP port it listens on, from its log line `listening on="tcp 127.0.0.1:"`. */ -function startDaemon(binary: string, args: string[]): Promise<{ child: ChildProcess; port: number }> { - const child = spawn(binary, args, { stdio: ["ignore", "ignore", "pipe"] }); - return new Promise((done, fail) => { - let log = ""; - child.stderr!.setEncoding("utf8"); - child.stderr!.on("data", (chunk: string) => { - log += chunk; - const listening = /listening on="tcp 127\.0\.0\.1:(\d+)"/.exec(log); - if (listening) done({ child, port: Number(listening[1]) }); - }); - child.once("error", fail); - child.once("exit", (code, signal) => fail(new Error(`dot-agentd exited (${code ?? signal}) before it listened:\n${log}`))); - }); -} - -beforeAll(async () => { - dir = await mkdtemp(join(tmpdir(), "idots-agentd-")); - home = join(dir, "home"); - await mkdir(home); - const binary = join(dir, "dot-agentd"); - const build = agentdBuildCommand(REPO, binary); - await promisify(execFile)(build.command, build.args, { env: { ...process.env, ...build.env } }); - const config = join(dir, "config.json"); - await writeFile(config, JSON.stringify({ dotId: "dot_integration", token: TOKEN }), { mode: 0o600 }); - const agentSocket = join(dir, "agent.sock"); - agent = await startAgent(agentSocket); - const started = await startDaemon(binary, [ - "--config", config, - "--home", home, - "--run-dir", join(dir, "run"), - "--agent-socket", agentSocket, - "--listen", "127.0.0.1:0", - "--run-as=", - ]); - daemon = started.child; - port = started.port; - // The build: a cold Go cache compiles the standard library for linux/amd64 first, and a host whose Go is older - // than go.mod's toolchain downloads that one. -}, 600_000); - -afterAll(async () => { - for (const res of streams) res.destroy(); - if (daemon && daemon.exitCode === null && daemon.signalCode === null) { - const exited = new Promise((done) => daemon!.once("exit", done)); - daemon.kill("SIGTERM"); - await exited; - } - await new Promise((done) => (agent ? agent.close(() => done()) : done())); - if (dir) await rm(dir, { recursive: true, force: true }); -}); - -describe("GuestClient against dot-agentd", () => { - it("is given the proof of the token before it sends the token", async () => { - await expect(new GuestClient(port, TOKEN).prove()).resolves.toBeUndefined(); - const impostor = new GuestClient(port, "another-dot-token"); - await expect(impostor.health()).rejects.toMatchObject({ code: GUEST_UNPROVEN }); - }); - - it("is refused without the token: 401 with the Bearer challenge", async () => { - const answer = await new Promise<{ status: number; challenge: string | undefined; body: string }>((done, fail) => { - get({ host: "127.0.0.1", port, path: AGENTD_ROUTES.health }, (res) => { - let body = ""; - res.setEncoding("utf8"); - res.on("data", (chunk: string) => (body += chunk)); - res.on("end", () => done({ status: res.statusCode ?? 0, challenge: res.headers["www-authenticate"], body })); - }).once("error", fail); - }); - expect(answer.status).toBe(401); - expect(answer.challenge).toBe('Bearer realm="dot-agentd"'); - expect(JSON.parse(answer.body)).toMatchObject({ error: "unauthorized" }); - }); - - it("reads health with the engine's own answer passed through", async () => { - const health = await new GuestClient(port, TOKEN).health(); - expect(health).toMatchObject({ agentd: "ok", agent: { status: "ok", state: "idle", openrouter_configured: true } }); - expect(agentSeen).toContain("GET /health"); - }); - - it("runs a command in home and reads its exit code and both streams", async () => { - const client = new GuestClient(port, TOKEN); - expect(await client.exec({ command: "printf out; printf err >&2; exit 3" })).toMatchObject({ exit_code: 3, stdout: "out", stderr: "err", timed_out: false }); - expect((await client.exec({ command: "pwd" })).stdout.trim()).toBe(await realpath(home)); - }); - - it("writes, reads and lists a file under home, and refuses one outside it", async () => { - const client = new GuestClient(port, TOKEN); - await client.writeFile("notes/hello.txt", "hello from the host"); - expect((await client.readFile("~/notes/hello.txt")).toString("utf8")).toBe("hello from the host"); - const listing = await client.listFiles("notes"); - expect(listing.entries).toEqual([expect.objectContaining({ name: "hello.txt", type: "file", size: 19 })]); - const outside = await client.readFile("/etc/hostname").catch((error: unknown) => error); - expect(outside).toBeInstanceOf(GuestRequestError); - expect(outside).toMatchObject({ status: 403, code: "outside_home" }); - }); - - it("proxies the engine's routes, and its event stream as it is written", async () => { - const client = new GuestClient(port, TOKEN); - expect(await client.agentHealth()).toMatchObject({ status: "ok", state: "idle" }); - const controller = new AbortController(); - const got: number[] = []; - for await (const evt of client.events({ after: 4, signal: controller.signal })) { - got.push(evt.seq); - // The first event arrived while the engine still holds the stream open: the proxy did not wait for its end. - // The next one is written only now. - if (got.length === 1) sse(streams.at(-1)!, 6); - else controller.abort(); - } - expect(got).toEqual([5, 6]); - expect(agentSeen).toContain("GET /events/stream?after=4"); - }); -}); diff --git a/apps/vm-manager/test/doctor-fakes.ts b/apps/vm-manager/test/doctor-fakes.ts deleted file mode 100644 index 9f294371..00000000 --- a/apps/vm-manager/test/doctor-fakes.ts +++ /dev/null @@ -1,57 +0,0 @@ -/** Fakes at the process boundary, shared by the vm-manager's doctor tests and the CLI's: a runner that records every command line and answers from a script, and a host where every check passes. */ -import type { DoctorCheck } from "@invisible-dots/shared"; -import type { DoctorDeps, FoundQemu, RunOptions, RunResult } from "../src/index.js"; - -export interface RecordedRun { - command: string; - args: readonly string[]; - options: RunOptions; -} - -export type Answer = Partial; - -export function answer(partial: Answer = {}): RunResult { - return { code: 0, signal: null, stdout: "", stderr: "", timedOut: false, ...partial }; -} - -/** A runner whose answers come from `respond`; every call is kept in `calls`. */ -export function fakeRunner(respond: (command: string, args: readonly string[], options: RunOptions) => Answer | Promise) { - const calls: RecordedRun[] = []; - const run = async (command: string, args: readonly string[], options: RunOptions = {}) => { - calls.push({ command, args, options }); - return answer(await respond(command, args, options)); - }; - return { run, calls }; -} - -export const QEMU = "/opt/qemu/bin/qemu-system-x86_64"; -export const QEMU_IMG = "/opt/qemu/bin/qemu-img"; -export const FOUND: FoundQemu = { system: QEMU, img: QEMU_IMG, searched: ["INVISIBLE_DOTS_QEMU_DIR", "PATH"] }; - -export function ok(id: DoctorCheck["id"], label: string, detail = "fine"): DoctorCheck { - return { id, label, status: "ok", detail }; -} - -/** A host where every check passes; tests override one dependency at a time. */ -export function healthyDoctor(overrides: Partial = {}) { - const runner = fakeRunner((command, args) => { - if (args[0] === "--version") { - return { stdout: command === QEMU ? "QEMU emulator version 8.2.2 (Debian 1:8.2.2+ds-0ubuntu1.18)\nCopyright\n" : "qemu-img version 8.2.2 (Debian 1:8.2.2+ds-0ubuntu1.18)\n" }; - } - // The probe: the firmware found nothing to boot and its reset ended QEMU (section 11.1). - return { code: 0 }; - }); - const deps: DoctorDeps = { - nodeVersion: "24.19.0", - accelerator: () => "kvm", - findQemu: async () => FOUND, - run: runner.run, - acceleratorAccess: async () => ok("accelerator", "accelerator", "/dev/kvm opens read-write"), - home: "/home/someone/.invisible-dots", - freeSpace: async () => ({ path: "/home/someone/.invisible-dots", bytes: 100 * 1024 ** 3 }), - images: async () => [ok("golden-image", "golden image", "golden-1.qcow2 matches its manifest"), ok("runtime-image", "runtime ISO", "runtime-1.iso matches its manifest")], - openRouterKey: async () => ok("openrouter", "OpenRouter key", "stored"), - ...overrides, - }; - return { deps, calls: runner.calls }; -} diff --git a/apps/vm-manager/test/doctor.test.ts b/apps/vm-manager/test/doctor.test.ts deleted file mode 100644 index 277eccf8..00000000 --- a/apps/vm-manager/test/doctor.test.ts +++ /dev/null @@ -1,234 +0,0 @@ -import type { DoctorCheck } from "@invisible-dots/shared"; -import { describe, expect, it } from "vitest"; -import { acceleratorProbeArgs, checkDataDirectory, checkNode, checkWebBuild, doctorAnswer, MIN_FREE_BYTES, openRouterCheck, PROBE_TIMEOUT_MS, runDoctor } from "../src/index.js"; -import { answer, healthyDoctor, QEMU } from "./doctor-fakes.js"; - -const ids = (results: DoctorCheck[]) => results.map((r) => r.id); -const find = (results: DoctorCheck[], id: DoctorCheck["id"]) => results.find((r) => r.id === id)!; - -describe("doctor checks", () => { - it("reports an unsupported host as a row, never by stopping the report", async () => { - const { deps, calls } = healthyDoctor({ - accelerator: () => { - throw new Error("invisible_dots does not run on darwin"); - }, - acceleratorAccess: async () => ({ id: "accelerator", label: "accelerator", status: "failed", detail: "darwin hosts are not supported", fix: "run invisible_dots on Linux or Windows (x86-64)" }), - }); - const results = await runDoctor(deps); - expect(ids(results)).toEqual(["node", "qemu", "qemu-img", "accelerator", "accelerator-probe", "disk", "golden-image", "runtime-image", "openrouter"]); - expect(find(results, "accelerator")).toMatchObject({ status: "failed", detail: "darwin hosts are not supported" }); - expect(find(results, "accelerator-probe")).toMatchObject({ status: "failed", detail: expect.stringContaining("does not run on darwin") }); - expect(calls.filter((c) => c.args.includes("-accel"))).toEqual([]); - }); - - it("runs every check of section 11.1 in the contract's order", async () => { - const results = await runDoctor(healthyDoctor().deps); - expect(ids(results)).toEqual(["node", "qemu", "qemu-img", "accelerator", "accelerator-probe", "disk", "golden-image", "runtime-image", "openrouter"]); - expect(results.every((r) => r.status === "ok")).toBe(true); - }); - - it("probes the accelerator by running the firmware of an empty q35 machine until its reset ends QEMU", async () => { - const { deps, calls } = healthyDoctor({ accelerator: () => "whpx" }); - const results = await runDoctor(deps); - const probe = calls.find((c) => c.args.includes("-accel"))!; - expect(probe.command).toBe(QEMU); - expect(probe.args).toEqual([ - "-nodefaults", "-no-user-config", "-machine", "q35", "-accel", "whpx", "-cpu", "host,-vmx,-svm", - "-display", "none", "-no-reboot", "-boot", "reboot-timeout=0", - ]); // prettier-ignore - expect(probe.options).toEqual({ timeoutMs: PROBE_TIMEOUT_MS }); - expect(find(results, "accelerator-probe").detail).toBe("qemu-system-x86_64 -accel whpx -cpu host,-vmx,-svm -machine q35 ran the firmware and exited"); - expect(acceleratorProbeArgs("kvm")).toContain("kvm"); - }); - - it("never holds the probe before its first instruction or talks to a monitor", () => { - const args = acceleratorProbeArgs("kvm"); - for (const flag of ["-S", "-qmp", "-monitor", "-mon", "-chardev"]) expect(args).not.toContain(flag); - }); - - it("reports QEMU and qemu-img missing with the setup fix, and does not probe", async () => { - const { deps, calls } = healthyDoctor({ findQemu: async () => ({ searched: ["C:\\Program Files\\qemu", "C:\\a", "C:\\b"] }) }); - const results = await runDoctor(deps); - expect(find(results, "qemu")).toEqual({ - id: "qemu", - label: "QEMU", - status: "missing", - detail: "qemu-system-x86_64 not found in the official installer's directory or on PATH (3 directories searched)", - fix: "invisible-dots setup", - }); - expect(find(results, "qemu-img").status).toBe("missing"); - expect(find(results, "accelerator-probe")).toMatchObject({ status: "missing", detail: "not run: QEMU is not ready", fix: "invisible-dots setup" }); - expect(calls).toHaveLength(0); - }); - - it("says INVISIBLE_DOTS_QEMU_DIR was the only place searched while it is set", async () => { - const { deps } = healthyDoctor({ findQemu: async () => ({ searched: ["D:\\tools\\qemu"], configured: true }) }); - expect(find(await runDoctor(deps), "qemu").detail).toBe( - "qemu-system-x86_64 not found in INVISIBLE_DOTS_QEMU_DIR (D:\\tools\\qemu), the only place searched while it is set", - ); - }); - - it("fails a QEMU older than 8.2 and one that reports no version", async () => { - const old = healthyDoctor(); - old.deps.run = async (command, args) => - answer(args[0] === "--version" ? { stdout: command === QEMU ? "QEMU emulator version 8.1.5 (Debian 1:8.1.5+ds-1)\n" : "qemu-img version 8.1.5\n" } : {}); - const results = await runDoctor(old.deps); - expect(find(results, "qemu")).toMatchObject({ status: "failed", detail: `${QEMU} is version 8.1.5, older than 8.2` }); - expect(find(results, "qemu-img").status).toBe("ok"); - expect(find(results, "accelerator-probe")).toMatchObject({ status: "failed", detail: "not run: QEMU is not ready" }); - - const broken = healthyDoctor(); - broken.deps.run = async () => answer({ code: 127, stderr: "error while loading shared libraries: libSDL2\n" }); - expect(find(await runDoctor(broken.deps), "qemu").detail).toBe( - `${QEMU} --version did not report a version (error while loading shared libraries: libSDL2)`, - ); - }); - - it("probes even when the host-side check says missing, and gives the failure that check's fix", async () => { - const missing: DoctorCheck = { id: "accelerator", label: "accelerator", status: "missing", detail: "/dev/kvm exists but this user cannot open it read-write", fix: "sudo usermod -aG kvm $USER" }; - const { deps, calls } = healthyDoctor({ acceleratorAccess: async () => missing }); - const versions = deps.run; - deps.run = async (command, args, options) => - args.includes("-accel") - ? answer({ code: 1, stderr: "qemu-system-x86_64: -accel kvm: Could not access KVM kernel module: Permission denied\n" }) - : versions(command, args, options); - const results = await runDoctor(deps); - expect(find(results, "accelerator")).toEqual(missing); - expect(find(results, "accelerator-probe")).toEqual({ - id: "accelerator-probe", - label: "accelerator probe", - status: "failed", - detail: "qemu-system-x86_64 -accel kvm -cpu host,-vmx,-svm -machine q35 failed: qemu-system-x86_64: -accel kvm: Could not access KVM kernel module: Permission denied", - fix: "sudo usermod -aG kvm $USER", - }); - expect(calls).toHaveLength(2); - }); - - it("reports the accelerator ok when the probe contradicts a host-side missing (WHPX without HypervisorPlatform)", async () => { - const { deps } = healthyDoctor({ - accelerator: () => "whpx", - acceleratorAccess: async () => ({ id: "accelerator", label: "accelerator", status: "missing", detail: "the HypervisorPlatform feature is disabled", fix: "invisible-dots setup" }), - }); - const results = await runDoctor(deps); - expect(find(results, "accelerator")).toEqual({ - id: "accelerator", - label: "accelerator", - status: "ok", - detail: "the HypervisorPlatform feature is disabled, but the probe below shows QEMU uses the accelerator", - }); - expect(find(results, "accelerator-probe").status).toBe("ok"); - }); - - it("fails the probe when it does not exit by itself: the virtual CPU never ran the firmware to its reset", async () => { - const hanging = healthyDoctor(); - const versions = hanging.deps.run; - hanging.deps.run = async (command, args, options) => - args.includes("-accel") ? answer({ code: null, signal: "SIGKILL", timedOut: true }) : versions(command, args, options); - expect(find(await runDoctor(hanging.deps), "accelerator-probe")).toMatchObject({ - status: "failed", - detail: "qemu-system-x86_64 -accel kvm -cpu host,-vmx,-svm -machine q35 failed: did not exit within 30 s, so the virtual CPU does not run", - fix: "enable hardware virtualization (Intel VT-x or AMD-V) in the firmware settings; after enabling the accelerator, restart the computer", - }); - - // Measured on Windows with QEMU 11.1: a refused vCPU leaves QEMU running with this on stderr. - const paused = healthyDoctor(); - paused.deps.run = async (command, args, options) => - args.includes("-accel") ? answer({ code: null, signal: "SIGKILL", timedOut: true, stderr: "WHPX: Unexpected VP exit code 4\n" }) : versions(command, args, options); - expect(find(await runDoctor(paused.deps), "accelerator-probe").detail).toBe( - "qemu-system-x86_64 -accel kvm -cpu host,-vmx,-svm -machine q35 failed: did not exit within 30 s, so the virtual CPU does not run (QEMU said: WHPX: Unexpected VP exit code 4)", - ); - }); - - it("turns a check that throws into a failed line instead of ending the report", async () => { - const { deps } = healthyDoctor({ - freeSpace: async () => { - throw new Error("statfs exploded"); - }, - images: async () => { - throw new Error("readdir exploded"); - }, - findQemu: async () => { - throw new Error("PATH unreadable"); - }, - }); - const results = await runDoctor(deps); - expect(results).toHaveLength(9); - expect(find(results, "disk")).toMatchObject({ status: "failed", detail: "the check itself failed: statfs exploded" }); - expect(find(results, "golden-image").status).toBe("failed"); - expect(find(results, "runtime-image").status).toBe("failed"); - expect(find(results, "qemu").detail).toBe("qemu-system-x86_64 not found (the search failed: PATH unreadable)"); - }); - - it("has a web row, just before the OpenRouter key, only when the caller serves a web client", async () => { - const webBuild = async () => ({ entry: "/repo/apps/web/.next/standalone/apps/web/server.js", missing: undefined }); - const results = await runDoctor(healthyDoctor({ webBuild }).deps); - expect(ids(results)).toEqual(["node", "qemu", "qemu-img", "accelerator", "accelerator-probe", "disk", "golden-image", "runtime-image", "web", "openrouter"]); - expect(results.every((r) => r.status === "ok")).toBe(true); - expect(ids(await runDoctor(healthyDoctor().deps))).not.toContain("web"); - }); - - it("reports the web client built, or missing with the command that builds it", async () => { - const entry = "/repo/apps/web/.next/standalone/apps/web/server.js"; - expect(checkWebBuild({ entry, missing: undefined })).toEqual({ id: "web", label: "web client", status: "ok", detail: `built at ${entry}` }); - const missing = "/repo/apps/web/.next/standalone/apps/web/.next/static"; - const { deps } = healthyDoctor({ webBuild: async () => ({ entry, missing }) }); - expect(find(await runDoctor(deps), "web")).toEqual({ - id: "web", - label: "web client", - status: "missing", - detail: `not built: ${missing} does not exist`, - fix: "npm run build --workspace @invisible-dots/web", - }); - }); - - it("turns a web check that throws into a failed line", async () => { - const { deps } = healthyDoctor({ - webBuild: async () => { - throw new Error("stat exploded"); - }, - }); - expect(find(await runDoctor(deps), "web")).toMatchObject({ status: "failed", detail: "the check itself failed: stat exploded" }); - }); - - it("checks Node and free space against their minimums", () => { - expect(checkNode("24.0.0").status).toBe("ok"); - expect(checkNode("22.11.0")).toMatchObject({ status: "failed", fix: "install Node 24 or newer from https://nodejs.org" }); - expect(checkDataDirectory("/data", { path: "/data", bytes: MIN_FREE_BYTES }).status).toBe("ok"); - expect(checkDataDirectory("/data", { path: "/data", bytes: 5 * 1024 ** 3 })).toEqual({ - id: "disk", - label: "data directory", - status: "failed", - detail: "5.0 GiB free at /data, less than 20.0 GiB", - fix: "free some space, or set INVISIBLE_DOTS_HOME to a directory on a larger disk", - }); - }); - - it("refuses a data directory QEMU cannot be given, before any build or Dot does", async () => { - // The default home sits under the account name; on Windows QEMU cannot open a path with an accent in it. - const { deps } = healthyDoctor({ home: "C:\\Users\\Jos\u00e9\\.invisible-dots" }); - expect(find(await runDoctor(deps), "disk")).toMatchObject({ - status: "failed", - fix: "set INVISIBLE_DOTS_HOME to a directory whose path is plain ASCII without commas", - }); - expect(checkDataDirectory("/srv/a,b", { path: "/srv", bytes: MIN_FREE_BYTES }).detail).toMatch(/comma/); - }); - - it("answers the OpenRouter row from what the control plane knows", () => { - expect(openRouterCheck(true)).toEqual({ id: "openrouter", label: "OpenRouter key", status: "ok", detail: "stored" }); - expect(openRouterCheck(false)).toEqual({ - id: "openrouter", - label: "OpenRouter key", - status: "missing", - detail: "no key stored", - fix: "invisible-dots secret openrouter", - }); - }); - - it("is ok as a whole only when every check is", async () => { - const healthy = await runDoctor(healthyDoctor().deps); - expect(doctorAnswer(healthy)).toEqual({ ok: true, checks: healthy }); - const missing = await runDoctor(healthyDoctor({ openRouterKey: async () => openRouterCheck(false) }).deps); - expect(doctorAnswer(missing).ok).toBe(false); - expect(doctorAnswer([])).toEqual({ ok: true, checks: [] }); - }); -}); diff --git a/apps/vm-manager/test/fakes.ts b/apps/vm-manager/test/fakes.ts deleted file mode 100644 index ad37058a..00000000 --- a/apps/vm-manager/test/fakes.ts +++ /dev/null @@ -1,245 +0,0 @@ -import { createHmac } from "node:crypto"; -import { appendFileSync, writeFileSync } from "node:fs"; -import { readFile } from "node:fs/promises"; -import { createServer, type Server } from "node:http"; -import type { Socket } from "node:net"; -import { GUEST_PROOF_CONTEXT, type ProcessPresence } from "@invisible-dots/shared"; -import { - CommandError, - type CommandResult, - type CommandRunner, - type DetachedProcess, - type ProcessControl, - type ProcessExit, - type RunOptions, - type SpawnDetachedOptions, -} from "../src/index.js"; - -export interface RecordedCall { - command: string; - args: string[]; - options: RunOptions | undefined; -} - -export type Handler = (command: string, args: string[]) => CommandResult | undefined | Promise; - -/** A failure as the runner would report it. */ -export function fail(command: string, args: string[], stderr: string, exitCode = 1): never { - throw new CommandError({ command, args, exitCode, stderr }); -} - -/** Records every command; `handler` answers them (undefined means success with empty output). */ -export class FakeRunner implements CommandRunner { - calls: RecordedCall[] = []; - constructor(public handler: Handler = () => undefined) {} - - async run(command: string, args: readonly string[], options?: RunOptions): Promise { - const copy = [...args]; - this.calls.push({ command, args: copy, options }); - return (await this.handler(command, copy)) ?? { stdout: "", stderr: "" }; - } - - /** Every call as one "command arg arg" line, for readable assertions. */ - lines(): string[] { - return this.calls.map((call) => [call.command, ...call.args].join(" ")); - } -} - -/** How a fake QEMU behaves when spawned. */ -export type FakeBehaviour = - /** Exits at once after writing `output` to its log, like QEMU refusing its command line or the accelerator. */ - | { kind: "exit"; code: number; output: string } - /** Sets up the port forward, then exits a moment later: QEMU refusing -cpu host when it builds the machine. */ - | { kind: "exitAfterForward"; code: number; output: string } - /** Keeps running without ever listening on the guest port. */ - | { kind: "hang" } - /** - * Runs and forwards the guest port. `guestUp: false` is a VM whose guest - * never boots (the forward accepts and drops every connection); - * `ignorePoweroff` a guest that answers 202 and then does not power off. - * `serial` and `qemuOutput` are written to the serial console and QEMU's log. - */ - | { kind: "run"; guestUp?: boolean; ignorePoweroff?: boolean; serial?: string; qemuOutput?: string }; - -export interface FakeVm { - pid: number; - name: string; - guestPort: number; - /** The Dot token the guest read from its seed. */ - token: string; - guestUp: boolean; - ignorePoweroff: boolean; - /** Guest requests as "METHOD /path", in order. */ - requests: string[]; - resolveExit: (exit: ProcessExit) => void; - server?: Server; - sockets: Set; -} - -/** The value after `flag` in an argv. */ -export function argAfter(args: readonly string[], flag: string): string { - const index = args.indexOf(flag); - if (index < 0 || index + 1 >= args.length) throw new Error(`no ${flag} in ${args.join(" ")}`); - return args[index + 1]!; -} - -/** The host port of the forward in an argv. */ -export function guestPortOf(args: readonly string[]): number { - return Number(/hostfwd=tcp:127\.0\.0\.1:(\d+)-:1024$/.exec(argAfter(args, "-netdev"))![1]); -} - -/** - * The token a guest would read from its seed: the seed ISO QEMU was given - * holds config.json as a quoted YAML scalar inside user-data. - */ -export async function tokenInSeed(args: readonly string[]): Promise { - const drive = args.find((arg) => arg.includes("seed.iso")); - const path = drive && /file=([^,]+)/.exec(drive)?.[1]; - if (!path) throw new Error(`no seed drive in ${args.join(" ")}`); - const match = /token\\?":\\?"([A-Za-z0-9_-]+)/.exec((await readFile(path)).toString("latin1")); - if (!match) throw new Error(`no token in ${path}`); - return match[1]!; -} - -function send(response: import("node:http").ServerResponse, status: number, body: unknown): void { - response.writeHead(status, { "content-type": "application/json" }).end(JSON.stringify(body)); -} - -/** - * dot-agentd's `GET /v1/proof` (section 5.1), computed the guest's way, with - * the token the guest read from its seed. Answers true when it served it. - */ -export function serveProof( - request: import("node:http").IncomingMessage, - response: import("node:http").ServerResponse, - token: string, -): boolean { - const url = new URL(request.url ?? "/", "http://guest"); - if (request.method !== "GET" || url.pathname !== "/v1/proof") return false; - const nonce = url.searchParams.get("nonce") ?? ""; - send(response, 200, { proof: createHmac("sha256", token).update(`${GUEST_PROOF_CONTEXT}${nonce}`).digest("hex") }); - return true; -} - -/** - * A pretend QEMU: the ProcessControl the vm-manager spawns and kills it - * with, and, like QEMU's user networking, a real TCP listener on the guest - * port of its argv for as long as it "runs", with dot-agentd's health and - * poweroff routes behind it. It parses the real argv, so a wrong argv breaks - * these tests too. - */ -export class FakeQemuHost implements ProcessControl { - spawned: { command: string; args: string[]; logPath: string; cwd: string; env: Record }[] = []; - vms = new Map(); - alive = new Set(); - killed: number[] = []; - /** Decides each spawn's behaviour; `attempt` counts spawns from 1. */ - behaviour: (args: string[], attempt: number) => FakeBehaviour = () => ({ kind: "run" }); - /** Pids that exist for this user but are not QEMU (a recycled pid). */ - foreignPids = new Set(); - /** Pids that exist and belong to another user (process.kill answers EPERM). */ - otherUserPids = new Set(); - private nextPid = 4000; - - async spawnDetached(command: string, args: readonly string[], options: SpawnDetachedOptions): Promise { - const argv = [...args]; - this.spawned.push({ command, args: argv, logPath: options.logPath, cwd: options.cwd, env: options.env }); - const pid = this.nextPid++; - const behaviour = this.behaviour(argv, this.spawned.length); - let resolveExit!: (exit: ProcessExit) => void; - const exited = new Promise((resolve) => (resolveExit = resolve)); - if (behaviour.kind === "exit") { - appendFileSync(options.logPath, behaviour.output); - resolveExit({ code: behaviour.code, signal: null }); - return { pid, exited }; - } - const run = behaviour.kind === "run" ? behaviour : undefined; - const vm: FakeVm = { - pid, - name: argAfter(argv, "-name"), - guestPort: guestPortOf(argv), - token: await tokenInSeed(argv), - guestUp: run?.guestUp ?? true, - ignorePoweroff: run?.ignorePoweroff ?? false, - requests: [], - resolveExit, - sockets: new Set(), - }; - this.alive.add(pid); - this.vms.set(pid, vm); - if (run?.qemuOutput) appendFileSync(options.logPath, run.qemuOutput); - if (run?.serial) writeFileSync(/^file:(.*)$/.exec(argAfter(argv, "-serial"))![1]!, run.serial); - if (behaviour.kind !== "hang") await this.openForward(vm); - if (behaviour.kind === "exitAfterForward") { - setTimeout(() => { - appendFileSync(options.logPath, behaviour.output); - this.terminate(vm, { code: behaviour.code, signal: null }); - }, 2); - } - return { pid, exited }; - } - - /** QEMU's user networking listening on the guest port, with dot-agentd behind it. */ - protected async openForward(vm: FakeVm): Promise { - const server = createServer((request, response) => { - vm.requests.push(`${request.method} ${request.url}`); - // Nothing listens on port 1024 in a guest that is not up: the forward drops the connection. - if (!vm.guestUp) { - request.socket.destroy(); - return; - } - if (serveProof(request, response, vm.token)) return; - if (request.headers.authorization !== `Bearer ${vm.token}`) return send(response, 401, { error: "unauthorized", message: "bad token" }); - if (request.method === "GET" && request.url === "/v1/health") { - return send(response, 200, { agentd: "ok", agent: { status: "ok", state: "IDLE", openrouter_configured: false }, uptime_s: 1 }); - } - if (request.method === "POST" && request.url === "/v1/system/poweroff") { - send(response, 202, { status: "powering_off" }); - if (!vm.ignorePoweroff) setTimeout(() => this.terminate(vm), 5); - return; - } - send(response, 404, { error: "not_found", message: `${request.method} ${request.url}` }); - }); - server.on("connection", (socket) => { - vm.sockets.add(socket); - socket.on("close", () => vm.sockets.delete(socket)); - }); - await new Promise((resolve, reject) => { - server.once("error", reject); - server.listen(vm.guestPort, "127.0.0.1", () => resolve()); - }); - vm.server = server; - } - - /** QEMU's listener goes away with the process. */ - protected closeForward(vm: FakeVm): void { - for (const socket of vm.sockets) socket.destroy(); - vm.server?.close(); - } - - presence(pid: number): ProcessPresence { - if (this.alive.has(pid) || this.foreignPids.has(pid)) return "ours"; - return this.otherUserPids.has(pid) ? "foreign" : "gone"; - } - - kill(pid: number): void { - this.killed.push(pid); - const vm = this.vms.get(pid); - if (vm) this.terminate(vm, { code: null, signal: "SIGKILL" }); - } - - /** The one running VM, for tests that start one. */ - only(): FakeVm { - const running = [...this.vms.values()]; - if (running.length !== 1) throw new Error(`expected one fake VM, there are ${running.length}`); - return running[0]!; - } - - /** The QEMU process ends: its listener closes and its exit is reported. */ - terminate(vm: FakeVm, exit: ProcessExit = { code: 0, signal: null }): void { - if (!this.alive.delete(vm.pid)) return; - this.vms.delete(vm.pid); - this.closeForward(vm); - vm.resolveExit(exit); - } -} diff --git a/apps/vm-manager/test/guest-client.test.ts b/apps/vm-manager/test/guest-client.test.ts deleted file mode 100644 index e888cc10..00000000 --- a/apps/vm-manager/test/guest-client.test.ts +++ /dev/null @@ -1,474 +0,0 @@ -import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; -import type { AddressInfo } from "node:net"; -import { FILE_TOO_LARGE, GUEST_UNPROVEN, REFUSED_PROBLEM_MAX, type HealthAnswer, type OutboundEvent, type RefusedEvent } from "@invisible-dots/shared"; -import { afterEach, describe, expect, it } from "vitest"; -import { GuestClient, GuestHealthTimeoutError, GuestRequestError, guestProof, waitForGuestHealth } from "../src/index.js"; - -const TOKEN = "dot-token-123"; - -interface Seen { - method: string; - url: string; - auth: string | undefined; - body: string; -} - -let server: Server | undefined; -afterEach(async () => { - server?.closeAllConnections(); - await new Promise((resolve) => (server ? server.close(() => resolve()) : resolve())); - server = undefined; -}); - -/** - * A dot-agentd stand-in. It answers `GET /v1/proof` the guest's way (the - * HMAC of the nonce under TOKEN), or, as `impostor`, with a proof it cannot - * compute. Proof requests are kept apart from `seen`, which lists every - * other request. - */ -async function serve( - handler: (req: IncomingMessage, res: ServerResponse, seen: Seen) => void, - options: { impostor?: boolean } = {}, -): Promise<{ port: number; seen: Seen[]; proofs: Seen[] }> { - const seen: Seen[] = []; - const proofs: Seen[] = []; - server = createServer((req, res) => { - const chunks: Buffer[] = []; - req.on("data", (chunk: Buffer) => chunks.push(chunk)); - req.on("end", () => { - const entry = { method: req.method!, url: req.url!, auth: req.headers.authorization, body: Buffer.concat(chunks).toString() }; - const url = new URL(req.url!, "http://guest"); - if (url.pathname === "/v1/proof") { - proofs.push(entry); - const nonce = url.searchParams.get("nonce") ?? ""; - return json(res, 200, { proof: options.impostor ? "0".repeat(64) : guestProof(TOKEN, nonce) }); - } - seen.push(entry); - if (req.headers.authorization !== `Bearer ${TOKEN}`) { - res.writeHead(401, { "content-type": "application/json" }).end(JSON.stringify({ error: "unauthorized", message: "bad token" })); - return; - } - handler(req, res, entry); - }); - }); - await new Promise((resolve) => server!.listen(0, "127.0.0.1", resolve)); - return { port: (server!.address() as AddressInfo).port, seen, proofs }; -} - -function json(res: ServerResponse, status: number, body: unknown) { - res.writeHead(status, { "content-type": "application/json" }).end(JSON.stringify(body)); -} - -function event(seq: number): OutboundEvent { - return { seq, id: `evt_${seq}`, type: "automation.next_run", ts: "2026-10-02T10:00:00.000Z", data: { next_run_at_ms: seq } }; -} - -describe("GuestClient", () => { - it("calls every dot-agentd route with the bearer token", async () => { - const { port, seen } = await serve((req, res) => { - if (req.url === "/v1/system") return json(res, 200, { hostname: "h", uptime_s: 1, cpus: 2, mem_total_bytes: 1, mem_available_bytes: 1, disk_total_bytes: 1, disk_free_bytes: 1 }); - if (req.url === "/v1/exec") return json(res, 200, { exit_code: 0, stdout: "hi\n", stderr: "", timed_out: false }); - if (req.url!.startsWith("/v1/files/list")) return json(res, 200, { entries: [] }); - if (req.url!.startsWith("/v1/files") && req.method === "GET") return res.writeHead(200).end(Buffer.from([1, 2, 3])); - if (req.url!.startsWith("/v1/files") && req.method === "PUT") return res.writeHead(204).end(); - if (req.url === "/v1/screenshot") return res.writeHead(200, { "content-type": "image/png" }).end(Buffer.from("PNG")); - if (req.url === "/v1/system/poweroff" && req.method === "POST") return json(res, 202, { status: "powering_off" }); - json(res, 404, { error: "not_found", message: req.url }); - }); - const client = new GuestClient(port, TOKEN); - expect((await client.system()).hostname).toBe("h"); - expect(await client.exec({ command: "echo hi", timeout_ms: 1000 })).toMatchObject({ exit_code: 0, stdout: "hi\n" }); - expect(await client.readFile("workspace/a b.txt")).toEqual(Buffer.from([1, 2, 3])); - await client.writeFile("/home/dot/x&y", "content"); - expect(await client.listFiles(".")).toEqual({ entries: [] }); - expect((await client.screenshot()).toString()).toBe("PNG"); - await client.powerOff(); - - expect(seen.map((s) => `${s.method} ${s.url}`)).toEqual([ - "GET /v1/system", - "POST /v1/exec", - "GET /v1/files?path=workspace%2Fa%20b.txt", - "PUT /v1/files?path=%2Fhome%2Fdot%2Fx%26y", - "GET /v1/files/list?path=.", - "GET /v1/screenshot", - "POST /v1/system/poweroff", - ]); - expect(seen.every((s) => s.auth === `Bearer ${TOKEN}`)).toBe(true); - expect(JSON.parse(seen[1]!.body)).toEqual({ command: "echo hi", timeout_ms: 1000 }); - expect(seen[3]!.body).toBe("content"); - }); - - it("calls the agent routes under /v1/agent", async () => { - const { port, seen } = await serve((req, res) => { - switch (`${req.method} ${req.url}`) { - case "POST /v1/agent/secrets": - case "PUT /v1/agent/config": - case "DELETE /v1/agent/browser-identities/shop-abc123": - case "POST /v1/agent/prepare-sleep": - return res.writeHead(204).end(); - case "POST /v1/agent/events": - return json(res, 202, { accepted: true }); - case "GET /v1/agent/state": - return json(res, 200, { state: "IDLE", current_task_id: null, pending_approval: null }); - case "GET /v1/agent/browser-identities": - return json(res, 200, { identities: [] }); - case "POST /v1/agent/browser-identities": - return json(res, 201, { id: "shop-abc123", name: "shop", createdAt: "x", lastUsedAt: null, status: "available", profilePath: "/p" }); - default: - return json(res, 404, { error: "not_found", message: req.url }); - } - }); - const client = new GuestClient(port, TOKEN); - await client.pushSecrets({ openrouter_api_key: "sk-or-test", mcp_secrets: {} }); - await client.putConfig({ name: "n" } as never); - expect(await client.postEvent({ id: "e1", type: "user.message", ts: "2026-10-02T10:00:00Z", data: { text: "hi" } })).toEqual({ accepted: true }); - expect((await client.state()).state).toBe("IDLE"); - expect(await client.listBrowserIdentities()).toEqual({ identities: [] }); - expect((await client.createBrowserIdentity({ name: "shop" })).id).toBe("shop-abc123"); - await client.deleteBrowserIdentity("shop-abc123"); - await client.prepareSleep(); - expect(JSON.parse(seen[0]!.body)).toEqual({ openrouter_api_key: "sk-or-test", mcp_secrets: {} }); - expect(seen.map((s) => s.url)).toContain("/v1/agent/browser-identities/shop-abc123"); - }); - - it("asks an identity's frame as bytes and closes it with a POST, both under /v1/agent", async () => { - const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xd9]); - const { port, seen } = await serve((req, res) => { - switch (`${req.method} ${req.url}`) { - case "GET /v1/agent/browser-identities/shop-abc123/frame": - return res.writeHead(200, { "content-type": "image/jpeg" }).end(jpeg); - case "POST /v1/agent/browser-identities/shop-abc123/close": - return res.writeHead(204).end(); - case "GET /v1/agent/browser-identities/shut-abc123/frame": - return json(res, 409, { error: "not_open", message: "identity shut-abc123 is not open" }); - default: - return json(res, 404, { error: "not_found", message: req.url }); - } - }); - const client = new GuestClient(port, TOKEN); - - expect(await client.getBrowserIdentityFrame("shop-abc123")).toEqual(jpeg); - await client.closeBrowserIdentity("shop-abc123"); - await expect(client.getBrowserIdentityFrame("shut-abc123")).rejects.toMatchObject({ status: 409, code: "not_open" }); - - expect(seen.map((s) => `${s.method} ${s.url}`)).toEqual([ - "GET /v1/agent/browser-identities/shop-abc123/frame", - "POST /v1/agent/browser-identities/shop-abc123/close", - "GET /v1/agent/browser-identities/shut-abc123/frame", - ]); - expect(seen.every((s) => s.auth === `Bearer ${TOKEN}`)).toBe(true); - }); - - it("calls the tool and skill routes with the bearer token", async () => { - const { port, seen } = await serve((req, res) => { - switch (`${req.method} ${req.url}`) { - case "GET /v1/agent/tools": - return json(res, 200, { tools: [{ name: "exec", permission: "computer.exec", offered: true, description: "Run." }] }); - case "GET /v1/agent/skills": - return json(res, 200, { skills: [{ name: "shop-login", description: "Log in.", source: "dot", path: "/home/dot/skills/shop-login/SKILL.md", content: "x" }] }); - default: - return json(res, 404, { error: "not_found", message: req.url }); - } - }); - const client = new GuestClient(port, TOKEN); - - expect((await client.listTools()).tools.map((tool) => tool.name)).toEqual(["exec"]); - expect((await client.listSkills()).skills.map((skill) => skill.name)).toEqual(["shop-login"]); - expect(seen.map((s) => s.auth)).toEqual([`Bearer ${TOKEN}`, `Bearer ${TOKEN}`]); - }); - - it("passes the guest's outside_home refusal of a file path on", async () => { - const { port } = await serve((_req, res) => json(res, 403, { error: "outside_home", message: "the path leads outside /home/dot" })); - const client = new GuestClient(port, TOKEN); - expect(await client.readFile("environ").catch((e: unknown) => e)).toMatchObject({ name: "GuestRequestError", status: 403, code: "outside_home" }); - expect(await client.listFiles("etc").catch((e: unknown) => e)).toMatchObject({ status: 403, code: "outside_home" }); - expect(await client.writeFile("etc/x", "y").catch((e: unknown) => e)).toMatchObject({ status: 403, code: "outside_home" }); - }); - - it("turns error bodies into GuestRequestError", async () => { - const { port } = await serve((_req, res) => json(res, 409, { error: "computer_busy", message: "try later" })); - const error = await new GuestClient(port, TOKEN).state().catch((e: unknown) => e); - expect(error).toBeInstanceOf(GuestRequestError); - expect(error).toMatchObject({ status: 409, code: "computer_busy" }); - expect((error as Error).message).toContain("computer_busy: try later"); - - // A client holding another token cannot check the guest's proof: it never sends that token. - const unproven = await new GuestClient(port, "wrong").health().catch((e: unknown) => e); - expect(unproven).toMatchObject({ code: GUEST_UNPROVEN }); - }); - - it("computes the proof dot-agentd computes (the known answer its Go test checks too)", () => { - expect(guestProof("dot-token-123", "00112233445566778899aabbccddeeff")).toBe("b1cb86aa470fe878a6366f62ea4493011c01d22736a9c89d3e7f88582920d1b9"); - }); - - it("asks a listener for its proof before the token goes anywhere, once per client", async () => { - const { port, seen, proofs } = await serve((_req, res) => json(res, 200, health("ok"))); - const client = new GuestClient(port, TOKEN); - await client.health(); - await client.health(); - expect(proofs).toHaveLength(1); - expect(proofs[0]!.auth).toBeUndefined(); - expect(proofs[0]!.url).toMatch(/^\/v1\/proof\?nonce=[0-9a-f]{32}$/); - expect(seen).toHaveLength(2); - }); - - it("never sends the token to a listener that cannot prove it holds it", async () => { - // What takes over a stale guest port after a host restart or a QEMU that died. - const { port, seen, proofs } = await serve((_req, res) => json(res, 200, health("ok")), { impostor: true }); - const client = new GuestClient(port, TOKEN); - await expect(client.pushSecrets({ openrouter_api_key: "sk-or-must-not-leak", mcp_secrets: {} })).rejects.toMatchObject({ code: GUEST_UNPROVEN }); - await expect(waitForGuestHealth(client, { intervalMs: 5, timeoutMs: 5000 })).rejects.toMatchObject({ code: GUEST_UNPROVEN }); - expect(proofs.length).toBeGreaterThan(0); - expect(proofs.every((p) => p.auth === undefined)).toBe(true); - expect(seen).toEqual([]); - }); - - it("streams events, reconnecting with ?after= and never repeating one", async () => { - let connection = 0; - const { port, seen } = await serve((req, res) => { - connection++; - res.writeHead(200, { "content-type": "text/event-stream" }); - if (connection === 1) { - // Two events, then the connection drops mid-message. - res.write(`: hello\n\nid: 1\ndata: ${JSON.stringify(event(1))}\n\n`); - res.write(`id: 2\r\ndata: ${JSON.stringify(event(2))}\r\n\r\n`); - res.end(`id: 3\ndata: {"seq":3`); - } else { - const after = Number(new URL(req.url!, "http://x").searchParams.get("after")); - // A server replaying one too many must not produce a duplicate. - for (let seq = after; seq <= after + 2; seq++) res.write(`id: ${seq}\ndata: ${JSON.stringify(event(seq))}\n\n`); - } - }); - const client = new GuestClient(port, TOKEN); - const controller = new AbortController(); - const got: number[] = []; - const reconnects: number[] = []; - for await (const evt of client.events({ signal: controller.signal, reconnectDelayMs: 10, onReconnect: (info) => reconnects.push(info.after) })) { - got.push(evt.seq); - if (got.length === 4) controller.abort(); - } - expect(got).toEqual([1, 2, 3, 4]); - expect(reconnects).toEqual([2]); - expect(seen.map((s) => s.url)).toEqual(["/v1/agent/events/stream?after=0", "/v1/agent/events/stream?after=2"]); - }); - - it("hands over a message that is not an event the host knows, in order with the events, and resumes after it", async () => { - let connection = 0; - const bad = { ...event(2), type: "approval.requested", data: { approval_id: "a1", tool: "exec", permission: "computer.teleport", arguments: {}, reason: "x" } }; - const { port, seen } = await serve((req, res) => { - connection++; - res.writeHead(200, { "content-type": "text/event-stream" }); - if (connection === 1) { - res.write(`id: 1\ndata: ${JSON.stringify(event(1))}\n\n`); - res.write(`id: 2\ndata: ${JSON.stringify(bad)}\n\n`); - res.write(`id: x\ndata: this is not json\n\n`); - res.end(`id: 3\ndata: ${JSON.stringify(event(3))}\n\n`); - } else { - const after = Number(new URL(req.url!, "http://x").searchParams.get("after")); - for (let seq = after + 1; seq <= after + 2; seq++) res.write(`id: ${seq}\ndata: ${JSON.stringify(event(seq))}\n\n`); - } - }); - const controller = new AbortController(); - const order: string[] = []; - const refused: RefusedEvent[] = []; - const client = new GuestClient(port, TOKEN); - for await (const evt of client.events({ - signal: controller.signal, - reconnectDelayMs: 10, - onRefused: async (info) => { - // Awaited before the stream goes on: the event after it is not yielded until this settles. - await new Promise((resolve) => setTimeout(resolve, 5)); - order.push(`refused ${info.seq}`); - refused.push(info); - }, - })) { - order.push(`event ${evt.seq}`); - if (evt.seq === 5) controller.abort(); - } - expect(order).toEqual(["event 1", "refused 2", "refused null", "event 3", "event 4", "event 5"]); - expect(refused).toEqual([ - { seq: 2, type: "approval.requested", problem: expect.stringContaining("data.permission") }, - { seq: null, type: null, problem: expect.stringContaining("JSON") }, - ]); - expect(refused[0]!.problem.length).toBeLessThanOrEqual(REFUSED_PROBLEM_MAX); - // The reconnect resumed after the last event, the refused one behind it, so it was not handed over again. - expect(seen.map((s) => s.url)).toEqual(["/v1/agent/events/stream?after=0", "/v1/agent/events/stream?after=3"]); - }); - - it("goes on past a refused message when nobody asked to hear of it", async () => { - const { port } = await serve((_req, res) => { - res.writeHead(200, { "content-type": "text/event-stream" }); - res.write(`id: 1\ndata: {"seq":1}\n\n`); - res.write(`id: 2\ndata: ${JSON.stringify(event(2))}\n\n`); - }); - const client = new GuestClient(port, TOKEN); - for await (const evt of client.events()) { - expect(evt.seq).toBe(2); - break; - } - }); - - it("resumes from the given cursor and stops when the consumer breaks", async () => { - const { port, seen } = await serve((_req, res) => { - res.writeHead(200, { "content-type": "text/event-stream" }); - res.write(`id: 8\ndata: ${JSON.stringify(event(8))}\n\n`); - }); - for await (const evt of new GuestClient(port, TOKEN).events({ after: 7 })) { - expect(evt.seq).toBe(8); - break; - } - expect(seen[0]!.url).toBe("/v1/agent/events/stream?after=7"); - }); - - it("does not retry the stream on a refused token", async () => { - const { port } = await serve((_req, res) => json(res, 401, { error: "unauthorized", message: "bad token" })); - const stream = new GuestClient(port, TOKEN).events({ reconnectDelayMs: 1 }); - await expect(stream.next()).rejects.toMatchObject({ status: 401 }); - }); - - it("ends the stream when nothing listens on the guest port: the VM is gone, waiting there cannot help", async () => { - const port = await closedPort(); - const reconnects: number[] = []; - const stream = new GuestClient(port, TOKEN).events({ reconnectDelayMs: 1, onReconnect: (info) => reconnects.push(info.attempt) }); - await expect(stream.next()).rejects.toMatchObject({ code: "ECONNREFUSED" }); - expect(reconnects).toEqual([]); - }); -}); - -function health(agent: "ok" | "starting" | "down"): HealthAnswer { - return { - agentd: "ok", - uptime_s: 3, - agent: - agent === "down" - ? { status: "down" } - : { - status: agent, - state: "IDLE", - openrouter_configured: false, - browser: { identities: 0, open: 0 }, - checks: { filesystem_writable: true, network_reachable: true, browser_installed: true }, - }, - }; -} - -/** A port nothing listens on: bound by the kernel, then released. */ -async function closedPort(): Promise { - const probe = createServer(); - await new Promise((resolve) => probe.listen(0, "127.0.0.1", resolve)); - const port = (probe.address() as AddressInfo).port; - await new Promise((resolve) => probe.close(() => resolve())); - return port; -} - -describe("GuestClient transport", () => { - it("talks to 127.0.0.1 on the given port and reads one identity", async () => { - const { port, seen } = await serve((req, res) => - json(res, 200, { id: "a b", name: "a", createdAt: "x", lastUsedAt: null, status: "available", profilePath: "/p" }), - ); - const client = new GuestClient(port, TOKEN); - expect(client.address).toBe(`127.0.0.1:${port}`); - expect((await client.getBrowserIdentity("a b")).name).toBe("a"); - expect(seen[0]!.url).toBe("/v1/agent/browser-identities/a%20b"); - }); - - describe("readFile with a size limit", () => { - it("returns a file within the limit, whole", async () => { - const { port } = await serve((req, res) => res.writeHead(200).end(Buffer.alloc(100, 7))); - expect(await new GuestClient(port, TOKEN).readFile("a", { maxBytes: 100 })).toEqual(Buffer.alloc(100, 7)); - }); - - it("refuses a file whose announced length is over it, before reading it", async () => { - const { port } = await serve((req, res) => { - res.writeHead(200, { "content-length": 101 }); - res.write(Buffer.alloc(10)); - // The rest is never sent: a client that waited for it would run into the test's timeout. - }); - const error = await new GuestClient(port, TOKEN).readFile("a", { maxBytes: 100 }).catch((e: unknown) => e); - expect(error).toBeInstanceOf(GuestRequestError); - expect(error).toMatchObject({ status: 413, code: FILE_TOO_LARGE }); - expect((error as Error).message).toContain("larger than 100 bytes"); - }); - - it("stops reading a body of unannounced length once it passes the limit", async () => { - const { port } = await serve((req, res) => { - // Chunked: no content-length, so only the running count can tell. - res.writeHead(200); - res.write(Buffer.alloc(60)); - res.write(Buffer.alloc(60)); - res.write(Buffer.alloc(60)); - }); - const error = await new GuestClient(port, TOKEN).readFile("a", { maxBytes: 100 }).catch((e: unknown) => e); - expect(error).toMatchObject({ status: 413, code: FILE_TOO_LARGE }); - }); - - it("keeps the guest's own error for a failed read, whatever the limit", async () => { - const { port } = await serve((req, res) => json(res, 404, { error: "not_found", message: "no such file: " + "x".repeat(300) })); - const error = await new GuestClient(port, TOKEN).readFile("a", { maxBytes: 10 }).catch((e: unknown) => e); - expect(error).toMatchObject({ status: 404, code: "not_found" }); - }); - - it("reads without a limit as before", async () => { - const { port } = await serve((req, res) => res.writeHead(200).end(Buffer.alloc(5000, 1))); - expect((await new GuestClient(port, TOKEN).readFile("a")).length).toBe(5000); - }); - }); - - it("names the guest port when nothing listens there", async () => { - const port = await closedPort(); - const error = await new GuestClient(port, TOKEN).system().catch((e: unknown) => e); - expect(error).toBeInstanceOf(GuestRequestError); - expect((error as Error).message).toContain(`127.0.0.1:${port}`); - expect((error as GuestRequestError).code).toBe("ECONNREFUSED"); - }); - - it("times out a request that gets no answer", async () => { - const { port } = await serve(() => {}); - // The handler above only answers 401s; this request carries the right token and hangs. - const error = await new GuestClient(port, TOKEN, { timeoutMs: 100 }).system().catch((e: unknown) => e); - expect((error as Error).message).toContain("within 100 ms"); - }); - - it("refuses a port that is not a TCP port", () => { - expect(() => new GuestClient(0, TOKEN)).toThrow(/TCP port/); - expect(() => new GuestClient(70000, TOKEN)).toThrow(/TCP port/); - expect(() => new GuestClient(1234, "")).toThrow(/token/); - }); -}); - -describe("waitForGuestHealth", () => { - it("retries while the guest boots and returns once the agent is ok", async () => { - const answers = [health("down"), health("starting"), health("ok")]; - const { port, seen } = await serve((_req, res) => json(res, 200, answers.shift() ?? health("ok"))); - const answer = await waitForGuestHealth(new GuestClient(port, TOKEN), { intervalMs: 5, timeoutMs: 5000 }); - expect(answer.agent.status).toBe("ok"); - expect(seen.length).toBe(3); - }); - - it("stops at dot-agentd when asked to", async () => { - const { port } = await serve((_req, res) => json(res, 200, health("down"))); - const answer = await waitForGuestHealth(new GuestClient(port, TOKEN), { until: "agentd", intervalMs: 5 }); - expect(answer.agentd).toBe("ok"); - }); - - it("keeps trying through refused connections, then reports the last error", async () => { - const port = await closedPort(); - const error = await waitForGuestHealth(new GuestClient(port, TOKEN), { intervalMs: 5, timeoutMs: 60 }).catch((e: unknown) => e); - expect(error).toBeInstanceOf(GuestHealthTimeoutError); - expect((error as Error).message).toMatch(/not healthy within 60 ms.*ECONNREFUSED|not healthy within 60 ms.*connect/); - }); - - it("fails at once on a refused token", async () => { - const { port } = await serve((_req, res) => json(res, 401, { error: "unauthorized", message: "bad token" })); - await expect(waitForGuestHealth(new GuestClient(port, TOKEN), { intervalMs: 5 })).rejects.toMatchObject({ status: 401 }); - }); - - it("stops when the check says the VM is gone", async () => { - const port = await closedPort(); - let calls = 0; - const check = () => { - if (++calls === 3) throw new Error("QEMU exited"); - }; - await expect(waitForGuestHealth(new GuestClient(port, TOKEN), { intervalMs: 5, check })).rejects.toThrow("QEMU exited"); - }); -}); diff --git a/apps/vm-manager/test/host-doctor.test.ts b/apps/vm-manager/test/host-doctor.test.ts deleted file mode 100644 index dac6a906..00000000 --- a/apps/vm-manager/test/host-doctor.test.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { freeSpace, hostDoctorDeps, locateQemu } from "../src/index.js"; -import { ok } from "./doctor-fakes.js"; - -let dir: string; -beforeEach(async () => { - dir = await mkdtemp(join(tmpdir(), "idots-doctor-")); -}); -afterEach(async () => { - await rm(dir, { recursive: true, force: true }); -}); - -async function program(name: string): Promise { - const path = join(dir, "qemu", name); - await mkdir(join(dir, "qemu"), { recursive: true }); - await writeFile(path, "#!/bin/sh\n"); - await chmod(path, 0o755); - return path; -} - -describe("locateQemu", () => { - it("finds each program separately, so a missing qemu-img still leaves qemu-system to be checked", async () => { - const system = await program("qemu-system-x86_64"); - const env = { INVISIBLE_DOTS_QEMU_DIR: join(dir, "qemu") }; - expect(await locateQemu(env)).toEqual({ system, searched: [join(dir, "qemu")], configured: true }); - - const img = await program("qemu-img"); - expect(await locateQemu(env)).toEqual({ system, img, searched: [join(dir, "qemu")], configured: true }); - }); - - it("names the places searched when nothing is found, and says whether INVISIBLE_DOTS_QEMU_DIR was the only one", async () => { - expect(await locateQemu({ INVISIBLE_DOTS_QEMU_DIR: join(dir, "nothing") })).toEqual({ searched: [join(dir, "nothing")], configured: true }); - expect(await locateQemu({ PATH: join(dir, "bin"), ProgramFiles: "", ProgramW6432: "" })).toEqual({ searched: [join(dir, "bin")], configured: false }); - }); -}); - -describe("freeSpace", () => { - it("measures the nearest directory that exists, since doctor creates nothing", async () => { - const home = join(dir, "not", "yet", "created"); - const space = await freeSpace(home); - expect(space.path).toBe(dir); - expect(space.bytes).toBeGreaterThan(0); - }); -}); - -describe("hostDoctorDeps", () => { - it("hands the caller's home, images and key to the report and reads the rest from this machine", async () => { - const deps = hostDoctorDeps({ - env: { INVISIBLE_DOTS_QEMU_DIR: join(dir, "nothing") }, - home: dir, - images: async () => [ok("golden-image", "golden image")], - openRouterKey: async () => ok("openrouter", "OpenRouter key"), - }); - expect(deps.nodeVersion).toBe(process.versions.node); - expect(deps.home).toBe(dir); - expect((await deps.freeSpace()).path).toBe(dir); - expect(await deps.findQemu()).toMatchObject({ searched: [join(dir, "nothing")], configured: true }); - expect((await deps.images())[0]?.id).toBe("golden-image"); - expect((await deps.openRouterKey()).status).toBe("ok"); - }); -}); diff --git a/apps/vm-manager/test/host.test.ts b/apps/vm-manager/test/host.test.ts deleted file mode 100644 index 424f49bc..00000000 --- a/apps/vm-manager/test/host.test.ts +++ /dev/null @@ -1,150 +0,0 @@ -import { readFileSync } from "node:fs"; -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { delimiter, join, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { - accelerator, - findQemu, - isSupportedQemuVersion, - MIN_QEMU_VERSION_TEXT, - parseQemuVersion, - QemuNotFoundError, - qemuSearchDirs, - officialQemuDir, - OFFICIAL_QEMU_SUBDIR, -} from "../src/index.js"; - -const repo = resolve(fileURLToPath(new URL(".", import.meta.url)), "../../.."); - -describe("accelerator", () => { - it("is KVM on Linux and WHPX on Windows", () => { - expect(accelerator("linux")).toBe("kvm"); - expect(accelerator("win32")).toBe("whpx"); - }); - - it("refuses other hosts instead of falling back to emulation", () => { - expect(() => accelerator("darwin")).toThrow(/does not run on darwin/); - expect(() => accelerator("freebsd")).toThrow(/invisible-dots setup/); - }); - - it("answers for the host the tests run on", () => { - if (process.platform === "linux" || process.platform === "win32") expect(["kvm", "whpx"]).toContain(accelerator()); - }); -}); - -describe("qemuSearchDirs", () => { - const abs = (name: string) => resolve(tmpdir(), name); - - it("searches INVISIBLE_DOTS_QEMU_DIR alone when it is set, as doctor says", () => { - expect(qemuSearchDirs({ INVISIBLE_DOTS_QEMU_DIR: abs("q"), PATH: [abs("a"), abs("b")].join(delimiter), ProgramW6432: abs("pf") })).toEqual([abs("q")]); - }); - - it("searches the official installer's directory before PATH, from the variable Windows sets", () => { - // An older QEMU on PATH must not shadow the one setup just installed. - const dirs = qemuSearchDirs({ ProgramW6432: abs("pf"), PATH: [abs("a"), abs("b")].join(delimiter) }); - expect(dirs).toEqual([join(abs("pf"), "qemu"), abs("a"), abs("b")]); - // Program Files on another drive, and a 32-bit view of the variables, are followed too. - expect(officialQemuDir({ ProgramFiles: abs("d-pf") })).toBe(join(abs("d-pf"), "qemu")); - // Linux has neither variable: no such directory, and no platform check. - expect(officialQemuDir({})).toBeUndefined(); - expect(qemuSearchDirs({ PATH: abs("a") })).toEqual([abs("a")]); - }); - - it("drops relative and duplicate entries and strips quotes", () => { - const dirs = qemuSearchDirs({ PATH: ["bin", abs("a"), `"${abs("b")}"`, abs("a"), ""].join(delimiter) }); - expect(dirs.slice(0, 2)).toEqual([abs("a"), abs("b")]); - }); - - it("reads the Windows spelling of PATH", () => { - expect(qemuSearchDirs({ Path: abs("w") })[0]).toBe(abs("w")); - }); -}); - -describe("findQemu", () => { - let dir: string; - beforeEach(async () => { - dir = await mkdtemp(join(tmpdir(), "idots-qemu-")); - }); - afterEach(async () => { - await rm(dir, { recursive: true, force: true }); - }); - - const fakeExists = (present: string[]) => async (path: string) => present.includes(path); - - it("finds both binaries, with or without .exe, in the first directory that has them", async () => { - const a = join(dir, "a"); - const b = join(dir, "b"); - const found = await findQemu( - { PATH: [a, b].join(delimiter) }, - fakeExists([join(b, "qemu-system-x86_64.exe"), join(b, "qemu-img.exe"), join(a, "qemu-img")]), - ); - expect(found).toEqual({ system: join(b, "qemu-system-x86_64.exe"), img: join(a, "qemu-img") }); - }); - - it("uses only INVISIBLE_DOTS_QEMU_DIR when it is set", async () => { - const configured = join(dir, "c"); - const onPath = join(dir, "p"); - await expect( - findQemu({ INVISIBLE_DOTS_QEMU_DIR: configured, PATH: onPath }, fakeExists([join(configured, "qemu-system-x86_64"), join(onPath, "qemu-img")])), - ).rejects.toThrow(/qemu-img was not found/); - }); - - it("names setup, doctor and the directories it searched", async () => { - const error = (await findQemu({ PATH: dir }, fakeExists([])).catch((e: unknown) => e)) as QemuNotFoundError; - expect(error).toBeInstanceOf(QemuNotFoundError); - expect(error.searched[0]).toBe(dir); - expect(error.message).toContain("invisible-dots setup"); - expect(error.message).toContain("invisible-dots doctor"); - expect(error.message).toContain("INVISIBLE_DOTS_QEMU_DIR"); - }); - - it("refuses a relative INVISIBLE_DOTS_QEMU_DIR", async () => { - await expect(findQemu({ INVISIBLE_DOTS_QEMU_DIR: "qemu" }, fakeExists([]))).rejects.toThrow(/not an absolute path/); - }); - - it("checks real files by default", async () => { - await mkdir(join(dir, "bin")); - for (const name of ["qemu-system-x86_64", "qemu-img"]) await writeFile(join(dir, "bin", name), "", { mode: 0o755 }); - expect(await findQemu({ INVISIBLE_DOTS_QEMU_DIR: join(dir, "bin") })).toEqual({ - system: join(dir, "bin", "qemu-system-x86_64"), - img: join(dir, "bin", "qemu-img"), - }); - }); -}); - -describe("QEMU version", () => { - it("parses both binaries' --version output", () => { - expect(parseQemuVersion("QEMU emulator version 11.1.0 (v11.1.0-12130-ge470268ff4)\nCopyright")).toEqual({ major: 11, minor: 1, micro: 0 }); - expect(parseQemuVersion("qemu-img version 8.2.2 (Debian 1:8.2.2+ds-0ubuntu1)")).toEqual({ major: 8, minor: 2, micro: 2 }); - expect(parseQemuVersion("nothing here")).toBeUndefined(); - }); - - it("supports 8.2 (Ubuntu 24.04's package) and newer", () => { - expect(MIN_QEMU_VERSION_TEXT).toBe("8.2"); - expect(isSupportedQemuVersion({ major: 8, minor: 2, micro: 0 })).toBe(true); - expect(isSupportedQemuVersion({ major: 8, minor: 2, micro: 2 })).toBe(true); - expect(isSupportedQemuVersion({ major: 10, minor: 0, micro: 0 })).toBe(true); - expect(isSupportedQemuVersion({ major: 8, minor: 1, micro: 9 })).toBe(false); - expect(isSupportedQemuVersion({ major: 7, minor: 2, micro: 0 })).toBe(false); - }); -}); - -describe("the pinned Windows installer", () => { - const pin = JSON.parse(readFileSync(join(repo, "virtualization/qemu/windows.json"), "utf8")); - - it("records version, URL and both hashes of one official build", () => { - expect(pin.url).toBe(`https://qemu.weilnetz.de/w64/${pin.build.slice(0, 4)}/qemu-w64-setup-${pin.build}.exe`); - expect(pin.sha512_url).toBe(pin.url.replace(/\.exe$/, ".sha512")); - expect(pin.sha256).toMatch(/^[0-9a-f]{64}$/); - expect(pin.sha512).toMatch(/^[0-9a-f]{128}$/); - expect(pin.version).toMatch(/^\d+\.\d+\.\d+$/); - expect(isSupportedQemuVersion(parseQemuVersion(`version ${pin.version}`)!)).toBe(true); - expect(pin.silent_args).toEqual(["/S"]); - }); - - it("installs where the vm-manager looks", () => { - expect(pin.install_subdir).toBe(OFFICIAL_QEMU_SUBDIR); - }); -}); diff --git a/apps/vm-manager/test/ports.test.ts b/apps/vm-manager/test/ports.test.ts deleted file mode 100644 index ae902fb6..00000000 --- a/apps/vm-manager/test/ports.test.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { createServer } from "node:net"; -import { describe, expect, it } from "vitest"; -import { pickFreePort } from "../src/index.js"; - -describe("pickFreePort", () => { - it("returns a port on 127.0.0.1 that can be bound again right away", async () => { - const port = await pickFreePort(); - expect(port).toBeGreaterThan(0); - expect(port).toBeLessThan(65536); - const server = createServer(); - await new Promise((resolve, reject) => { - server.once("error", reject); - server.listen(port, "127.0.0.1", resolve); - }); - await new Promise((resolve) => server.close(() => resolve())); - }); - - it("gives different ports to back-to-back calls most of the time", async () => { - const ports = new Set(await Promise.all([pickFreePort(), pickFreePort(), pickFreePort()])); - expect(ports.size).toBeGreaterThan(1); - }); -}); diff --git a/apps/vm-manager/test/qemu-args.test.ts b/apps/vm-manager/test/qemu-args.test.ts deleted file mode 100644 index 0999587f..00000000 --- a/apps/vm-manager/test/qemu-args.test.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { CPU_MODEL, machineArgs, qemuArgs, qemuPathProblem, VmManagerError, type QemuArgsSpec } from "../src/index.js"; - -const linux: QemuArgsSpec = { - dotId: "dot_01k6", - accelerator: "kvm", - cpus: 2, - memoryMiB: 4096, - diskPath: "/home/u/.invisible-dots/vms/dot_01k6/disk.qcow2", - seedPath: "/home/u/.invisible-dots/vms/dot_01k6/seed.iso", - runtimeIsoPath: "/home/u/.invisible-dots/images/runtime-7.iso", - guestPort: 40123, - serialLogPath: "/home/u/.invisible-dots/vms/dot_01k6/serial.log", -}; - -describe("qemuArgs", () => { - it("is the command line of section 3.4, argument by argument", () => { - expect(qemuArgs(linux)).toEqual([ - "-name", "invisible-dot-dot_01k6", - "-machine", "q35", "-accel", "kvm", "-cpu", "host,-vmx,-svm", - "-smp", "2", "-m", "4096", - "-drive", "if=virtio,file=/home/u/.invisible-dots/vms/dot_01k6/disk.qcow2,format=qcow2,discard=unmap", - "-drive", "if=virtio,file=/home/u/.invisible-dots/vms/dot_01k6/seed.iso,format=raw,readonly=on", - "-drive", "if=virtio,file=/home/u/.invisible-dots/images/runtime-7.iso,format=raw,readonly=on", - "-netdev", "user,id=net0,hostfwd=tcp:127.0.0.1:40123-:1024", - "-device", "virtio-net-pci,netdev=net0", - "-device", "virtio-rng-pci", - "-serial", "file:/home/u/.invisible-dots/vms/dot_01k6/serial.log", - "-display", "none", - ]); // prettier-ignore - }); - - it("differs on Windows only in the accelerator and the paths it is given", () => { - const base = "C:\\Users\\u\\.invisible-dots"; - const windows: QemuArgsSpec = { - ...linux, - accelerator: "whpx", - diskPath: `${base}\\vms\\dot_01k6\\disk.qcow2`, - seedPath: `${base}\\vms\\dot_01k6\\seed.iso`, - runtimeIsoPath: `${base}\\images\\runtime-7.iso`, - serialLogPath: `${base}\\vms\\dot_01k6\\serial.log`, - }; - const a = qemuArgs(linux); - const b = qemuArgs(windows); - expect(b.length).toBe(a.length); - const differing = a.flatMap((value, index) => (value === b[index] ? [] : [index])); - expect(differing.map((index) => a[index - 1])).toEqual(["-accel", "-drive", "-drive", "-drive", "-serial"]); - }); - - it("never asks for a fallback CPU model or software emulation", () => { - const args = qemuArgs(linux).join(" "); - expect(args).not.toMatch(/tcg|max/); - expect(args).toMatch(/-cpu host,/); - }); - - it("gives every accelerator the host CPU without the virtualization extensions", () => { - // Measured with QEMU 11.1 under WHPX (architecture section 3.4): plain `host` and `max` pause the VM - // with "WHPX: Unexpected VP exit code 4" because the guest sees VMX; `host,-vmx` boots to the login. - expect(CPU_MODEL).toBe("host,-vmx,-svm"); - for (const accelerator of ["kvm", "whpx"] as const) { - const args = machineArgs({ accelerator, cpus: 2, memoryMiB: 2048 }); - expect(args[args.indexOf("-cpu") + 1]).toBe(CPU_MODEL); - const features = CPU_MODEL.split(",").slice(1); - expect(features).toEqual(expect.arrayContaining(["-vmx", "-svm"])); - expect(CPU_MODEL.split(",")[0]).toBe("host"); - } - }); - - it("has no monitor and nothing that pauses the VM", () => { - // The VM is reached through the guest channel only, and a paused VM would look started (section 3.4). - const args = qemuArgs(linux); - for (const flag of ["-S", "-qmp", "-qmp-pretty", "-monitor", "-mon", "-chardev", "-pidfile", "-daemonize", "-no-shutdown", "-incoming"]) { - expect(args).not.toContain(flag); - } - expect(args.join(" ")).not.toMatch(/qmp|monitor|stdio/); - }); - - it("refuses paths QEMU's option syntax would split", () => { - expect(() => qemuArgs({ ...linux, diskPath: "/home/a,b/disk.qcow2" })).toThrow(VmManagerError); - expect(() => qemuArgs({ ...linux, serialLogPath: "/home/a,b/serial.log" })).toThrow(/INVISIBLE_DOTS_HOME/); - }); - - it("refuses a path with any character outside plain ASCII, on every host", () => { - // Measured with the pinned Windows QEMU 11.1: both fail to open as -drive files. - for (const home of ["/home/caf\u00e9", "C:\\Users\\\u0416\u0443\u043a", "/home/\u6d4b\u8bd5"]) { - expect(() => qemuArgs({ ...linux, diskPath: `${home}/.invisible-dots/vms/dot_01k6/disk.qcow2` })).toThrow(/plain ASCII.*INVISIBLE_DOTS_HOME/); - } - expect(qemuPathProblem("/home/u/.invisible-dots/vms/dot_01k6/disk.qcow2")).toBeUndefined(); - expect(qemuPathProblem("C:\\Program Files\\x")).toBeUndefined(); - }); - - it("checks the numbers it is given", () => { - expect(() => qemuArgs({ ...linux, cpus: 0 })).toThrow(/cpus/); - expect(() => qemuArgs({ ...linux, cpus: 17 })).toThrow(/cpus/); - expect(() => qemuArgs({ ...linux, memoryMiB: 1.5 })).toThrow(/memoryMiB/); - expect(() => qemuArgs({ ...linux, guestPort: 0 })).toThrow(/guestPort/); - expect(() => qemuArgs({ ...linux, guestPort: 65536 })).toThrow(/guestPort/); - }); -}); diff --git a/apps/vm-manager/test/runner.test.ts b/apps/vm-manager/test/runner.test.ts deleted file mode 100644 index 96397a21..00000000 --- a/apps/vm-manager/test/runner.test.ts +++ /dev/null @@ -1,141 +0,0 @@ -import { mkdtemp, readFile, rm } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { describe, expect, it } from "vitest"; -import { CommandError, firstLine, NodeCommandRunner, NodeProcessControl, retryWhileInUse, runProcess, startProcess } from "../src/index.js"; - -const childOptions = { cwd: tmpdir(), env: { PATH: process.env.PATH ?? "" } }; - -// The real spawn against the running Node: the same on every host, no shell involved. -const node = process.execPath; - -describe("runProcess", () => { - it("collects exit code, stdout and stderr", async () => { - const result = await runProcess(node, ["-e", "process.stdout.write('out');process.stderr.write('err');process.exit(3)"], { timeoutMs: 20_000 }); - expect(result).toEqual({ code: 3, signal: null, stdout: "out", stderr: "err", timedOut: false }); - }); - - it("closes stdin at once, so a program reading it sees end of input", async () => { - const result = await runProcess(node, ["-e", "process.stdin.on('data',()=>{}).on('end',()=>process.exit(4))"], { timeoutMs: 20_000 }); - expect(result.code).toBe(4); - }); - - it("passes env and cwd", async () => { - const result = await runProcess(node, ["-e", "process.stdout.write(process.env.IDOTS_RUNNER_TEST+'|'+process.cwd())"], { - env: { ...process.env, IDOTS_RUNNER_TEST: "seen" }, - cwd: process.execPath.replace(/[\\/][^\\/]+$/, ""), - timeoutMs: 20_000, - }); - expect(result.stdout.startsWith("seen|")).toBe(true); - }); - - it("kills a process that outlives its timeout and says so", async () => { - const result = await runProcess(node, ["-e", "setInterval(()=>{},1000)"], { timeoutMs: 300 }); - expect(result.timedOut).toBe(true); - expect(result.code === null || result.code !== 0).toBe(true); - }); - - it("reports a program that does not exist instead of throwing", async () => { - const result = await runProcess("invisible-dots-no-such-program-x", ["--version"]); - expect(result.code).toBeNull(); - expect(result.startError?.code).toBe("ENOENT"); - }); -}); - -describe("NodeCommandRunner", () => { - const runner = new NodeCommandRunner(); - - it("resolves with the output of a command that exits 0", async () => { - expect(await runner.run(node, ["-e", "process.stdout.write('{\"a\":1}')"])).toEqual({ stdout: '{"a":1}', stderr: "" }); - }); - - it("rejects with a CommandError carrying the exit code and stderr", async () => { - const error = (await runner.run(node, ["-e", "process.stderr.write('bad disk');process.exit(2)"]).catch((e: unknown) => e)) as CommandError; - expect(error).toBeInstanceOf(CommandError); - expect(error.exitCode).toBe(2); - expect(error.message).toContain("exited with code 2: bad disk"); - }); - - it("rejects a timeout and a missing program in their own words", async () => { - const slow = (await runner.run(node, ["-e", "setInterval(()=>{},1000)"], { timeoutMs: 200 }).catch((e: unknown) => e)) as CommandError; - expect(slow.timedOut).toBe(true); - expect(slow.message).toContain("timed out after 200 ms"); - const missing = (await runner.run("invisible-dots-no-such-program-x", []).catch((e: unknown) => e)) as CommandError; - expect(missing.notFound).toBe(true); - expect(missing.message).toContain("invisible-dots doctor"); - }); -}); - -describe("startProcess", () => { - it("reports the exit once and keeps the end of stderr", async () => { - const child = startProcess(node, ["-e", "process.stderr.write('x'.repeat(10000)+'END');process.exit(5)"], childOptions); - const exit = await new Promise((resolve) => child.onExit((code) => resolve(code))); - expect(exit).toBe(5); - expect(child.stderrTail().endsWith("END")).toBe(true); - expect(child.stderrTail().length).toBeLessThanOrEqual(4096); - // A listener added after the exit is called at once. - expect(await new Promise((resolve) => child.onExit((code) => resolve(code)))).toBe(5); - }); - - it("reports a program that cannot start through onExit", async () => { - const child = startProcess("invisible-dots-no-such-program-x", [], childOptions); - const error = await new Promise((resolve) => child.onExit((_code, _signal, e) => resolve(e))); - expect((error as NodeJS.ErrnoException | undefined)?.code).toBe("ENOENT"); - }); -}); - -describe("NodeProcessControl.spawnDetached", () => { - it("runs the child in the directory and with exactly the environment it is given", async () => { - const dir = await mkdtemp(join(tmpdir(), "idots-detached-")); - try { - const logPath = join(dir, "child.log"); - const control = new NodeProcessControl(); - const child = await control.spawnDetached( - node, - ["-e", "process.stdout.write(JSON.stringify({ cwd: process.cwd(), token: process.env.INVISIBLE_DOTS_TOKEN ?? null, only: process.env.IDOTS_ONLY ?? null }))"], - { logPath, cwd: dir, env: { PATH: process.env.PATH ?? "", IDOTS_ONLY: "yes" } }, - ); - await child.exited; - const seen = JSON.parse(await readFile(logPath, "utf8")) as { cwd: string; token: string | null; only: string | null }; - expect(seen.cwd.toLowerCase()).toBe(dir.toLowerCase()); - expect(seen.only).toBe("yes"); - expect(seen.token).toBeNull(); - expect(control.presence(child.pid)).toBe("gone"); - expect(control.presence(process.pid)).toBe("ours"); - } finally { - await rm(dir, { recursive: true, force: true }); - } - }); -}); - -describe("retryWhileInUse", () => { - it("tries again while the file is in use, and gives every other error back at once", async () => { - let calls = 0; - const busy = Object.assign(new Error("resource busy"), { code: "EBUSY" }); - expect( - await retryWhileInUse(async () => { - calls++; - if (calls < 3) throw busy; - return "done"; - }, 5, 1), - ).toBe("done"); - expect(calls).toBe(3); - const missing = Object.assign(new Error("no such file"), { code: "ENOENT" }); - calls = 0; - await expect( - retryWhileInUse(async () => { - calls++; - throw missing; - }, 5, 1), - ).rejects.toBe(missing); - expect(calls).toBe(1); - await expect(retryWhileInUse(async () => Promise.reject(busy), 2, 1)).rejects.toBe(busy); - }); -}); - -describe("firstLine", () => { - it("skips blank lines and trims", () => { - expect(firstLine("\r\n qemu: error one \r\nmore")).toBe("qemu: error one"); - expect(firstLine("")).toBe(""); - }); -}); diff --git a/apps/vm-manager/test/seed.test.ts b/apps/vm-manager/test/seed.test.ts deleted file mode 100644 index 8c9a6f27..00000000 --- a/apps/vm-manager/test/seed.test.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { readFile, mkdtemp, rm, stat } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; -import { fileURLToPath } from "node:url"; -import { permissionBitsEnforced } from "@invisible-dots/shared"; -import { parse } from "yaml"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { - guestHostname, - loadSeedTemplates, - renderSeed, - renderTemplate, - RUNTIME_ISO_LABEL, - writeSeedIso, - yamlScalar, -} from "../src/index.js"; - -const DOT = "dot_01k6h3w2ze8m4qv7r1xk9bntc5"; - -describe("templates", () => { - it("fills placeholders and refuses a missing value", () => { - expect(renderTemplate("a {{ x }} b {{y}}", { x: 1, y: "z" }, (v) => `<${v}>`)).toBe("a <1> b "); - expect(() => renderTemplate("{{missing}}", {}, String)).toThrow(/missing/); - }); - - it("quotes YAML scalars so a token cannot break out", () => { - const tricky = 'a"b\nc: d #e'; - expect(parse(`k: ${yamlScalar(tricky)}`)).toEqual({ k: tricky }); - }); - - it("turns dot ids into hostnames", () => { - expect(guestHostname(DOT)).toBe("invisible-dot-dot-01k6h3w2ze8m4qv7r1xk9bntc5"); - }); -}); - -describe("the seed of a Dot", () => { - it("renders cloud-config that writes config.json and mounts the runtime ISO by label", async () => { - const seed = renderSeed(await loadSeedTemplates(), DOT, "tok\"en"); - expect(seed.userData.startsWith("#cloud-config\n")).toBe(true); - const userData = parse(seed.userData); - const config = userData.write_files.find((file: { path: string }) => file.path === "/etc/invisible-dots/config.json"); - expect(JSON.parse(config.content)).toEqual({ dotId: DOT, token: 'tok"en' }); - expect(config.permissions).toBe("0600"); - // dot-agentd's own user owns it: nothing the model runs (user dot) can read the token (section 4.1). - expect(config.owner).toBe("dotagentd:dotagentd"); - expect(userData.mounts[0][0]).toBe(`LABEL=${RUNTIME_ISO_LABEL}`); - expect(userData.mounts[0][1]).toBe("/opt/invisible-dots"); - expect(userData.hostname).toBe(guestHostname(DOT)); - expect(parse(seed.metaData)).toEqual({ "instance-id": seed.instanceId, "local-hostname": guestHostname(DOT) }); - }); - - it("lets dot-agentd's user run exactly the poweroff it starts as root, and nothing else, and dot, the model's user, nothing (section 4.1)", async () => { - const userData = parse(renderSeed(await loadSeedTemplates(), DOT, "token").userData); - const user = (name: string) => userData.users.find((entry: { name: string }) => entry.name === name); - expect(user("dotagentd").sudo).toBe("ALL=(root) NOPASSWD: /usr/bin/systemctl poweroff"); - expect(user("dot").sudo).toBe(false); - const dot = user("dot"); - expect(dot.groups).toEqual(["audio", "video", "systemd-journal"]); - // dot-agentd's user has no group, no login and no home to speak of. - expect(user("dotagentd")).not.toHaveProperty("groups"); - expect(user("dotagentd").shell).toBe("/usr/sbin/nologin"); - // cloud-init applies groups only where it creates the user, the golden image's builder seed: the two must agree. - const builder = parse(await readFile(fileURLToPath(new URL("../../../guest/image-builder/builder/user-data.yaml", import.meta.url)), "utf8")); - const builderUser = (name: string) => builder.users.find((entry: { name: string }) => entry.name === name); - expect(builderUser("dot").groups).toEqual(dot.groups); - for (const key of ["gecos", "homedir", "no_create_home", "shell", "lock_passwd"]) { - expect(builderUser("dotagentd")[key], key).toEqual(user("dotagentd")[key]); - } - expect(builderUser("dotagentd")).not.toHaveProperty("groups"); - // The argv dot-agentd runs (guest/dot-agentd server.go DefaultPowerOff) is the command the rule names. - const server = await readFile(fileURLToPath(new URL("../../../guest/dot-agentd/internal/agentd/server.go", import.meta.url)), "utf8"); - expect(server).toContain('var DefaultPowerOff = []string{"/usr/bin/sudo", "-n", "/usr/bin/systemctl", "poweroff"}'); - }); - - it("changes the instance-id exactly when the seed content changes", async () => { - const templates = await loadSeedTemplates(); - const a = renderSeed(templates, DOT, "token-a"); - expect(renderSeed(templates, DOT, "token-a").instanceId).toBe(a.instanceId); - expect(renderSeed(templates, DOT, "token-b").instanceId).not.toBe(a.instanceId); - expect(renderSeed({ ...templates, userData: `${templates.userData}\n# new\n` }, DOT, "token-a").instanceId).not.toBe(a.instanceId); - expect(a.instanceId.startsWith(`iid-${DOT}-`)).toBe(true); - }); - - it("templates have LF endings and only ASCII, as files the guest reads", async () => { - for (const name of ["user-data.yaml.tmpl", "meta-data.yaml.tmpl"]) { - const text = await readFile(join(new URL("../../../virtualization/cloud-init/", import.meta.url).pathname.replace(/^\/([A-Za-z]:)/, "$1"), name), "utf8"); - expect(text).not.toContain("\r"); - expect(/^[\x09\x0a\x20-\x7e]*$/.test(text)).toBe(true); - } - }); -}); - -describe("writeSeedIso", () => { - let dir: string; - beforeEach(async () => { - dir = await mkdtemp(join(tmpdir(), "idots-seed-")); - }); - afterEach(async () => { - await rm(dir, { recursive: true, force: true }); - }); - - it("writes the same bytes for the same seed, labelled cidata, private to the user", async () => { - const seed = renderSeed(await loadSeedTemplates(), DOT, "token"); - const first = join(dir, "a.iso"); - const second = join(dir, "b.iso"); - await writeSeedIso(first, seed); - await writeSeedIso(second, seed); - const bytes = await readFile(first); - expect(bytes.equals(await readFile(second))).toBe(true); - expect(bytes.subarray(16 * 2048 + 40, 16 * 2048 + 46).toString("latin1")).toBe("cidata"); - expect(bytes.includes(Buffer.from(seed.instanceId))).toBe(true); - if (permissionBitsEnforced()) expect((await stat(first)).mode & 0o777).toBe(0o600); - }); -}); diff --git a/apps/vm-manager/test/vm-manager.test.ts b/apps/vm-manager/test/vm-manager.test.ts deleted file mode 100644 index 7c797597..00000000 --- a/apps/vm-manager/test/vm-manager.test.ts +++ /dev/null @@ -1,525 +0,0 @@ -import { existsSync } from "node:fs"; -import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; -import { createServer, type Server } from "node:net"; -import { tmpdir, uptime } from "node:os"; -import { join } from "node:path"; -import { hostPaths, type HostPaths, silentLogger } from "@invisible-dots/shared"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { - AcceleratorUnavailableError, - CpuModelError, - GuestRequestError, - NotImplementedError, - pickFreePort, - QemuNotFoundError, - VmManager, - VmManagerError, - VmStartError, - VmStateError, - generateToken, - qemuArgs, - vmSpecFromConfig, - type VmManagerOptions, - type VmSpec, -} from "../src/index.js"; -import { FakeQemuHost, FakeRunner, argAfter, guestPortOf } from "./fakes.js"; - -const DOT = "dot_01k6h3w2ze8m4qv7r1xk9bntc5"; -const QEMU = { system: "/opt/qemu/bin/qemu-system-x86_64", img: "/opt/qemu/bin/qemu-img" }; - -let root: string; -let paths: HostPaths; -let runner: FakeRunner; -let host: FakeQemuHost; -let manager: VmManager; -let spec: VmSpec; -let diskSize: number; - -function makeManager(extra: Partial = {}): VmManager { - return new VmManager({ - paths, - logger: silentLogger, - qemu: QEMU, - accelerator: "kvm", - runner, - processes: host, - sleep: (ms) => new Promise((resolve) => setTimeout(resolve, Math.min(ms, 2))), - pollIntervalMs: 1, - startSettleMs: 1, - // Generous: a guest that powers off must never be mistaken for one that does - // not on a loaded host. The tests of the forced kill pass their own short timeoutMs. - shutdownTimeoutMs: 5_000, - killTimeoutMs: 40, - startTimeoutMs: 200, - ...extra, - }); -} - -async function processFile(dotId = DOT): Promise<{ pid: number; guest_port: number }> { - return JSON.parse(await readFile(paths.processFilePath(dotId), "utf8")) as { pid: number; guest_port: number }; -} - -/** A port nothing listens on: bound once by the kernel's choice, then released. */ -const deadPort = () => pickFreePort(); - -beforeEach(async () => { - root = await mkdtemp(join(tmpdir(), "idots-vm-")); - paths = hostPaths({ INVISIBLE_DOTS_HOME: root }); - await mkdir(paths.imagesDir, { recursive: true }); - const golden = paths.goldenImagePath("1"); - const runtime = paths.runtimeIsoPath("1"); - await writeFile(golden, "golden"); - await writeFile(runtime, "runtime"); - diskSize = 40 * 1024 ** 3; - runner = new FakeRunner(async (command, args) => { - expect(command).toBe(QEMU.img); - if (args[0] === "create") await writeFile(args.at(-2)!, "qcow2"); - if (args[0] === "info") return { stdout: JSON.stringify({ "virtual-size": diskSize, format: "qcow2" }), stderr: "" }; - if (args[0] === "resize") diskSize = Number(args.at(-1)); - return undefined; - }); - host = new FakeQemuHost(); - manager = makeManager(); - spec = { dotId: DOT, token: "tok-1", goldenImage: golden, runtimeImage: runtime, cpus: 2, memoryMiB: 4096, diskBytes: diskSize }; -}); - -afterEach(async () => { - for (const vm of [...host.vms.values()]) host.terminate(vm); - await rm(root, { recursive: true, force: true, maxRetries: 5 }); -}); - -describe("create", () => { - it("creates the overlay on the golden image and writes the seed", async () => { - const result = await manager.create(spec); - expect(result.diskCreated).toBe(true); - expect(runner.lines()).toEqual([ - `${QEMU.img} create -f qcow2 -F qcow2 -b ${spec.goldenImage} ${paths.diskPath(DOT)} ${spec.diskBytes}`, - ]); - const seed = await readFile(paths.seedPath(DOT)); - // The primary volume descriptor at sector 16 carries the label cloud-init looks for. - expect(seed.subarray(16 * 2048 + 40, 16 * 2048 + 46).toString("latin1")).toBe("cidata"); - expect(seed.includes(Buffer.from("tok-1"))).toBe(true); - expect(result.instanceId).toMatch(new RegExp(`^iid-${DOT}-[0-9a-f]{16}$`)); - }); - - it("keeps an existing disk and the instance-id when nothing changed", async () => { - const first = await manager.create(spec); - const second = await manager.create(spec); - expect(second.diskCreated).toBe(false); - expect(runner.lines().filter((line) => line.includes(" create "))).toHaveLength(1); - expect(second.instanceId).toBe(first.instanceId); - const third = await manager.create({ ...spec, token: "tok-2" }); - expect(third.instanceId).not.toBe(first.instanceId); - }); - - it("gives a seed that goes back to an earlier content an id cloud-init has not seen, so its config.json is written again", async () => { - // A VM proxy set and then cleared: the third seed has the first one's content. With the content's id alone it - // got the first boot's id again, cloud-init skipped write_files, and the Dot kept the proxy it no longer had. - const ids = []; - for (const proxy of [undefined, "socks5://10.0.2.2:1081", undefined, "socks5://10.0.2.2:1081", undefined]) { - ids.push((await manager.create({ ...spec, ...(proxy ? { proxy } : {}) })).instanceId); - } - expect(new Set(ids).size).toBe(ids.length); - // The same seed again is still the same id: a restart re-runs nothing. - expect((await manager.create(spec)).instanceId).toBe(ids[4]); - }); - - it("refuses missing images, bad ids and relative image paths", async () => { - await expect(manager.create({ ...spec, goldenImage: join(root, "nope.qcow2") })).rejects.toThrow(/golden image .* does not exist.*image build/); - await expect(manager.create({ ...spec, dotId: "../etc" })).rejects.toBeInstanceOf(VmManagerError); - await expect(manager.create({ ...spec, runtimeImage: "runtime.iso" })).rejects.toThrow(/absolute/); - await expect(manager.create({ ...spec, cpus: 0 })).rejects.toThrow(/cpus/); - }); - - it("refuses to rewrite the seed of a running VM", async () => { - await manager.create(spec); - await manager.start(spec); - await expect(manager.create(spec)).rejects.toBeInstanceOf(VmStateError); - }); -}); - -describe("start", () => { - beforeEach(async () => { - await manager.create(spec); - }); - - it("spawns QEMU with the argv of section 3.4, records pid and port, and reports them", async () => { - const started = await manager.start(spec); - expect(host.spawned).toHaveLength(1); - const { command, args, logPath } = host.spawned[0]!; - expect(command).toBe(QEMU.system); - expect(args).toEqual( - qemuArgs({ - dotId: DOT, - accelerator: "kvm", - cpus: 2, - memoryMiB: 4096, - diskPath: paths.diskPath(DOT), - seedPath: paths.seedPath(DOT), - runtimeIsoPath: spec.runtimeImage, - guestPort: started.guestPort, - serialLogPath: paths.serialLogPath(DOT), - }), - ); - expect(logPath).toBe(join(paths.logsDir, `qemu-${DOT}.log`)); - expect(started).toEqual({ pid: host.only().pid, guestPort: host.only().guestPort, alreadyRunning: false }); - expect(await processFile()).toEqual({ pid: started.pid, guest_port: started.guestPort, host_uptime_s: expect.any(Number) }); - expect(await manager.state(DOT)).toEqual({ state: "RUNNING", pid: started.pid, guestPort: started.guestPort }); - }); - - it("does nothing when the VM already runs", async () => { - const first = await manager.start(spec); - const second = await manager.start(spec); - expect(second).toEqual({ pid: first.pid, guestPort: first.guestPort, alreadyRunning: true }); - expect(host.spawned).toHaveLength(1); - }); - - it("takes the new cpus, memory and runtime ISO from the spec", async () => { - const runtime2 = paths.runtimeIsoPath("2"); - await writeFile(runtime2, "runtime 2"); - await manager.start({ ...spec, cpus: 4, memoryMiB: 8192, runtimeImage: runtime2 }); - const args = host.spawned[0]!.args; - expect(argAfter(args, "-smp")).toBe("4"); - expect(argAfter(args, "-m")).toBe("8192"); - expect(args).toContain(`if=virtio,file=${runtime2},format=raw,readonly=on`); - }); - - it("retries with another port when QEMU cannot bind the forward", async () => { - host.behaviour = (_args, attempt) => - attempt === 1 ? { kind: "exit", code: 1, output: "qemu: -netdev user,id=net0: Could not set up host forwarding rule 'tcp:127.0.0.1:1-:1024'\n" } : { kind: "run" }; - const started = await manager.start(spec); - expect(host.spawned).toHaveLength(2); - expect(started.guestPort).toBe(guestPortOf(host.spawned[1]!.args)); - expect(await processFile()).toEqual({ pid: started.pid, guest_port: started.guestPort, host_uptime_s: expect.any(Number) }); - }); - - it("gives up after the configured number of taken ports, leaving no pid file", async () => { - host.behaviour = () => ({ kind: "exit", code: 1, output: "Could not set up host forwarding rule 'x'\n" }); - const manager3 = makeManager({ maxPortAttempts: 3 }); - await expect(manager3.start(spec)).rejects.toBeInstanceOf(VmStartError); - expect(host.spawned).toHaveLength(3); - expect(existsSync(paths.processFilePath(DOT))).toBe(false); - }); - - it("names setup and doctor when the accelerator is unusable", async () => { - host.behaviour = () => ({ kind: "exit", code: 1, output: "Could not access KVM kernel module: No such file or directory\nqemu-system-x86_64: -accel kvm: failed to initialize kvm: No such file or directory\n" }); - const error = await manager.start(spec).catch((e: unknown) => e); - expect(error).toBeInstanceOf(AcceleratorUnavailableError); - expect((error as Error).message).toContain("invisible-dots setup"); - expect((error as Error).message).toContain("invisible-dots doctor"); - expect((error as Error).message).toContain("never falls back to software emulation"); - expect(existsSync(paths.processFilePath(DOT))).toBe(false); - }); - - it("reports the CPU model refused even when QEMU set up its forward first, without trying another model", async () => { - host.behaviour = () => ({ kind: "exitAfterForward", code: 1, output: "qemu-system-x86_64: CPU model 'host' requires KVM or HVF\n" }); - const settling = makeManager({ startSettleMs: 100, sleep: (ms) => new Promise((resolve) => setTimeout(resolve, ms)) }); - const error = await settling.start(spec).catch((e: unknown) => e); - expect(error).toBeInstanceOf(CpuModelError); - expect((error as Error).message).toContain('"-cpu host,-vmx,-svm"'); - expect(host.spawned).toHaveLength(1); - expect((await settling.state(DOT)).state).toBe("STOPPED"); - }); - - it("kills a QEMU that never sets up its port forward, and says so", async () => { - host.behaviour = () => ({ kind: "hang" }); - const error = (await manager.start(spec).catch((e: unknown) => e)) as VmStartError; - expect(error).toBeInstanceOf(VmStartError); - expect(error.message).toMatch(/did not set up its port forward .* and was killed/); - expect(host.killed).toHaveLength(1); - expect(host.alive.size).toBe(0); - expect(existsSync(paths.processFilePath(DOT))).toBe(false); - }); - - it("includes what QEMU wrote during this start only", async () => { - await mkdir(paths.logsDir, { recursive: true }); - await writeFile(join(paths.logsDir, `qemu-${DOT}.log`), "old run noise\n"); - host.behaviour = () => ({ kind: "exit", code: 1, output: "qemu-system-x86_64: -drive if=virtio: Could not open 'x': Permission denied\n" }); - const error = (await manager.start(spec).catch((e: unknown) => e)) as VmStartError; - expect(error).toBeInstanceOf(VmStartError); - expect(error.qemuOutput).toContain("Permission denied"); - expect(error.qemuOutput).not.toContain("old run noise"); - expect(error.message).toContain("exited with code 1"); - }); - - it("refuses to start before create", async () => { - await expect(manager.start({ ...spec, dotId: "dot_other" })).rejects.toThrow(/create the VM first/); - }); - - it("fails clearly when QEMU is not installed", async () => { - const empty = join(root, "no-qemu"); - await mkdir(empty); - const bare = makeManager({ qemu: undefined, env: { INVISIBLE_DOTS_QEMU_DIR: empty } }); - const error = await bare.start(spec).catch((e: unknown) => e); - expect(error).toBeInstanceOf(QemuNotFoundError); - expect((error as Error).message).toContain("invisible-dots setup"); - }); - - it("starts over a pid file whose process is gone", async () => { - await writeFile(paths.processFilePath(DOT), JSON.stringify({ pid: 999999, guest_port: await deadPort() })); - expect((await manager.state(DOT)).state).toBe("STOPPED"); - const started = await manager.start(spec); - expect(started.alreadyRunning).toBe(false); - }); - - it("never kills or starts over a live pid whose guest port does not listen: it may be another program", async () => { - await writeFile(paths.processFilePath(DOT), JSON.stringify({ pid: 777, guest_port: await deadPort() })); - host.foreignPids.add(777); - const state = await manager.state(DOT); - expect(state.state).toBe("ERROR"); - expect(state.detail).toContain("the pid now belongs to another process"); - await expect(manager.start(spec)).rejects.toBeInstanceOf(VmStateError); - await expect(manager.stop(DOT, spec.token)).rejects.toThrow(/cannot stop it safely/); - await expect(manager.destroy(DOT)).rejects.toThrow(/cannot destroy it safely/); - expect(host.killed).toEqual([]); - }); - - it("never takes another user's process for this Dot's QEMU, even when the recorded port listens", async () => { - const listener: Server = createServer(); - await new Promise((resolve) => listener.listen(0, "127.0.0.1", resolve)); - const port = (listener.address() as { port: number }).port; - try { - await writeFile(paths.processFilePath(DOT), JSON.stringify({ pid: 780, guest_port: port })); - host.otherUserPids.add(780); - const state = await manager.state(DOT); - // Not ours, and not gone either: reported, never killed, never adopted. - expect(state.state).toBe("ERROR"); - await expect(manager.stop(DOT, spec.token)).rejects.toThrow(/cannot stop it safely/); - expect(host.killed).toEqual([]); - } finally { - listener.close(); - } - }); - - it("accepts a live pid whose recorded guest port listens as this Dot's QEMU", async () => { - // A restarted control plane holds no ChildProcess: pid and port from the pid file prove it. - const listener: Server = createServer(); - await new Promise((resolve) => listener.listen(0, "127.0.0.1", resolve)); - const port = (listener.address() as { port: number }).port; - try { - await writeFile(paths.processFilePath(DOT), JSON.stringify({ pid: 778, guest_port: port })); - host.foreignPids.add(778); - expect(await manager.state(DOT)).toEqual({ state: "RUNNING", pid: 778, guestPort: port }); - } finally { - listener.close(); - } - }); - - it("treats a pid file it cannot read as unknown, never as stopped", async () => { - await writeFile(paths.processFilePath(DOT), "12345\n"); - const state = await manager.state(DOT); - expect(state.state).toBe("ERROR"); - expect(state.detail).toContain("is not a pid file this version wrote"); - await expect(manager.start(spec)).rejects.toBeInstanceOf(VmStateError); - }); -}); - -describe("stop, reboot, destroy", () => { - beforeEach(async () => { - await manager.create(spec); - }); - - it("asks the guest to power off through the guest channel and waits for QEMU to exit", async () => { - await manager.start(spec); - const vm = host.only(); - const result = await manager.stop(DOT, spec.token); - expect(result).toEqual({ forced: false, wasRunning: true }); - expect(vm.requests.map((r) => r.replace(/\?nonce=[0-9a-f]+$/, ""))).toEqual(["GET /v1/proof", "POST /v1/system/poweroff"]); - expect(host.killed).toEqual([]); - expect(existsSync(paths.processFilePath(DOT))).toBe(false); - expect((await manager.state(DOT)).state).toBe("STOPPED"); - }); - - it("kills QEMU when the guest accepts the poweroff but does not power off in time", async () => { - host.behaviour = () => ({ kind: "run", ignorePoweroff: true }); - const started = await manager.start(spec); - const result = await manager.stop(DOT, spec.token, { timeoutMs: 20 }); - expect(result).toEqual({ forced: true, wasRunning: true }); - expect(host.killed).toEqual([started.pid]); - }); - - it("kills QEMU at once when the guest is not up to take the poweroff", async () => { - host.behaviour = () => ({ kind: "run", guestUp: false }); - const started = await manager.start(spec); - const result = await makeManager({ shutdownTimeoutMs: 60_000 }).stop(DOT, spec.token); - expect(result.forced).toBe(true); - expect(host.killed).toEqual([started.pid]); - }); - - it("kills a QEMU found after a restart only once its pid and port prove it is the Dot's", async () => { - host.behaviour = () => ({ kind: "run", ignorePoweroff: true }); - const started = await manager.start(spec); - const restarted = makeManager(); - expect(await restarted.stop(DOT, spec.token, { timeoutMs: 10 })).toEqual({ forced: true, wasRunning: true }); - expect(host.killed).toEqual([started.pid]); - }); - - it("is a no-op on a stopped VM", async () => { - expect(await manager.stop(DOT, spec.token)).toEqual({ forced: false, wasRunning: false }); - }); - - it("reboots as stop then start, on a new process and port", async () => { - const first = await manager.start(spec); - const second = await manager.reboot(spec); - expect(second.alreadyRunning).toBe(false); - expect(second.pid).not.toBe(first.pid); - expect(host.spawned).toHaveLength(2); - expect(await processFile()).toEqual({ pid: second.pid, guest_port: second.guestPort, host_uptime_s: expect.any(Number) }); - }); - - it("destroys a running VM, its directory and its log", async () => { - await manager.start(spec); - await manager.destroy(DOT); - expect(host.alive.size).toBe(0); - expect(existsSync(paths.vmDir(DOT))).toBe(false); - expect(existsSync(join(paths.logsDir, `qemu-${DOT}.log`))).toBe(false); - }); - - it("spawns QEMU in the data directory with an allowlisted environment, never the server's secrets", async () => { - process.env.INVISIBLE_DOTS_TOKEN = "server-api-token-must-not-leak"; - process.env.DATABASE_URL = "postgres://user:secret@db/idots"; - try { - await manager.start(spec); - } finally { - delete process.env.INVISIBLE_DOTS_TOKEN; - delete process.env.DATABASE_URL; - } - const spawned = host.spawned.at(-1)!; - expect(spawned.cwd).toBe(paths.home); - expect(Object.keys(spawned.env)).not.toContain("INVISIBLE_DOTS_TOKEN"); - expect(Object.keys(spawned.env)).not.toContain("DATABASE_URL"); - expect(Object.keys(spawned.env).map((k) => k.toUpperCase())).toContain("PATH"); - }); - - it("serializes operations on the same Dot", async () => { - const [a, b] = await Promise.all([manager.start(spec), manager.start(spec)]); - expect(host.spawned).toHaveLength(1); - expect([a.alreadyRunning, b.alreadyRunning].sort()).toEqual([false, true]); - }); -}); - -describe("reconciliation after a control plane restart", () => { - it("a new manager finds a running VM by its pid file, and state() removes the pid file of one that is gone", async () => { - await manager.create(spec); - const started = await manager.start(spec); - const deadId = "dot_dead"; - await mkdir(paths.vmDir(deadId), { recursive: true }); - await writeFile(paths.processFilePath(deadId), JSON.stringify({ pid: 999998, guest_port: await deadPort() })); - - const restarted = makeManager(); - expect(await restarted.state(DOT)).toEqual({ state: "RUNNING", pid: started.pid, guestPort: started.guestPort }); - expect(await restarted.state(deadId)).toEqual({ state: "STOPPED", pid: null, guestPort: null }); - // Removed at once: a pid recycled later can never make it look alive again. - expect(existsSync(paths.processFilePath(deadId))).toBe(false); - host.foreignPids.add(999998); - expect((await restarted.state(deadId)).state).toBe("STOPPED"); - }); - - it("a pid file from before a host restart is stopped, whatever runs under its pid now", async () => { - const listener: Server = createServer(); - await new Promise((resolve) => listener.listen(0, "127.0.0.1", resolve)); - const port = (listener.address() as { port: number }).port; - try { - await mkdir(paths.vmDir(DOT), { recursive: true }); - // Written when the host had been up a year longer than it has now. - await writeFile(paths.processFilePath(DOT), JSON.stringify({ pid: 779, guest_port: port, host_uptime_s: 365 * 86_400 + uptime() })); - host.foreignPids.add(779); - expect((await manager.state(DOT)).state).toBe("STOPPED"); - expect(existsSync(paths.processFilePath(DOT))).toBe(false); - expect(host.killed).toEqual([]); - } finally { - listener.close(); - } - }); -}); - -describe("resize", () => { - beforeEach(async () => { - await manager.create(spec); - }); - - it("grows the disk of a stopped VM and never shrinks it", async () => { - const bigger = diskSize + 1024 ** 3; - expect(await manager.resizeDisk(DOT, bigger)).toEqual({ resized: true, previousBytes: 40 * 1024 ** 3 }); - expect(runner.lines().at(-1)).toBe(`${QEMU.img} resize -f qcow2 ${paths.diskPath(DOT)} ${bigger}`); - expect(await manager.resizeDisk(DOT, bigger)).toEqual({ resized: false, previousBytes: bigger }); - await expect(manager.resizeDisk(DOT, 1024)).rejects.toThrow(/shrinking/); - }); - - it("leaves cpus and memory to the next start of a running VM, and refuses its disk", async () => { - await manager.start(spec); - expect(await manager.resize({ ...spec, cpus: 8 })).toEqual({ restartRequired: true, diskResized: false }); - await expect(manager.resize({ ...spec, diskBytes: spec.diskBytes * 2 })).rejects.toBeInstanceOf(VmStateError); - await expect(manager.resizeDisk(DOT, spec.diskBytes * 2)).rejects.toThrow(/while the VM is stopped/); - }); - - it("grows the disk through resize when stopped", async () => { - expect(await manager.resize({ ...spec, diskBytes: spec.diskBytes * 2 })).toEqual({ restartRequired: false, diskResized: true }); - }); -}); - -describe("guest health through the manager", () => { - beforeEach(async () => { - await manager.create(spec); - }); - - it("answers once the guest is up", async () => { - const started = await manager.start(spec); - expect(await manager.waitForGuestHealth(DOT, started.guestPort, spec.token, { intervalMs: 1, timeoutMs: 2_000 })).toMatchObject({ agentd: "ok" }); - }); - - it("stops waiting when QEMU exits, with the end of both logs", async () => { - host.behaviour = () => ({ kind: "run", guestUp: false, qemuOutput: "qemu: last words\n" }); - const started = await manager.start(spec); - host.terminate(host.only()); - const error = (await manager.waitForGuestHealth(DOT, started.guestPort, spec.token, { intervalMs: 1, timeoutMs: 1000 }).catch((e: unknown) => e)) as VmStartError; - expect(error).toBeInstanceOf(VmStartError); - expect(error.message).toMatch(/QEMU is not running any more/); - expect(error.qemuOutput).toContain("qemu: last words"); - }); - - it("reports a QEMU that runs but never brings the guest up with its log and the serial console", async () => { - // What a paused vCPU looks like from outside: QEMU alive, its forward up, nothing behind it. - host.behaviour = () => ({ kind: "run", guestUp: false, qemuOutput: "WHPX: Unexpected VP exit code 4\n", serial: "SeaBIOS (version 1.16.3)\nBooting from Hard Disk...\n" }); - const started = await manager.start(spec); - const error = (await manager.waitForGuestHealth(DOT, started.guestPort, spec.token, { intervalMs: 1, timeoutMs: 50, requestTimeoutMs: 20 }).catch((e: unknown) => e)) as VmStartError; - expect(error).toBeInstanceOf(VmStartError); - expect(error.message).toContain("the guest did not come up"); - expect(error.message).toContain(paths.qemuLogPath(DOT)); - expect(error.message).toContain(paths.serialLogPath(DOT)); - expect(error.qemuOutput).toContain("WHPX: Unexpected VP exit code 4"); - expect(error.serialOutput).toContain("Booting from Hard Disk"); - expect(error.message).toContain("The serial console ends with: SeaBIOS"); - }); - - it("passes a guest that cannot prove it holds the token on as it is, at once", async () => { - const started = await manager.start(spec); - await expect(manager.waitForGuestHealth(DOT, started.guestPort, "wrong-token", { intervalMs: 1, timeoutMs: 1000 })).rejects.toSatisfy( - (error: unknown) => error instanceof GuestRequestError && error.code === "guest_unproven", - ); - }); -}); - -describe("helpers", () => { - it("builds a spec from a Dot config", () => { - const built = vmSpecFromConfig(DOT, { computer: { cpu: 2, memory: "4gb", disk: "40gb", idle_timeout: "15m" } }, { - token: "t", - goldenImage: "/g", - runtimeImage: "/r", - }); - expect(built).toEqual({ dotId: DOT, token: "t", goldenImage: "/g", runtimeImage: "/r", cpus: 2, memoryMiB: 4096, diskBytes: 40 * 1024 ** 3 }); - }); - - it("generates distinct url-safe tokens", () => { - const a = generateToken(); - expect(a).toMatch(/^[A-Za-z0-9_-]{43}$/); - expect(generateToken()).not.toBe(a); - }); - - it("rejects snapshots as out of scope", async () => { - await expect(manager.createSnapshot(DOT, "s")).rejects.toBeInstanceOf(NotImplementedError); - await expect(manager.listSnapshots(DOT)).rejects.toThrow(/out of scope/); - await expect(manager.restoreSnapshot(DOT, "s")).rejects.toThrow(/section 10/); - await expect(manager.deleteSnapshot(DOT, "s")).rejects.toBeInstanceOf(NotImplementedError); - }); -}); diff --git a/apps/vm-manager/tsconfig.json b/apps/vm-manager/tsconfig.json deleted file mode 100644 index 31a061f4..00000000 --- a/apps/vm-manager/tsconfig.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "include": [ - "src/**/*.ts", - "test/**/*.ts" - ] -} diff --git a/apps/web/.gitignore b/apps/web/.gitignore deleted file mode 100644 index 61350467..00000000 --- a/apps/web/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -.next/ -next-env.d.ts -# Playwright's output: traces of a failed run, and the JUnit report CI reads -test-results/ -playwright-report.xml diff --git a/apps/web/e2e/activity.spec.ts b/apps/web/e2e/activity.spec.ts deleted file mode 100644 index 6d23ecfa..00000000 --- a/apps/web/e2e/activity.spec.ts +++ /dev/null @@ -1,164 +0,0 @@ -import { readFile } from "node:fs/promises"; -import type { OutboundEventDataMap, StoredEvent } from "@invisible-dots/shared"; -import { expect, test } from "./fixtures.js"; -import { lookOf } from "./look.js"; - -/** A tool call as the engine reports it once it has ended. */ -function called(tool: string, target: string, change: Partial = {}): OutboundEventDataMap["tool.called"] { - return { tool, permission: "computer.exec", decision: "allow", ok: true, duration_ms: 40, target, ...change }; -} - -const PAGE = 200; - -test("the Activity tab reads the log as lines, filters by family at the host, follows the Dot live, and the old Timeline address leads to it", async ({ page, harness }) => { - const dot = await harness.createDot("activity-lines"); - const guest = harness.driver.guestOf(dot.id); - guest.emit("tool.called", called("exec", "ls -la /home/dot")); - guest.emit("browser.identity.created", { identity_id: "bi_fares", name: "fares.md" }); - guest.emit("tool.called", called("exec", "rm -rf /", { decision: "deny", ok: false, duration_ms: 0 })); - await harness.api.sendMessage(dot.id, "hello there"); - await expect.poll(async () => (await harness.api.events(dot.id, { types: ["tool.called", "browser.identity.created"] })).length).toBe(3); - - // The old Timeline address leads here, and the tab is in the bar. - await page.goto(`${harness.webUrl}/dots/${dot.id}/timeline`); - await expect(page).toHaveURL(new RegExp(`/dots/${dot.id}/activity$`)); - await expect(page.getByRole("navigation", { name: "Dot sections" }).getByRole("link", { name: "Activity" })).toHaveAttribute("aria-current", "page"); - await expect(page.getByRole("navigation", { name: "Dot sections" }).getByRole("link", { name: "Timeline" })).toHaveCount(0); - - const rows = page.getByRole("list", { name: "Events" }).getByRole("listitem"); - await expect(rows.filter({ hasText: "Ran a command" })).toHaveCount(2); - const allowed = rows.filter({ hasText: "ls -la /home/dot" }); - await expect(allowed).toContainText("done in 40 ms | exec [computer.exec] allow"); - // The refused call is a warning line (the config refused it; nothing failed), and says it never ran. - const refused = rows.filter({ hasText: "rm -rf /" }); - await expect(refused).toContainText("denied in 0 ms | exec [computer.exec] deny"); - await expect(refused).toHaveAttribute("data-tone", "warn"); - await expect(rows.filter({ hasText: "You sent a message" })).toContainText("hello there"); - // The engine's own start, in words, not as empty data. - await expect(rows.filter({ hasText: "The engine started" })).toContainText("The key was sent again"); - // The data under a line is the event as stored. - await allowed.getByText("Data", { exact: true }).click(); - await expect(allowed.getByLabel(/^Data of event/)).toContainText('"target": "ls -la /home/dot"'); - - // A family narrows what the host is asked for. - const asked: string[] = []; - page.on("request", (request) => { - if (/\/api\/dots\/[^/]+\/events\?/.test(request.url())) asked.push(new URL(request.url()).search); - }); - await page.getByRole("button", { name: "Browser", exact: true }).click(); - await expect(rows).toHaveCount(1); - await expect(rows.first()).toContainText("fares.md"); - expect(asked.some((search) => search.includes("browser.identity.created") && search.includes("order=desc"))).toBe(true); - - // Live: an event of the chosen family arrives on top, one of another family does not. - guest.emit("agent.state", { state: "THINKING" }); - guest.emit("browser.identity.created", { identity_id: "bi_rome", name: "trips/rome.md" }); - await expect(rows).toHaveCount(2); - await expect(rows.first()).toContainText("trips/rome.md"); - await expect(page.getByRole("list", { name: "Events" }).getByText("THINKING")).toHaveCount(0); - - // Search the lines read. - await page.getByRole("button", { name: "All" }).click(); - await page.getByRole("searchbox", { name: "Search the events read so far" }).fill("ROME"); - await expect(rows).toHaveCount(1); - await page.getByRole("searchbox", { name: "Search the events read so far" }).fill(""); -}); - -test("a long log is read a page at a time from the newest, with nothing skipped or repeated, and the oldest first when asked", async ({ page, harness }) => { - const dot = await harness.createDot("activity-paging"); - const guest = harness.driver.guestOf(dot.id); - const total = 2 * PAGE + 30; - for (let i = 0; i < total; i++) guest.emit("browser.identity.created", { identity_id: `bi_${i}`, name: `note-${String(i).padStart(4, "0")}.md` }); - await expect.poll(async () => (await harness.api.events(dot.id, { types: ["browser.identity.created"], order: "desc", limit: 1 }))[0]?.data.name).toBe(`note-${String(total - 1).padStart(4, "0")}.md`); - - await page.goto(`${harness.webUrl}/dots/${dot.id}/activity`); - await page.getByRole("button", { name: "Browser", exact: true }).click(); - const rows = page.getByRole("list", { name: "Events" }).getByRole("listitem"); - await expect(rows).toHaveCount(PAGE); - await expect(rows.first()).toContainText(`note-${String(total - 1).padStart(4, "0")}.md`); - await page.getByRole("button", { name: "Load older events" }).click(); - await expect(rows).toHaveCount(2 * PAGE); - await page.getByRole("button", { name: "Load older events" }).click(); - await expect(rows).toHaveCount(total); - await expect(page.getByText("That is the start of the log.")).toBeVisible(); - await expect(page.getByRole("button", { name: "Load older events" })).toHaveCount(0); - const keys = await rows.evaluateAll((items) => items.map((item) => /note-(\d+)\.md/.exec(item.textContent ?? "")![1])); - expect(keys).toEqual(Array.from({ length: total }, (_, i) => String(total - 1 - i).padStart(4, "0"))); - - await page.getByRole("switch", { name: "Newest first" }).click(); - await expect(rows.first()).toContainText("note-0000.md"); - await expect(rows.last()).toContainText(`note-${String(total - 1).padStart(4, "0")}.md`); -}); - -test("the events on screen are saved as JSON Lines, as the host stored them", async ({ page, harness }) => { - const dot = await harness.createDot("activity-export"); - const guest = harness.driver.guestOf(dot.id); - guest.emit("browser.identity.created", { identity_id: "bi_a", name: "a.md" }); - guest.emit("browser.identity.created", { identity_id: "bi_b", name: "b.md" }); - await expect.poll(async () => (await harness.api.events(dot.id, { types: ["browser.identity.created"] })).length).toBe(2); - await page.goto(`${harness.webUrl}/dots/${dot.id}/activity`); - await page.getByRole("button", { name: "Browser", exact: true }).click(); - await expect(page.getByRole("list", { name: "Events" }).getByRole("listitem")).toHaveCount(2); - - const stored = await harness.api.events(dot.id, { types: ["browser.identity.created"] }); - const [download] = await Promise.all([page.waitForEvent("download"), page.getByRole("button", { name: "Export 2 events" }).click()]); - expect(download.suggestedFilename()).toBe(`${dot.id}-events-${stored[0]!.id}-${stored[1]!.id}.jsonl`); - const text = await readFile((await download.path())!, "utf8"); - expect(text.trimEnd().split("\n").map((line) => JSON.parse(line) as StoredEvent)).toEqual(stored); -}); - -test("the Activity page is usable from the keyboard alone, and fits a phone", async ({ page, harness }) => { - const dot = await harness.createDot("activity-keys"); - const guest = harness.driver.guestOf(dot.id); - guest.emit("tool.called", called("exec", "a very long command ".repeat(20))); - await expect.poll(async () => (await harness.api.events(dot.id, { types: ["tool.called"] })).length).toBe(1); - await page.goto(`${harness.webUrl}/dots/${dot.id}/activity`); - const rows = page.getByRole("list", { name: "Events" }).getByRole("listitem"); - await expect(rows.filter({ hasText: "Ran a command" })).toHaveCount(1); - - // A family is chosen with the keyboard, and the line's data opens with it. - await page.getByRole("button", { name: "Tools" }).focus(); - await page.keyboard.press("Enter"); - await expect(page.getByRole("button", { name: "Tools" })).toHaveAttribute("aria-pressed", "true"); - await page.getByRole("switch", { name: "Newest first" }).focus(); - await page.keyboard.press("Space"); - await expect(page.getByRole("switch", { name: "Newest first" })).not.toBeChecked(); - const data = rows.first().locator("summary"); - await data.focus(); - await page.keyboard.press("Enter"); - await expect(rows.first().getByLabel(/^Data of event/)).toBeVisible(); - - await page.setViewportSize({ width: 390, height: 844 }); - await page.goto(`${harness.webUrl}/dots/${dot.id}/activity`); - await expect(rows.filter({ hasText: "Ran a command" })).toHaveCount(1); - expect(await page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth)).toBe(0); -}); - -for (const scheme of ["light", "dark"] as const) { - test(`the Activity page is readable in ${scheme}: the chips, the lines, the notes under them`, async ({ page, harness }) => { - await page.emulateMedia({ colorScheme: scheme }); - const dot = await harness.createDot(`activity-look-${scheme}`); - const guest = harness.driver.guestOf(dot.id); - guest.emit("tool.called", called("exec", "ls")); - guest.emit("tool.called", called("exec", "rm", { decision: "deny", ok: false })); - await harness.control.scheduler.sendMessage(dot.id, "from the phone", { channel: "telegram", binding_id: "b", chat_id: "1", external_id: "2" }); - await expect.poll(async () => (await harness.api.events(dot.id, { types: ["tool.called"] })).length).toBe(2); - await page.goto(`${harness.webUrl}/dots/${dot.id}/activity`); - await expect(page.getByRole("list", { name: "Events" }).getByRole("listitem").filter({ hasText: "via Telegram" })).toBeVisible(); - - const spots: Record = { - "a family that is not chosen": "button[aria-pressed=false]:text-is('Tools')", - "the family that is chosen": "button[aria-pressed=true]:text-is('All')", - "the count": "p[role=status]", - "a line's title": "li[data-testid=activity-row] span.text-sm", - "a line's detail": "li[data-testid=activity-row] p", - "a line's type": "li[data-testid=activity-row] code", - "the time of a line": "li[data-testid=activity-row] time", - "the channel chip": "li[data-testid=activity-row] [data-slot=badge]", - "the way to the data": "li[data-testid=activity-row] summary", - }; - for (const [name, selector] of Object.entries(spots)) { - expect((await lookOf(page, selector)).contrast, name).toBeGreaterThanOrEqual(4.5); - } - }); -} diff --git a/apps/web/e2e/channels.spec.ts b/apps/web/e2e/channels.spec.ts deleted file mode 100644 index 9cb862aa..00000000 --- a/apps/web/e2e/channels.spec.ts +++ /dev/null @@ -1,300 +0,0 @@ -import type { Page } from "@playwright/test"; -import { expect, test } from "./fixtures.js"; -import type { Harness } from "./harness.js"; -import { lookOf } from "./look.js"; - -/** A token as @BotFather makes them: the bot's id, a colon, a secret. The Bot API of the harness knows the bots a test adds. */ -function makeBot(harness: Harness, id: number, username: string): string { - const token = `${id}:E2E-SECRET-${username.replace(/\W/g, "")}-0123456789`; - harness.bots.addBot(token, username); - return token; -} - -// The Dots of these tests are marks on every page of the shared host (a channel to link again counts for the Inbox -// and the title), so each test removes its own, and the other specs find the host as they expect. -test.afterEach(async ({ harness }) => { - for (const dot of await harness.api.listDots()) if (dot.name.startsWith("channels-")) await harness.api.deleteDot(dot.id); -}); - -const channelsTab = (page: Page) => page.getByRole("navigation", { name: "Dot sections" }).getByRole("link", { name: /^Channels/ }); -const telegram = (page: Page) => page.getByRole("region", { name: "Telegram" }); -const whatsapp = (page: Page) => page.getByRole("region", { name: "WhatsApp" }); - -/** The pairing code in the link of a card's "Open ..." button. */ -async function codeOf(page: Page, name: "Open Telegram" | "Open WhatsApp"): Promise { - const href = await page.getByRole("link", { name }).getAttribute("href"); - const code = /(?:start=|pair(?:%20|\+| )?)([A-Z0-9]+)$/.exec(href ?? "")?.[1]; - if (!code) throw new Error(`no pairing code in ${href}`); - return code; -} - -test("the Channels tab connects Telegram, pairs a person with the code, and carries their messages to the chat and the answers back", async ({ page, harness }) => { - const dot = await harness.createDot("channels-telegram"); - const token = makeBot(harness, 700001, "e2e_fares_bot"); - const ann = { id: 4242, first_name: "Ann" }; - await page.goto(`${harness.webUrl}/dots/${dot.id}/chat`); - - await channelsTab(page).click(); - await expect(page).toHaveURL(/\/channels$/); - await expect(page.getByRole("heading", { level: 1, name: "channels-telegram" })).toBeVisible(); - await expect(telegram(page).getByText("Not connected")).toBeVisible(); - - // The token is checked with Telegram and then forgotten by the page. - await telegram(page).getByLabel("Bot token").fill(token); - await telegram(page).getByRole("button", { name: "Connect Telegram" }).click(); - await expect(telegram(page).getByText("Connected")).toBeVisible(); - await expect(telegram(page).getByRole("link", { name: "@e2e_fares_bot" })).toHaveAttribute("href", "https://t.me/e2e_fares_bot"); - await expect(page.locator("body")).not.toContainText("E2E-SECRET"); - await expect(page.getByLabel("Bot token")).toHaveCount(0); - expect(JSON.stringify(await harness.api.channels(dot.id))).not.toContain("E2E-SECRET"); - - // Link: the deep link, its QR code and the words to type, with the code counting down. - await telegram(page).getByRole("button", { name: "Link your Telegram" }).click(); - const open = telegram(page).getByRole("link", { name: "Open Telegram" }); - await expect(open).toHaveAttribute("href", /^https:\/\/t\.me\/e2e_fares_bot\?start=[A-Z0-9]+$/); - await expect(telegram(page).getByRole("img", { name: "QR code that opens Telegram with the pairing code" })).toBeVisible(); - await expect(telegram(page).getByRole("timer")).toHaveText(/^Expires in (10:00|9:\d\d)\.$/); - const code = await codeOf(page, "Open Telegram"); - await expect(telegram(page).getByText(`/start ${code}`)).toBeVisible(); - - // A stranger who writes first is not paired and not answered; the person who sends the code is. - harness.bots.say(token, "hello?", { id: 999, first_name: "Stranger" }); - harness.bots.say(token, `/start ${code}`, ann); - const people = telegram(page).getByRole("region", { name: "People paired on Telegram" }); - await expect(people.getByRole("listitem")).toHaveCount(1); - await expect(people.getByRole("listitem")).toContainText("Ann"); - await expect(people.getByRole("listitem")).toContainText("Owner"); - await expect(telegram(page).getByText(`/start ${code}`)).toHaveCount(0); - await expect(page.getByText("Ann is paired on Telegram.")).toBeVisible(); - expect(harness.bots.sent(token, 999)).toEqual([]); - - // What Ann writes arrives in the chat with where it came from; the Dot's answer goes back to her chat. - harness.bots.say(token, "Is the 9 am flight still the cheapest?", ann); - await page.getByRole("navigation", { name: "Dot sections" }).getByRole("link", { name: "Chat" }).click(); - const bubble = page.getByRole("article", { name: "You" }).filter({ hasText: "Is the 9 am flight still the cheapest?" }); - await expect(bubble.getByText("via Telegram")).toBeVisible(); - await expect(page.getByRole("log").getByRole("article", { name: "channels-telegram" }).getByText("echo: Is the 9 am flight still the cheapest?")).toBeVisible(); - await expect.poll(() => harness.bots.sent(token, 4242).map((message) => message.text)).toContain("echo: Is the 9 am flight still the cheapest?"); - - // The same message is in the Activity log with its chip. - await page.goto(`${harness.webUrl}/dots/${dot.id}/activity`); - await expect(page.getByRole("list", { name: "Events" }).getByRole("listitem").filter({ hasText: "via Telegram" })).toBeVisible(); -}); - -test("the switches of a channel are saved and kept, a person is revoked and Telegram is disconnected, each after asking where it is final", async ({ page, harness }) => { - const dot = await harness.createDot("channels-settings"); - const token = makeBot(harness, 700002, "e2e_settings_bot"); - await harness.api.putTelegramChannel(dot.id, token); - const code = (await harness.api.pairChannel(dot.id, "telegram")).code; - harness.bots.say(token, `/start ${code}`, { id: 5151, first_name: "Bea" }); - await expect.poll(async () => (await harness.api.channels(dot.id))[0]?.peers.length).toBe(1); - await page.goto(`${harness.webUrl}/dots/${dot.id}/channels`); - - const show = telegram(page).getByRole("switch", { name: "Show what the Dot wants to run" }); - // A new channel has everything on. - await expect(show).toHaveAttribute("aria-checked", "true"); - await show.click(); - await expect(show).toHaveAttribute("aria-checked", "false"); - await telegram(page).getByRole("switch", { name: "Tell me when a task ends" }).click(); - await expect(telegram(page).getByRole("switch", { name: "Tell me when a task ends" })).toHaveAttribute("aria-checked", "false"); - expect((await harness.api.channels(dot.id))[0]!.settings).toEqual({ approvals: true, notify_tasks: false, show_arguments: false }); - // Kept: the host has them, so a reload shows them. - await page.reload(); - await expect(telegram(page).getByRole("switch", { name: "Show what the Dot wants to run" })).toHaveAttribute("aria-checked", "false"); - await expect(telegram(page).getByRole("switch", { name: "Tell me when a task ends" })).toHaveAttribute("aria-checked", "false"); - - await telegram(page).getByRole("button", { name: "Pause" }).click(); - await expect(telegram(page).getByText("Paused")).toBeVisible(); - await telegram(page).getByRole("button", { name: "Resume" }).click(); - await expect(telegram(page).getByText("Connected")).toBeVisible(); - - // Revoked: the host drops her, and the bot no longer answers her. - await telegram(page).getByRole("button", { name: "Revoke Bea" }).click(); - await page.getByRole("dialog", { name: "Revoke Bea?" }).getByRole("button", { name: "Revoke" }).click(); - await expect(telegram(page).getByText(/Nobody yet/)).toBeVisible(); - expect((await harness.api.channels(dot.id))[0]!.peers).toEqual([]); - - await telegram(page).getByRole("button", { name: "Disconnect" }).click(); - await expect(page.getByRole("dialog", { name: "Disconnect Telegram?" })).toBeVisible(); - expect(await harness.api.channels(dot.id)).toHaveLength(1); - await page.getByRole("dialog", { name: "Disconnect Telegram?" }).getByRole("button", { name: "Disconnect" }).click(); - await expect(telegram(page).getByLabel("Bot token")).toBeVisible(); - expect(await harness.api.channels(dot.id)).toEqual([]); -}); - -// The Inbox counts what needs the person across every Dot, so this test has a host of its own (`fresh`) and leaves it -// empty, as the Inbox's own tests expect to find it. -test.describe("a channel that has to be linked again", () => { - test.afterEach(async ({ fresh }) => { - for (const dot of await fresh.api.listDots()) await fresh.api.deleteDot(dot.id); - await expect.poll(async () => (await fresh.api.listDots()).length).toBe(0); - }); - - test("a Telegram token that was revoked puts the Dot in the rail, the tab and the Inbox, and a new token clears them", async ({ page, fresh: harness }) => { - const dot = await harness.createDot("channels-relink"); - const token = makeBot(harness, 700003, "e2e_relink_bot"); - const next = makeBot(harness, 700004, "e2e_relink_two_bot"); - await harness.api.putTelegramChannel(dot.id, token); - const code = (await harness.api.pairChannel(dot.id, "telegram")).code; - harness.bots.say(token, `/start ${code}`, { id: 6161, first_name: "Cy" }); - await expect.poll(async () => (await harness.api.channels(dot.id))[0]?.peers.length).toBe(1); - await page.goto(`${harness.webUrl}/dots/${dot.id}/channels`); - await expect(telegram(page).getByText("Connected")).toBeVisible(); - await expect(page.getByLabel("Telegram needs linking again")).toHaveCount(0); - - // Revoked in @BotFather: the next poll is refused, and the page hears it without a reload. - harness.bots.revoke(token); - await expect(telegram(page).getByText("Telegram needs a new token")).toBeVisible(); - await expect(telegram(page).getByText("Needs linking again")).toBeVisible(); - // The Dot is marked in the rail and on its Channels tab, and the Inbox counts it. - await expect(page.getByRole("navigation", { name: "Dots" }).getByLabel("Telegram needs linking again")).toBeVisible(); - await expect(channelsTab(page).getByLabel("needs linking again")).toBeVisible(); - await expect(page.getByRole("link", { name: /^Inbox/ }).getByLabel("1 need you")).toBeVisible(); - await expect(telegram(page).getByRole("listitem").filter({ hasText: "Cy" })).toBeVisible(); - await page.getByRole("link", { name: /^Inbox/ }).click(); - const card = page.getByRole("article", { name: "Telegram of channels-relink needs linking again" }); - await expect(card).toBeVisible(); - await card.getByRole("link", { name: "Open channels" }).click(); - await expect(page).toHaveURL(new RegExp(`/dots/${dot.id}/channels$`)); - - // A new token: the channel starts again, the person paired stays, and nothing marks the Dot any more. - await telegram(page).getByLabel("New bot token").fill(next); - await telegram(page).getByRole("button", { name: "Use this token" }).click(); - await expect(telegram(page).getByText("Connected")).toBeVisible(); - await expect(telegram(page).getByRole("link", { name: "@e2e_relink_two_bot" })).toBeVisible(); - await expect(telegram(page).getByRole("listitem").filter({ hasText: "Cy" })).toBeVisible(); - await expect(page.getByLabel("Telegram needs linking again")).toHaveCount(0); - await expect(page.getByRole("link", { name: /^Inbox/ }).getByLabel(/need you/)).toHaveCount(0); - }); - - for (const how of ["pauses it", "disconnects it"] as const) { - test(`the marks go as soon as the person ${how}, without a reload`, async ({ page, fresh: harness }) => { - const dot = await harness.createDot("channels-relink"); - const token = makeBot(harness, how === "pauses it" ? 700005 : 700006, how === "pauses it" ? "e2e_pause_bot" : "e2e_remove_bot"); - await harness.api.putTelegramChannel(dot.id, token); - await page.goto(`${harness.webUrl}/dots/${dot.id}/channels`); - await expect(telegram(page).getByText("Connected")).toBeVisible(); - harness.bots.revoke(token); - await expect(telegram(page).getByText("Telegram needs a new token")).toBeVisible(); - await expect(channelsTab(page).getByLabel("needs linking again")).toBeVisible(); - await expect(page.getByRole("link", { name: /^Inbox/ }).getByLabel("1 need you")).toBeVisible(); - - if (how === "pauses it") { - await telegram(page).getByRole("button", { name: "Pause" }).click(); - await expect(telegram(page).getByText("Paused")).toBeVisible(); - } else { - await telegram(page).getByRole("button", { name: "Disconnect" }).click(); - await page.getByRole("dialog", { name: "Disconnect Telegram?" }).getByRole("button", { name: "Disconnect" }).click(); - await expect(telegram(page).getByText("Not connected")).toBeVisible(); - } - await expect(channelsTab(page).getByLabel("needs linking again")).toHaveCount(0); - await expect(page.getByRole("navigation", { name: "Dots" }).getByLabel("Telegram needs linking again")).toHaveCount(0); - await expect(page.getByRole("link", { name: /^Inbox/ }).getByLabel(/need you/)).toHaveCount(0); - }); - } -}); - -test("WhatsApp is linked by scanning the codes the host makes, then pairs a person and carries their messages", async ({ page, harness }) => { - const dot = await harness.createDot("channels-whatsapp"); - const made = harness.whatsapp.connections.length; - await page.goto(`${harness.webUrl}/dots/${dot.id}/channels`); - - await expect(whatsapp(page).getByText("Read this before you link a number")).toBeVisible(); - await expect(whatsapp(page).getByText(/can answer by banning the linked number/)).toBeVisible(); - await whatsapp(page).getByRole("button", { name: "Link WhatsApp" }).click(); - await expect.poll(() => harness.whatsapp.connections.length).toBe(made + 1); - - // Each code the host makes is drawn; the next one replaces it. - const qr = whatsapp(page).getByRole("img", { name: "QR code to link WhatsApp" }); - harness.whatsapp.current.showCode("2@e2e-code-one,AAAA,BBBB"); - await expect(qr).toBeVisible(); - await expect(whatsapp(page).getByText("Scan this code with the phone that has the number")).toBeVisible(); - const first = await qr.locator("path").getAttribute("d"); - harness.whatsapp.current.showCode("2@e2e-code-two,CCCC,DDDD"); - await expect.poll(() => qr.locator("path").getAttribute("d")).not.toBe(first); - - // The phone scanned it. - harness.whatsapp.current.open("15550001111"); - await expect(whatsapp(page).getByText("People paired")).toBeVisible(); - await expect(whatsapp(page).getByText("Connected")).toBeVisible(); - await expect(whatsapp(page).getByText("+15550001111")).toBeVisible(); - await expect(qr).toHaveCount(0); - expect((await harness.api.channels(dot.id))[0]).toMatchObject({ kind: "whatsapp", status: "connected", account: "15550001111" }); - - // Pair with the words, from a phone; then a message from it reaches the chat and the answer goes back. - await whatsapp(page).getByRole("button", { name: "Link your WhatsApp" }).click(); - const code = await codeOf(page, "Open WhatsApp"); - await expect(whatsapp(page).getByText(`pair ${code}`)).toBeVisible(); - const phone = "393331112222@s.whatsapp.net"; - harness.whatsapp.current.receive(phone, `pair ${code}`, { senderName: "Dee" }); - const people = whatsapp(page).getByRole("region", { name: "People paired on WhatsApp" }); - await expect(people.getByRole("listitem")).toContainText("Dee"); - harness.whatsapp.current.receive(phone, "Any news on the fares?", { senderName: "Dee" }); - await page.getByRole("navigation", { name: "Dot sections" }).getByRole("link", { name: "Chat" }).click(); - await expect(page.getByRole("article", { name: "You" }).filter({ hasText: "Any news on the fares?" }).getByText("via WhatsApp")).toBeVisible(); - await expect.poll(() => harness.whatsapp.current.texts(phone)).toContain("echo: Any news on the fares?"); -}); - -test("a WhatsApp link that fails says why and starts over", async ({ page, harness }) => { - const dot = await harness.createDot("channels-whatsapp-failed"); - const made = harness.whatsapp.connections.length; - await page.goto(`${harness.webUrl}/dots/${dot.id}/channels`); - await whatsapp(page).getByRole("button", { name: "Link WhatsApp" }).click(); - await expect.poll(() => harness.whatsapp.connections.length).toBe(made + 1); - harness.whatsapp.current.showCode("2@e2e-failing-code"); - await expect(whatsapp(page).getByRole("img", { name: "QR code to link WhatsApp" })).toBeVisible(); - - // The code was not scanned in time: the connection ends before the link did. - harness.whatsapp.current.end({ reason: "code_expired" }); - await expect(whatsapp(page).getByText("WhatsApp is not linked")).toBeVisible(); - await expect(whatsapp(page).getByText(/the code expired, or the connection dropped, before it was scanned/)).toBeVisible(); - await expect(whatsapp(page).getByRole("img", { name: "QR code to link WhatsApp" })).toHaveCount(0); - // It is a login only the person can redo: the Dot is marked, and the card offers it. - await expect(page.getByRole("navigation", { name: "Dots" }).getByLabel("WhatsApp needs linking again")).toBeVisible(); - - await whatsapp(page).getByRole("button", { name: "Link again" }).click(); - await expect.poll(() => harness.whatsapp.connections.length).toBe(made + 2); - harness.whatsapp.current.showCode("2@e2e-second-try"); - await expect(whatsapp(page).getByRole("img", { name: "QR code to link WhatsApp" })).toBeVisible(); - await expect(whatsapp(page).getByText("WhatsApp is not linked")).toHaveCount(0); -}); - -test("the Channels page fits a phone: no sideways scroll at 390 px, with a pairing code and a long bot name on it", async ({ page, harness }) => { - const dot = await harness.createDot("channels-phone"); - const token = makeBot(harness, 700005, "a_bot_with_a_very_long_name_that_has_no_break_in_it_at_all_bot"); - await harness.api.putTelegramChannel(dot.id, token); - await page.setViewportSize({ width: 390, height: 844 }); - await page.goto(`${harness.webUrl}/dots/${dot.id}/channels`); - await telegram(page).getByRole("button", { name: "Link your Telegram" }).click(); - await expect(telegram(page).getByRole("img", { name: /QR code/ })).toBeVisible(); - expect(await page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth)).toBeLessThanOrEqual(0); -}); - -for (const scheme of ["light", "dark"] as const) { - test(`the Channels page is readable in ${scheme}: chips, notes, switches' words, links and the countdown`, async ({ page, harness }) => { - await page.emulateMedia({ colorScheme: scheme }); - const dot = await harness.createDot(`channels-look-${scheme}`); - const token = makeBot(harness, scheme === "light" ? 700006 : 700007, `e2e_look_${scheme}_bot`); - await harness.api.putTelegramChannel(dot.id, token); - await page.goto(`${harness.webUrl}/dots/${dot.id}/channels`); - await telegram(page).getByRole("button", { name: "Link your Telegram" }).click(); - await expect(telegram(page).getByRole("timer")).toBeVisible(); - harness.bots.revoke(token); - await expect(telegram(page).getByText("Telegram needs a new token")).toBeVisible(); - - const texts: Record = { - "the state chip": "section[aria-labelledby=channel-telegram] header span:has-text('Needs linking again')", - "the bot link": "section[aria-labelledby=channel-telegram] header a", - "the alert's text": "[data-slot=alert-description] p", - "the hint of the token field": "#telegram-token-hint", - "the countdown": "[role=timer]", - "a switch's words": "section[aria-label='What the channel does'] p", - "the privacy note": "section[aria-labelledby=channel-telegram] p.text-xs:has-text('not end-to-end encrypted')", - "the Pause button": "button:has-text('Pause')", - "the open tab": "nav[aria-label='Dot sections'] a[aria-current=page]", - }; - for (const [name, selector] of Object.entries(texts)) { - expect((await lookOf(page, selector)).contrast, name).toBeGreaterThanOrEqual(4.5); - } - }); -} diff --git a/apps/web/e2e/chat.spec.ts b/apps/web/e2e/chat.spec.ts deleted file mode 100644 index c59ff7da..00000000 --- a/apps/web/e2e/chat.spec.ts +++ /dev/null @@ -1,298 +0,0 @@ -import type { Locator } from "@playwright/test"; -import type { FakeGuest } from "@invisible-dots/scheduler/testing"; -import type { OutboundEventDataMap } from "@invisible-dots/shared"; -import { expect, test } from "./fixtures.js"; - -/** A tool call as the engine reports it once it has ended. */ -function called(tool: string, target: string, change: Partial = {}): OutboundEventDataMap["tool.called"] { - return { tool, permission: "computer.exec", decision: "allow", ok: true, duration_ms: 120, target, ...change }; -} - -/** A guest that takes a message up, does some work, and keeps the turn open: the test says when it answers. */ -function startTurn(guest: FakeGuest): void { - guest.onInbound = (event, g) => { - if (event.type !== "user.message") return; - g.emit("agent.state", { state: "THINKING" }); - g.emit("tool.called", called("list_dir", "/home/dot/workspace")); - g.emit("tool.called", called("read_file", "/home/dot/workspace/fares.csv")); - }; -} - -test("the person talks to the Dot: the working row follows the turn, the steps sit between the messages, the answer is markdown", async ({ page, harness }) => { - const dot = await harness.createDot("chat-turn"); - const guest = harness.driver.guestOf(dot.id); - startTurn(guest); - await page.goto(`${harness.webUrl}/dots/${dot.id}/chat`); - - // A Dot nobody has talked to: ways to begin that fill the box and do not send. - await expect(page.getByRole("heading", { name: "Say hello to chat-turn" })).toBeVisible(); - await expect(page.getByText("Tell it what you need.")).toBeVisible(); - const box = page.getByRole("textbox", { name: "Message" }); - await expect(box).toBeFocused(); - await page.getByRole("button", { name: "What can you do on your computer?" }).click(); - await expect(box).toHaveValue("What can you do on your computer?"); - await box.fill(""); - - // Nothing below reloads the page: this marker would not survive it. - await page.evaluate(() => ((window as unknown as { marker: number }).marker = 11)); - - await box.fill("What is the cheapest fare to Lisbon?"); - await box.press("Enter"); - const log = page.getByRole("log"); - await expect(log.getByRole("article", { name: "You" }).getByText("What is the cheapest fare to Lisbon?")).toBeVisible(); - await expect(box).toHaveValue(""); - await expect(page.getByRole("heading", { name: "Say hello to chat-turn" })).toHaveCount(0); - - // The Dot is mid-turn: its row says so, and names the last thing it did. - const working = log.getByRole("status").filter({ hasText: "Thinking..." }); - await expect(working).toBeVisible(); - await expect(working).toContainText("Last step: read a file /home/dot/workspace/fares.csv"); - const steps = log.getByRole("list", { name: "What the Dot did" }); - await expect(steps.getByTestId("activity-step")).toHaveText(["Listed a folder/home/dot/workspace", "Read a file/home/dot/workspace/fares.csv"]); - - // A note it saves is the call that wrote it, a file of its memory folder: no chip, nothing to open elsewhere. - guest.emit("tool.called", called("write_file", "/home/dot/memory/trips/lisbon.md", { permission: "files.write" })); - await expect(steps.getByTestId("activity-step").last()).toHaveText("Wrote a file/home/dot/memory/trips/lisbon.md"); - await expect(log.getByText("Remembered")).toHaveCount(0); - - // A run of more than three calls folds into one line, which opens on a click: with the three above, seven. - for (const target of ["EUR", "TAP", "Ryanair", "easyJet"]) guest.emit("tool.called", called("grep", target)); - await expect(log.getByRole("button", { name: "7 steps" })).toBeVisible(); - await log.getByRole("button", { name: "7 steps" }).click(); - await expect(log.getByRole("list", { name: "Steps" }).getByTestId("activity-step")).toHaveCount(7); - - guest.emit("message.assistant", { text: "The cheapest is **EUR 41** on Tuesday:\n\n```\nTAP 41 EUR\n```\n\n[the source](https://example.com/fares)" }); - guest.emit("agent.state", { state: "IDLE" }); - const answer = log.getByRole("article", { name: "chat-turn" }); - await expect(answer.locator("strong")).toHaveText("EUR 41"); - await expect(answer.locator("pre")).toHaveText("TAP 41 EUR"); - await expect(answer.getByRole("link", { name: "the source" })).toHaveAttribute("href", "https://example.com/fares"); - await expect(answer.getByRole("link", { name: "the source" })).toHaveAttribute("rel", "noreferrer"); - await expect(working).toHaveCount(0); - - // Steps came before the answer, the question before the steps. - const text = (await log.textContent()) ?? ""; - expect(text.indexOf("What is the cheapest fare")).toBeLessThan(text.indexOf("7 steps")); - expect(text.indexOf("7 steps")).toBeLessThan(text.indexOf("The cheapest is")); - expect(await page.evaluate(() => (window as unknown as { marker: number }).marker)).toBe(11); - - // The conversation comes back as it was on a fresh load: messages from the host, steps from its log. - await page.reload(); - await expect(page.getByRole("log").getByRole("article", { name: "chat-turn" }).locator("strong")).toHaveText("EUR 41"); - await expect(page.getByRole("log").getByRole("button", { name: "7 steps" })).toBeVisible(); -}); - -test("a conversation longer than a page opens on its newest messages, and goes back a page at a time", async ({ page, harness }) => { - const dot = await harness.createDot("chat-long"); - // 60 questions, each answered by the fake guest's echo: 120 messages, one page and a bit. - for (let i = 0; i < 60; i++) await harness.api.sendMessage(dot.id, `question ${i}`); - await expect.poll(async () => (await harness.api.messages(dot.id)).length).toBe(120); - - await page.goto(`${harness.webUrl}/dots/${dot.id}/chat`); - const log = page.getByRole("log"); - await expect(log.getByText("echo: question 59", { exact: true })).toBeVisible(); - await expect(log.getByText("question 0", { exact: true })).toHaveCount(0); - await expect(log.getByRole("article")).toHaveCount(100); - - await page.getByRole("button", { name: "Show earlier messages" }).click(); - await expect(log.getByText("question 0", { exact: true })).toBeVisible(); - await expect(log.getByRole("article")).toHaveCount(120); - await expect(page.getByRole("button", { name: "Show earlier messages" })).toHaveCount(0); - - // What the Dot says next still shows below, after the page that was read. - harness.driver.guestOf(dot.id).emit("message.assistant", { text: "one more, live" }); - await expect(log.getByText("one more, live")).toBeVisible(); -}); - -test("a command that started a terminal session says so in the chat and in the Activity, and one that did not says it ran", async ({ page, harness }) => { - const dot = await harness.createDot("chat-terminal"); - const guest = harness.driver.guestOf(dot.id); - await harness.api.sendMessage(dot.id, "open a python shell"); - guest.emit("tool.called", called("exec", "python3", { tty: true })); - guest.emit("tool.called", called("exec", "ls /home/dot")); - await expect.poll(async () => (await harness.api.events(dot.id, { types: ["tool.called"] })).length).toBe(2); - - await page.goto(`${harness.webUrl}/dots/${dot.id}/chat`); - const steps = page.getByRole("log").getByTestId("activity-step"); - await expect(steps).toHaveText(["Started a terminal sessionpython3", "Ran a commandls /home/dot"]); - - await page.goto(`${harness.webUrl}/dots/${dot.id}/activity`); - const rows = page.getByRole("list", { name: "Events" }).getByRole("listitem"); - await expect(rows.filter({ hasText: "Started a terminal session" })).toHaveCount(1); - await expect(rows.filter({ hasText: "Ran a command" })).toHaveCount(1); -}); - -test("an approval the Dot asks for in the chat is a card where it was asked, the person answers it there, and its answer is a receipt", async ({ page, harness }) => { - const dot = await harness.createDot("chat-approval"); - const guest = harness.driver.guestOf(dot.id); - await harness.api.sendMessage(dot.id, "please delete the old profile"); - await page.goto(`${harness.webUrl}/dots/${dot.id}/chat`); - await expect(page.getByRole("log").getByText("please delete the old profile", { exact: true })).toBeVisible(); - - guest.requestApproval(undefined); - const log = page.getByRole("log"); - const card = log.getByRole("article", { name: "Wants to delete a browser identity" }); - await expect(card).toBeVisible(); - await expect(card.getByText("shop-abc123", { exact: true })).toBeVisible(); - // The Dot's header says it waits for the person too, and links to the Inbox filtered to it. - await expect(page.getByRole("link", { name: "Dot state: Waiting for you" })).toHaveAttribute("href", `/inbox?dot=${dot.id}`); - - await card.getByRole("button", { name: "Allow once" }).click(); - await expect(log.getByText(/^Allowed:/)).toBeVisible(); - await expect(card).toHaveCount(0); - expect((await harness.api.listApprovals("approved")).map((a) => a.dot_id)).toEqual([dot.id]); -}); - -test("the computer panel shows the desktop, where a browser the Dot opens is a window, and nothing else to pick", async ({ page, harness }) => { - const dot = await harness.createDot("chat-panel"); - const guest = harness.driver.guestOf(dot.id); - await page.setViewportSize({ width: 1400, height: 900 }); - await page.goto(`${harness.webUrl}/dots/${dot.id}/chat`); - // There is nothing to open: the computer is always beside the thread. - await expect(page.getByRole("button", { name: "Watch the computer" })).toHaveCount(0); - const panel = page.getByRole("complementary", { name: "Computer" }); - await expect(panel.getByText("The Dot has control. You are watching.")).toBeVisible(); - await expect(panel.getByRole("img", { name: /current picture of the desktop/ })).toHaveAttribute("src", /^blob:/); - // The thread is still beside it, and the person can still write. - await expect(page.getByRole("textbox", { name: "Message" })).toBeVisible(); - - // A browser the Dot opens is a window of that desktop: the panel offers nothing else to watch. - const identity = await harness.api.createIdentity(dot.id, { name: "shopping" }); - guest.launchIdentity(identity.id); - await expect.poll(async () => (await harness.api.listIdentities(dot.id))[0]?.status).toBe("open"); - await expect(panel.getByRole("group", { name: "What to watch" })).toHaveCount(0); - await expect(panel.getByRole("button", { name: /^Browser:/ })).toHaveCount(0); - await expect(panel.getByRole("img", { name: /current picture of the desktop/ })).toBeVisible(); - - // The thread and the computer are side by side, and both reach the bottom of the window (the page's own padding - // below them, no more): the page does not scroll, and nothing is left empty under them. - const thread = page.getByRole("log").locator("xpath=ancestor::div[contains(@class,'rounded-lg')][1]"); - const [left, right] = [await thread.boundingBox(), await panel.boundingBox()]; - expect(left!.x + left!.width).toBeLessThanOrEqual(right!.x); - for (const box of [left!, right!]) expect(900 - (box.y + box.height)).toBeLessThanOrEqual(24); - expect(await page.evaluate(() => document.getElementById("main")!.scrollHeight - document.getElementById("main")!.clientHeight)).toBeLessThanOrEqual(0); - await expectPictureInPlace(panel); -}); - -/** - * The desktop's picture (taller than its place, at 1280 x 800) is drawn whole inside the panel and above its controls: - * it is fitted to the place, never sized by its own pixels, which would push the controls out of view. - */ -async function expectPictureInPlace(panel: Locator) { - const picture = (await panel.getByRole("img", { name: /current picture of the desktop/ }).boundingBox())!; - const [refresh, outer] = [(await panel.getByRole("button", { name: "Refresh" }).boundingBox())!, (await panel.boundingBox())!]; - expect(picture.y + picture.height).toBeLessThanOrEqual(refresh.y); - expect(picture.x + picture.width).toBeLessThanOrEqual(outer.x + outer.width); - expect(refresh.y + refresh.height).toBeLessThanOrEqual(outer.y + outer.height); -} - -test("a stopped computer says so in the panel, and the chat still takes a message", async ({ page, harness }) => { - const dot = await harness.createDot("chat-stopped"); - await harness.api.stopComputer(dot.id); - await page.setViewportSize({ width: 1400, height: 900 }); - await page.goto(`${harness.webUrl}/dots/${dot.id}/chat`); - await expect(page.getByRole("button", { name: /^Computer: Stopped/ })).toBeVisible(); - await expect(page.getByRole("complementary", { name: "Computer" }).getByText(/The computer is stopped/)).toBeVisible(); - await expect(page.getByText(/Sending a message wakes it/)).toBeVisible(); -}); - -test("the chat works from the keyboard and keeps a draft across a reload", async ({ page, harness }) => { - const dot = await harness.createDot("chat-keys"); - await page.goto(`${harness.webUrl}/dots/${dot.id}/chat`); - const box = page.getByRole("textbox", { name: "Message" }); - await expect(box).toBeFocused(); - await page.keyboard.type("first line"); - await page.keyboard.press("Shift+Enter"); - await page.keyboard.type("second line"); - await expect(box).toHaveValue("first line\nsecond line"); - await expect(page.getByRole("log").getByRole("article", { name: "You" })).toHaveCount(0); - - await page.reload(); - await expect(box).toHaveValue("first line\nsecond line"); - await page.keyboard.press("Enter"); - const bubble = page.getByRole("log").getByRole("article", { name: "You" }); - await expect(bubble).toHaveText(/first line\s*second line/); - await expect(box).toHaveValue(""); - await expect(page.getByRole("log").getByRole("article", { name: "chat-keys" })).toContainText("echo: first line"); - await page.reload(); - await expect(box).toHaveValue(""); -}); - -test("the chat fits a phone: long lines wrap, code scrolls inside its block, and the computer is a strip above the thread", async ({ page, harness }) => { - const dot = await harness.createDot("chat-phone"); - const guest = harness.driver.guestOf(dot.id); - await page.setViewportSize({ width: 390, height: 844 }); - await harness.api.sendMessage(dot.id, "An unbroken line of text that is long enough to wrap on a narrow screen several times over before it ends"); - await page.goto(`${harness.webUrl}/dots/${dot.id}/chat`); - await expect(page.getByRole("log").getByRole("article", { name: "You" })).toBeVisible(); - guest.emit("tool.called", called("read_file", "/home/dot/workspace/a/very/long/path/that/goes/on/and/on/and/on/and/on/and/on/and/file.txt")); - guest.emit("message.assistant", { text: `Here it is:\n\n\`\`\`\n${"x".repeat(200)}\n\`\`\`\n\n| a | b |\n|---|---|\n| ${"y".repeat(120)} | 2 |` }); - await expect(page.getByRole("log").getByRole("article", { name: "chat-phone" }).locator("pre")).toBeVisible(); - const overflow = () => page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth); - expect(await overflow()).toBeLessThanOrEqual(0); - - const panel = page.getByRole("complementary", { name: "Computer" }); - await expect(panel.getByRole("img", { name: /current picture of the desktop/ })).toHaveAttribute("src", /^blob:/); - const box = page.getByRole("textbox", { name: "Message" }); - await expect(box).toBeVisible(); - expect((await panel.boundingBox())!.y).toBeLessThan((await box.boundingBox())!.y); - await expectPictureInPlace(panel); - expect(await overflow()).toBeLessThanOrEqual(0); -}); - -test("the user's bubble and the Dot's words are readable in light and in dark", async ({ page, harness }) => { - const dot = await harness.createDot("chat-look"); - await harness.api.sendMessage(dot.id, "hello there"); - for (const scheme of ["light", "dark"] as const) { - await page.emulateMedia({ colorScheme: scheme }); - await page.goto(`${harness.webUrl}/dots/${dot.id}/chat`); - await expect(page.getByRole("log").getByRole("article", { name: "chat-look" })).toContainText("echo: hello there"); - const ratio = await page.getByRole("log").getByRole("article", { name: "You" }).locator("p").evaluate((element) => { - const context = document.createElement("canvas").getContext("2d", { willReadFrequently: true })!; - const rgb = (color: string): number[] => { - context.clearRect(0, 0, 1, 1); - context.fillStyle = color; - context.fillRect(0, 0, 1, 1); - return [...context.getImageData(0, 0, 1, 1).data].slice(0, 3); - }; - const luminance = (channels: number[]) => { - const [r = 0, g = 0, b = 0] = channels.map((value) => { - const s = value / 255; - return s <= 0.03928 ? s / 12.92 : ((s + 0.055) / 1.055) ** 2.4; - }); - return 0.2126 * r + 0.7152 * g + 0.0722 * b; - }; - const style = getComputedStyle(element); - const [high = 0, low = 0] = [luminance(rgb(style.color)), luminance(rgb(style.backgroundColor))].sort((a, b) => b - a); - return (high + 0.05) / (low + 0.05); - }); - expect(ratio, scheme).toBeGreaterThanOrEqual(4.5); - } -}); - -test("a Dot opened by its name moves to its id, and the chat hears the Dot live", async ({ page, harness }) => { - const dot = await harness.createDot("chat-by-name"); - const guest = harness.driver.guestOf(dot.id); - await page.goto(`${harness.webUrl}/dots/chat-by-name/chat`); - await expect(page).toHaveURL(new RegExp(`/dots/${dot.id}/chat$`)); - await expect(page.getByRole("heading", { level: 1, name: "chat-by-name" })).toBeVisible(); - await expect(page.getByRole("navigation", { name: "Dot sections" }).getByRole("link", { name: "Chat" })).toHaveAttribute("aria-current", "page"); - - guest.emit("agent.state", { state: "THINKING" }); - await expect(page.getByRole("log").getByRole("status").filter({ hasText: "Thinking..." })).toBeVisible(); - guest.emit("message.assistant", { text: "Heard on the id." }); - await expect(page.getByRole("log").getByRole("article", { name: "chat-by-name" }).getByText("Heard on the id.")).toBeVisible(); -}); - -test("a message that came through a chat says which one, and one typed here says nothing", async ({ page, harness }) => { - const dot = await harness.createDot("chat-origin"); - const origin = { channel: "telegram", binding_id: "bind_e2e", chat_id: "4242", external_id: "99:1" } as const; - await harness.control.scheduler.sendMessage(dot.id, "Is the 9 am flight still the cheapest?", origin); - await harness.control.scheduler.sendMessage(dot.id, "And the one on Friday?"); - await page.goto(`${harness.webUrl}/dots/${dot.id}/chat`); - - const phone = page.getByRole("article", { name: "You" }).filter({ hasText: "Is the 9 am flight still the cheapest?" }); - await expect(phone.getByText("via Telegram")).toBeVisible(); - await expect(page.getByRole("article", { name: "You" }).filter({ hasText: "And the one on Friday?" })).toBeVisible(); - await expect(page.getByText(/^via /)).toHaveCount(1); -}); diff --git a/apps/web/e2e/computer.spec.ts b/apps/web/e2e/computer.spec.ts deleted file mode 100644 index ad597018..00000000 --- a/apps/web/e2e/computer.spec.ts +++ /dev/null @@ -1,136 +0,0 @@ -import { expect, test } from "./fixtures.js"; -import { lookOf } from "./look.js"; - -test("the Computer page shows the screen, the files and what the computer uses", async ({ page, harness }) => { - const dot = await harness.createDot("computer-views"); - const guest = harness.driver.guestOf(dot.id); - guest.putFile("/home/dot/notes.txt", "Remember the milk.\n"); - guest.putFile("/home/dot/memory/trips/rome.md", "Rome in May"); - guest.putFile("/home/dot/shot.png", Uint8Array.from([0x89, 0x50, 0x4e, 0x47])); - guest.putFile("/home/dot/archive.zip", Uint8Array.from([0x50, 0x4b, 3, 4])); - await page.goto(`${harness.webUrl}/dots/${dot.id}/computer`); - - await expect(page.getByRole("link", { name: "Computer", exact: true })).toHaveAttribute("aria-current", "page"); - await expect(page.getByRole("link", { name: "Browser identities" })).toHaveCount(0); - - // Screen: a live picture, and no pretence of control. - await expect(page.getByRole("img", { name: /current picture of the desktop/ })).toHaveAttribute("src", /^blob:/); - await expect(page.getByText("The Dot has control. You are watching.")).toBeVisible(); - await expect(page.getByText("LIVE", { exact: true })).toBeVisible(); - - // Files: folders first, a folder opens, a text file shows its text, and the way back is a link. - const views = page.getByRole("navigation", { name: "Computer views" }); - await views.getByRole("link", { name: "Files" }).click(); - await expect(page).toHaveURL(/view=files$/); - const table = page.getByRole("table"); - await expect(table.getByRole("row")).toHaveCount(5); - await expect(table.getByRole("row").nth(1)).toContainText("memory"); - await table.getByRole("link", { name: /^memory/ }).click(); - await table.getByRole("link", { name: /^trips/ }).click(); - await table.getByRole("link", { name: "rome.md" }).click(); - await expect(page.getByLabel("Contents of rome.md")).toHaveText("Rome in May"); - await expect(page.getByRole("navigation", { name: "Folder" }).getByRole("link")).toHaveText(["Home", "memory", "trips"]); - await page.getByRole("navigation", { name: "Folder" }).getByRole("link", { name: "Home" }).click(); - await expect(page).toHaveURL(/path=%2Fhome%2Fdot$/); - - // A reload lands on the same file: the address holds it. - await table.getByRole("link", { name: "notes.txt" }).click(); - await expect(page.getByLabel("Contents of notes.txt")).toHaveText("Remember the milk."); - await page.reload(); - await expect(page.getByLabel("Contents of notes.txt")).toHaveText("Remember the milk."); - - await table.getByRole("link", { name: "shot.png" }).click(); - await expect(page.getByRole("img", { name: "The picture shot.png" })).toHaveAttribute("src", /^blob:/); - - // Nothing but a download for what a page does not show. - await table.getByRole("link", { name: "archive.zip" }).click(); - const preview = page.getByRole("region", { name: "File archive.zip" }); - await expect(preview.getByText(/not shown here/)).toBeVisible(); - const download = page.waitForEvent("download"); - await preview.getByRole("button", { name: "Download" }).click(); - expect((await download).suggestedFilename()).toBe("archive.zip"); - - // A folder that is not there says so, and leads home. - await page.goto(`${harness.webUrl}/dots/${dot.id}/computer?view=files&path=/home/dot/nowhere`); - await expect(page.getByRole("alert").getByText("This folder does not exist (any more).")).toBeVisible(); - await page.getByRole("link", { name: "Go to the home folder" }).click(); - await expect(page.getByRole("table")).toBeVisible(); - - // Usage: given and used, the images, and the actions the state allows. - await views.getByRole("link", { name: "Usage" }).click(); - await expect(page.getByRole("meter", { name: "Memory used" })).toHaveAttribute("aria-valuenow", "25"); - await expect(page.getByRole("meter", { name: "Disk used" })).toHaveAttribute("aria-valuenow", "13"); - await expect(page.getByRole("region", { name: "Given to the computer" }).getByText("15m")).toBeVisible(); - await expect(page.getByRole("region", { name: "Images" })).toContainText("golden-"); - await expect(page.getByRole("region", { name: "Model spend" })).toContainText("$0.00"); - await expect(page.getByRole("button", { name: "Start", exact: true })).toBeDisabled(); - // The Dot's automations are the Dot's own: the page says when the next one is due (none is yet). - await expect(page.getByRole("region", { name: "Automations" })).toContainText("No automation is due."); - // The engine reports a run that is due: that is what the person's stop then holds back (a Dot with no automation has nothing to pause). - guest.emit("automation.next_run", { next_run_at_ms: Date.now() + 6 * 60 * 60 * 1000 }); - await expect(page.getByRole("region", { name: "Automations" })).toContainText("Next automation:"); - // Stopping asks, and says what it costs. - page.once("dialog", (dialog) => { - expect(dialog.message()).toContain("Its automations do not run while it is stopped"); - void dialog.accept(); - }); - await page.getByRole("button", { name: "Stop", exact: true }).click(); - await expect(page.getByRole("button", { name: /^Computer: Stopped/ })).toBeVisible(); - await expect(page.getByRole("region", { name: "Automations" })).toContainText("Paused: you stopped this computer, so its automations do not run"); - await expect(page.getByRole("button", { name: "Start", exact: true })).toBeEnabled(); - await expect(page.getByText("What it uses is shown while the computer runs.")).toBeVisible(); -}); - -test("a stopped computer says so on the screen and the files, and starts from there", async ({ page, harness }) => { - const dot = await harness.createDot("computer-stopped"); - harness.driver.guestOf(dot.id).putFile("/home/dot/notes.txt", "hello"); - await harness.api.stopComputer(dot.id); - await page.goto(`${harness.webUrl}/dots/${dot.id}/computer`); - await expect(page.getByRole("button", { name: /^Computer: Stopped/ })).toBeVisible(); - await expect(page.getByRole("status").filter({ hasText: "Start the computer to see its screen" })).toBeVisible(); - const views = page.getByRole("navigation", { name: "Computer views" }); - await expect(views.getByRole("link", { name: "Browser" })).toHaveCount(0); - await views.getByRole("link", { name: "Files" }).click(); - await expect(page.getByRole("status").filter({ hasText: "Start the computer to see its files" })).toBeVisible(); - - await page.getByRole("button", { name: "Start the computer" }).click(); - await expect(page.getByRole("link", { name: "notes.txt" })).toBeVisible(); - await expect(page.getByRole("button", { name: /^Computer: Running/ })).toBeVisible(); -}); - -test("the Computer page fits a phone: no sideways scroll at 390 px", async ({ page, harness }) => { - const dot = await harness.createDot("computer-phone"); - const guest = harness.driver.guestOf(dot.id); - guest.putFile("/home/dot/a-file-with-a-very-long-name-that-has-no-break-in-it-at-all-and-keeps-going-and-going.txt", "x"); - await page.setViewportSize({ width: 390, height: 844 }); - const overflow = () => page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth); - - await page.goto(`${harness.webUrl}/dots/${dot.id}/computer`); - await expect(page.getByRole("img", { name: /desktop/ })).toBeVisible(); - expect(await overflow()).toBeLessThanOrEqual(0); - for (const view of ["files", "usage"]) { - await page.getByRole("navigation", { name: "Computer views" }).getByRole("link", { name: new RegExp(`^${view}$`, "i") }).click(); - await expect(page).toHaveURL(new RegExp(`view=${view}`)); - if (view === "files") await expect(page.getByRole("table")).toBeVisible(); - if (view === "usage") await expect(page.getByRole("region", { name: "Images" })).toBeVisible(); - expect(await overflow(), view).toBeLessThanOrEqual(0); - } -}); - -for (const scheme of ["light", "dark"] as const) { - test(`the Computer page is readable in ${scheme}: the views bar and the labels of facts`, async ({ page, harness }) => { - await page.emulateMedia({ colorScheme: scheme }); - const dot = await harness.createDot(`computer-look-${scheme}`); - await page.goto(`${harness.webUrl}/dots/${dot.id}/computer?view=usage`); - await expect(page.getByRole("region", { name: "Images" })).toBeVisible(); - - const texts: Record = { - "the current view link": "nav[aria-label='Computer views'] a[aria-current=page]", - "another view link": "nav[aria-label='Computer views'] a:not([aria-current])", - "a fact's label": "main dl dt", - }; - for (const [name, selector] of Object.entries(texts)) { - expect((await lookOf(page, selector)).contrast, name).toBeGreaterThanOrEqual(4.5); - } - }); -} diff --git a/apps/web/e2e/create-dot.spec.ts b/apps/web/e2e/create-dot.spec.ts deleted file mode 100644 index c4e2e7a3..00000000 --- a/apps/web/e2e/create-dot.spec.ts +++ /dev/null @@ -1,140 +0,0 @@ -import { PERMISSIONS, resolvePermission } from "@invisible-dots/shared"; -import { expect, test } from "./fixtures.js"; - -test("a Dot is created by the form: its config is what the form showed, and the person lands in its chat", async ({ page, harness }) => { - await page.goto(`${harness.webUrl}/`); - await page.getByRole("link", { name: "New Dot" }).first().click(); - await expect(page).toHaveURL(/\/new$/); - await expect(page.getByRole("heading", { name: "Create a Dot" })).toBeVisible(); - // The checks beside the form come from the real control plane, which holds a key in this harness. - await expect(page.getByRole("region", { name: "Before you create" }).getByText("OpenRouter key: Ready")).toBeVisible(); - - await page.getByLabel("Name", { exact: true }).fill("form-made"); - await page.getByLabel("Instructions").fill("Write findings to fares.csv."); - await page.getByLabel("Processors", { exact: true }).fill("4"); - await page.getByLabel("Memory, exact value").fill("8"); - await page.getByRole("radio", { name: /Careful/ }).check(); - await page.getByRole("button", { name: "Create Dot" }).click(); - - await expect(page).toHaveURL(/\/dots\/[^/]+\/chat$/); - await expect(page.getByRole("heading", { level: 1, name: "form-made" })).toBeVisible(); - // The rail knows the new Dot without a reload. - await expect(page.getByRole("navigation", { name: "Dots" }).getByRole("link", { name: /form-made/ })).toBeVisible(); - - const stored = await harness.api.getDot("form-made"); - expect(stored.config).toMatchObject({ - name: "form-made", - instructions: "Write findings to fares.csv.", - model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, - computer: { cpu: 4, memory: "8gb", disk: "40gb", idle_timeout: "15m" }, - limits: { max_cost_per_task_usd: 1 }, - }); - const decisions = Object.fromEntries(PERMISSIONS.map((permission) => [permission, resolvePermission(stored.config, permission)])); - expect(decisions["computer.exec"]).toBe("ask"); - expect(decisions["files.write"]).toBe("ask"); - expect(decisions["files.read"]).toBe("allow"); -}); - -test("a Dot is created from YAML, edited in place", async ({ page, harness }) => { - await page.goto(`${harness.webUrl}/new`); - await page.getByLabel("Name", { exact: true }).fill("yaml-made"); - await page.getByRole("button", { name: "Advanced YAML" }).click(); - const editor = page.getByRole("textbox", { name: "Configuration (YAML)" }); - await expect(editor).toHaveValue(/name: yaml-made/); - // Something only the YAML can say. - await editor.fill((await editor.inputValue()).replace("limits:\n", "limits:\n max_steps_per_task: 12\n")); - await expect(page.getByText("The configuration is valid.")).toBeVisible(); - // The form cannot show it, so going back is refused and the text stays. - await page.getByRole("button", { name: "Form" }).click(); - await expect(page.getByText("The form cannot show this")).toBeVisible(); - await page.getByRole("button", { name: "Create Dot" }).click(); - - await expect(page).toHaveURL(/\/dots\/[^/]+\/chat$/); - const stored = await harness.api.getDot("yaml-made"); - expect(stored.config.limits.max_steps_per_task).toBe(12); -}); - -test("a name that is not valid, or is taken, is said so at once, and nothing is created", async ({ page, harness }) => { - await harness.createDot("name-taken"); - await page.goto(`${harness.webUrl}/new`); - const name = page.getByLabel("Name", { exact: true }); - await name.fill("Not Valid"); - await expect(name).toHaveAttribute("aria-invalid", "true"); - await expect(page.getByText(/lowercase letters, digits and '-'/).first()).toBeVisible(); - await name.fill("name-taken"); - await expect(page.getByText('A Dot named "name-taken" already exists')).toBeVisible(); - await page.getByRole("button", { name: "Create Dot" }).click(); - await expect(page.getByRole("alert").filter({ hasText: "to fix" })).toBeVisible(); - await expect(page).toHaveURL(/\/new$/); - expect((await harness.api.listDots()).filter((dot) => dot.name === "name-taken")).toHaveLength(1); -}); - -test("the checks beside the form say what the host lacks and the command that fixes it, and check again on request", async ({ page, harness }) => { - const missing = { id: "golden-image", label: "golden image", status: "missing", detail: "none in the images folder", fix: "invisible-dots image build" } as const; - const healthy = harness.host.images; - harness.host.images = [missing, healthy[1]!]; - try { - await page.goto(`${harness.webUrl}/new`); - const panel = page.getByRole("region", { name: "Before you create" }); - await expect(panel.getByText("QEMU: Ready")).toBeVisible(); - await expect(panel.getByText("golden image: Needs attention")).toBeVisible(); - await expect(panel.getByText("invisible-dots image build")).toBeVisible(); - // Fixed on the host: the next check shows it. - harness.host.images = healthy; - await panel.getByRole("button", { name: "Check again" }).click(); - await expect(panel.getByText("golden image: Ready")).toBeVisible(); - await expect(panel.getByText("invisible-dots image build")).toHaveCount(0); - } finally { - harness.host.images = healthy; - } -}); - -test("the Home page shows a card per Dot and the computer's power on it works", async ({ page, harness }) => { - const dot = await harness.createDot("home-card"); - await page.goto(`${harness.webUrl}/`); - const card = page.getByRole("article", { name: "home-card" }); - await expect(card.getByText(/goal/i)).toHaveCount(0); - await expect(card.getByText("test/model")).toBeVisible(); - await expect(card.getByText("$0.00")).toBeVisible(); - await expect(card.getByRole("img", { name: "Ready" })).toBeVisible(); - - await card.getByRole("button", { name: /^Computer: Running/ }).click(); - page.once("dialog", (dialog) => void dialog.accept()); - await page.getByRole("menuitem", { name: "Stop" }).click(); - await expect(card.getByRole("button", { name: /^Computer: Stopped/ })).toBeVisible(); - await expect(card.getByRole("img", { name: "Computer stopped" })).toBeVisible(); - - await card.getByRole("link", { name: "Open chat" }).click(); - await expect(page).toHaveURL(new RegExp(`/dots/${dot.id}/chat$`)); -}); - -test("a Dot made elsewhere appears on the Home page while it is open", async ({ page, harness }) => { - await page.goto(`${harness.webUrl}/`); - await expect(page.getByRole("heading", { level: 1, name: "Dots" })).toBeVisible(); - await harness.createDot("home-live"); - await expect(page.getByRole("article", { name: "home-live" })).toBeVisible(); -}); - -test("Home and the create page fit a phone: no sideways scroll at 390 px", async ({ page, harness }) => { - // The longest name a Dot may have. - const name = "phone-card-".padEnd(40, "x"); - await harness.createDot(name); - await page.setViewportSize({ width: 390, height: 844 }); - const loaded: Record Promise> = { - "/": async () => { - const card = page.getByRole("article", { name }); - await expect(card).toBeVisible(); - // The name is cut short inside its card: the shell hides what passes the card's edge, so the page alone does - // not scroll and only the boxes tell. - const [outer, title] = [(await card.boundingBox())!, (await card.getByRole("heading", { name }).boundingBox())!]; - expect(title.x + title.width, "the name passes the card's right edge").toBeLessThanOrEqual(outer.x + outer.width); - }, - "/new": () => expect(page.getByRole("region", { name: "Before you create" }).getByText("OpenRouter key: Ready")).toBeVisible(), - }; - for (const [path, ready] of Object.entries(loaded)) { - await page.goto(`${harness.webUrl}${path}`); - await ready(); - const overflow = await page.evaluate(() => document.documentElement.scrollWidth - window.innerWidth); - expect(overflow, `${path} scrolls sideways by ${overflow}px`).toBeLessThanOrEqual(0); - } -}); diff --git a/apps/web/e2e/dot-settings.spec.ts b/apps/web/e2e/dot-settings.spec.ts deleted file mode 100644 index 4a32a4a7..00000000 --- a/apps/web/e2e/dot-settings.spec.ts +++ /dev/null @@ -1,281 +0,0 @@ -import type { Page } from "@playwright/test"; -import { expect, test } from "./fixtures.js"; -import { lookOf } from "./look.js"; - -const settings = (webUrl: string, dotId: string) => `${webUrl}/dots/${dotId}/settings`; -const row = (page: Page, label: string) => page.getByRole("group", { name: label }); -/** The segmented control is three radios in labels; the label is what a person presses. */ -const choose = (page: Page, label: string, decision: "Allow" | "Ask" | "Deny") => row(page, label).locator("label", { hasText: decision }).click(); - -async function saveReviewed(page: Page) { - await page.getByRole("button", { name: "Review and save" }).click(); - const dialog = page.getByRole("dialog", { name: "Save these changes?" }); - await dialog.getByRole("button", { name: "Save changes" }).click(); - await expect(dialog).toHaveCount(0); -} - -test("allow, ask and deny a permission: the host saves one entry, the Dot's engine has it, and the tools it offers follow", async ({ page, harness }) => { - const dot = await harness.createDot("config-perms"); - const guest = harness.driver.guestOf(dot.id); - await page.goto(settings(harness.webUrl, dot.id)); - - // The tools of a permission, from the Dot's own table, offered while the permission is not denied. - const exec = page.getByRole("list", { name: "Tools of Run commands" }); - await expect(exec.getByRole("listitem")).toHaveText(["exec"]); - - // Deny: saved as an entry, pushed, and the model is no longer offered the tool. - await choose(page, "Run commands", "Deny"); - await expect(row(page, "Run commands").getByRole("radio", { name: "Deny" })).toBeChecked(); - await expect(page.getByText("1 unsaved change.")).toBeVisible(); - expect((await harness.api.getDot(dot.id)).config.permissions).toEqual({}); - await page.getByRole("button", { name: "Review and save" }).click(); - const dialog = page.getByRole("dialog", { name: "Save these changes?" }); - await expect(dialog.getByRole("list", { name: "Changes" }).getByRole("listitem")).toHaveText(["Run commandsfrom allow to deny"]); - // Still nothing written: the review is not the save. - expect((await harness.api.getDot(dot.id)).config.permissions).toEqual({}); - await dialog.getByRole("button", { name: "Save changes" }).click(); - await expect(page.getByText("Saved. The change applies from the Dot's next turn.")).toBeVisible(); - expect((await harness.api.getDot(dot.id)).config.permissions).toEqual({ "computer.exec": "deny" }); - await expect.poll(() => guest.config?.permissions["computer.exec"]).toBe("deny"); - expect((await harness.api.listTools(dot.id)).tools.find((tool) => tool.name === "exec")?.offered).toBe(false); - await expect(exec.getByRole("listitem")).toHaveText(["execnot offered"]); - - // Ask: the tool is offered again, and the Dot will wait for an answer. - await choose(page, "Run commands", "Ask"); - await saveReviewed(page); - expect((await harness.api.getDot(dot.id)).config.permissions).toEqual({ "computer.exec": "ask" }); - await expect.poll(() => guest.config?.permissions["computer.exec"]).toBe("ask"); - await expect(exec.getByRole("listitem")).toHaveText(["exec"]); - - // Allow is what the default says: the entry goes, the engine still allows. - await choose(page, "Run commands", "Allow"); - await saveReviewed(page); - expect((await harness.api.getDot(dot.id)).config.permissions).toEqual({}); - await expect.poll(() => guest.config?.permissions["computer.exec"]).toBe("allow"); - - // A reload shows what is saved. - await page.reload(); - await expect(row(page, "Run commands").getByRole("radio", { name: "Allow" })).toBeChecked(); - await expect(row(page, "Automations").getByRole("radio", { name: "Ask" })).toBeChecked(); -}); - -test("an MCP server is added in the settings, its secret reaches the Dot's engine and never the page, and its state shows", async ({ page, harness }) => { - const dot = await harness.createDot("config-mcp"); - const guest = harness.driver.guestOf(dot.id); - guest.mcpTools.set("search", [{ name: "mcp_search_query", description: "Search." }]); - await page.goto(settings(harness.webUrl, dot.id)); - const panel = page.locator("section", { has: page.getByRole("heading", { level: 2, name: "MCP servers" }) }); - - await panel.getByRole("button", { name: "Add an MCP server" }).click(); - const form = page.getByRole("form", { name: "New MCP server" }); - await form.getByLabel("Name").fill("search"); - await form.getByText("A URL").click(); - await form.getByRole("textbox", { name: "URL" }).fill("https://search.example/mcp"); - await form.getByLabel(/^Secrets/).fill("Authorization"); - await form.getByRole("button", { name: "Add" }).click(); - await saveReviewed(page); - await expect.poll(async () => (await harness.api.getDot(dot.id)).config.mcp_servers).toEqual({ - search: { url: "https://search.example/mcp", headers: {}, secrets: ["Authorization"], timeout_s: 120, startup_timeout_s: 60 }, - }); - - // Saved, the server's secret can be set; the engine gets it, the page keeps nothing of it. - const secret = panel.getByLabel("Authorization"); - await secret.fill("Bearer e2e-token"); - await panel.getByRole("button", { name: "Set" }).click(); - await expect(secret).toHaveValue(""); - await expect.poll(() => guest.mcpSecrets).toEqual({ search: { Authorization: "Bearer e2e-token" } }); - expect(await page.content()).not.toContain("e2e-token"); - - // Where it is, as the engine says, and its permission among the others, asked by default. - await expect(panel.getByText("Connected, 1 tool")).toBeVisible(); - await expect(row(page, "MCP server search").getByRole("radio", { name: "Ask" })).toBeChecked(); - await expect(page.getByRole("list", { name: "Tools of MCP server search" }).getByRole("listitem")).toHaveText(["mcp_search_query"]); -}); - -test("a preset sets the rows together, and the review lists every row that moved", async ({ page, harness }) => { - const dot = await harness.createDot("config-preset"); - await page.goto(settings(harness.webUrl, dot.id)); - await page.getByRole("radio", { name: /Careful/ }).check(); - await expect(row(page, "Run commands").getByRole("radio", { name: "Ask" })).toBeChecked(); - await expect(row(page, "Change files").getByRole("radio", { name: "Ask" })).toBeChecked(); - await page.getByRole("button", { name: "Review and save" }).click(); - await expect(page.getByRole("list", { name: "Changes" }).getByRole("listitem")).toHaveText(["Run commandsfrom allow to ask", "Change filesfrom allow to ask"]); - await page.getByRole("button", { name: "Keep editing" }).click(); - // Keep editing sends nothing, and the edits stay. - expect((await harness.api.getDot(dot.id)).config.permissions).toEqual({}); - await expect(row(page, "Run commands").getByRole("radio", { name: "Ask" })).toBeChecked(); - await page.getByRole("button", { name: "Discard changes" }).click(); - await expect(row(page, "Run commands").getByRole("radio", { name: "Allow" })).toBeChecked(); - await expect(page.getByText("No changes.")).toBeVisible(); -}); - -test("the other settings are saved with the permissions: the summary model, the spending cap and the step limit", async ({ page, harness }) => { - const dot = await harness.createDot("config-rest"); - const guest = harness.driver.guestOf(dot.id); - await page.goto(settings(harness.webUrl, dot.id)); - await page.getByLabel(/^Summary model/).fill("openai/gpt-5-mini"); - await page.getByLabel("Spending cap per task").fill("0.5"); - await page.getByLabel("Steps per task").fill("25"); - await choose(page, "Delete browser identities", "Deny"); - await expect(page.getByText("4 unsaved changes.")).toBeVisible(); - await saveReviewed(page); - - const config = (await harness.api.getDot(dot.id)).config; - expect(config.models).toEqual({ summary: "openai/gpt-5-mini" }); - expect(config.limits).toMatchObject({ max_cost_per_task_usd: 0.5, max_steps_per_task: 25 }); - expect("browser" in config).toBe(false); - // There is no memory to set: the Dot keeps its notes itself. - expect("memory" in config).toBe(false); - await expect(page.getByRole("switch", { name: /^Memory/ })).toHaveCount(0); - expect(config.permissions).toEqual({ "browser.identity.delete": "deny" }); - await expect.poll(() => guest.config?.models.summary).toBe("openai/gpt-5-mini"); - - // Emptying the summary model takes the role out of the config. - await page.getByLabel(/^Summary model/).fill(""); - await saveReviewed(page); - expect((await harness.api.getDot(dot.id)).config.models).toEqual({}); -}); - -test("a setting that is wrong is said where it is, and the disk cannot shrink", async ({ page, harness }) => { - const dot = await harness.createDot("config-rules"); - await page.goto(settings(harness.webUrl, dot.id)); - await page.getByRole("combobox", { name: "Model", exact: true }).fill("two words"); - await expect(page.getByText("model id must not contain whitespace")).toBeVisible(); - await expect(page.getByRole("button", { name: "Review and save" })).toBeDisabled(); - await page.getByRole("combobox", { name: "Model", exact: true }).fill("openai/gpt-5"); - await expect(page.getByRole("button", { name: "Review and save" })).toBeEnabled(); - await page.getByRole("button", { name: "Discard changes" }).click(); - - // The slider starts at what the disk is; the host refuses a smaller disk from the YAML, with its own words. - await expect(page.getByLabel("Disk", { exact: true })).toHaveAttribute("min", "40"); - await page.getByRole("button", { name: "Advanced YAML" }).click(); - const yaml = page.getByRole("textbox", { name: "Configuration (YAML)" }); - await yaml.fill((await yaml.inputValue()).replace(/disk: "?40gb"?/, 'disk: "30gb"')); - await page.getByRole("button", { name: "Review and save" }).click(); - await page.getByRole("dialog").getByRole("button", { name: "Save changes" }).click(); - const refusal = page.getByRole("dialog").getByRole("alert"); - await expect(refusal).toContainText("The configuration was not saved"); - await expect(refusal).toContainText("computer.disk cannot shrink from 40gb to 30gb"); - expect((await harness.api.getDot(dot.id)).config.computer.disk).toBe("40gb"); -}); - -test("the YAML view is the same config: the form's edits are in it, and what it says is what is saved", async ({ page, harness }) => { - const dot = await harness.createDot("config-yaml"); - await page.goto(settings(harness.webUrl, dot.id)); - await choose(page, "Change files", "Ask"); - await page.getByRole("button", { name: "Advanced YAML" }).click(); - const yaml = page.getByRole("textbox", { name: "Configuration (YAML)" }); - await expect(yaml).toHaveValue(/files\.write: ask/); - await yaml.fill((await yaml.inputValue()).replace("max_steps_per_task: 60", "max_steps_per_task: 15")); - await expect(page.getByText("2 unsaved changes.")).toBeVisible(); - await saveReviewed(page); - const config = (await harness.api.getDot(dot.id)).config; - expect(config.limits.max_steps_per_task).toBe(15); - expect(config.permissions).toEqual({ "files.write": "ask" }); - - // A text that is not a config says what is wrong, cannot be saved, and keeps the person in the YAML. - await yaml.fill("name: [unclosed"); - await expect(page.getByRole("list", { name: "Problems in the YAML" })).toBeVisible(); - await expect(page.getByRole("button", { name: "Review and save" })).toBeDisabled(); - await page.getByRole("button", { name: "Form", exact: true }).click(); - await expect(page.getByText("The form cannot show this")).toBeVisible(); -}); - -test("a config changed elsewhere while the page is open is never undone: the edits are kept on top of it, and the save is conditional on what the page read", async ({ page, harness }) => { - const dot = await harness.createDot("config-stale"); - await page.goto(settings(harness.webUrl, dot.id)); - await choose(page, "Run commands", "Deny"); - - // "Always allow" in another tab, which is the API here: the page hears it and keeps the edit on top of it. - const read = await harness.api.getDot(dot.id); - await harness.api.updateDot(dot.id, { ...read.config, permissions: { automations: "allow" } }); - await expect(page.getByText("The configuration changed while you were editing")).toBeVisible(); - await expect(row(page, "Automations").getByRole("radio", { name: "Allow" })).toBeChecked(); - await expect(row(page, "Run commands").getByRole("radio", { name: "Deny" })).toBeChecked(); - await expect(page.getByText("1 unsaved change.")).toBeVisible(); - await saveReviewed(page); - expect((await harness.api.getDot(dot.id)).config.permissions).toEqual({ automations: "allow", "computer.exec": "deny" }); - await expect(page.getByText("The configuration changed while you were editing")).toHaveCount(0); -}); - -test("a stopped computer says the tools need it running, and the permissions can still be set", async ({ page, harness }) => { - const dot = await harness.createDot("config-stopped"); - await harness.api.stopComputer(dot.id); - await page.goto(settings(harness.webUrl, dot.id)); - await expect(page.getByRole("button", { name: /^Computer: Stopped/ })).toBeVisible(); - await expect(page.getByText(/Start the computer to see the tools each permission covers/)).toBeVisible(); - await expect(page.getByRole("list", { name: /^Tools of/ })).toHaveCount(0); - await choose(page, "Run commands", "Ask"); - await saveReviewed(page); - expect((await harness.api.getDot(dot.id)).config.permissions).toEqual({ "computer.exec": "ask" }); - await expect(page.getByText("Saved. The Dot gets the change when its computer starts.")).toBeVisible(); -}); - -test("delete asks for the Dot's name, and the Dot is gone after it", async ({ page, harness }) => { - const dot = await harness.createDot("config-delete"); - await page.goto(settings(harness.webUrl, dot.id)); - await page.getByRole("button", { name: "Delete Dot" }).click(); - const dialog = page.getByRole("dialog", { name: "Delete config-delete?" }); - const confirm = dialog.getByRole("button", { name: "Delete Dot" }); - await expect(confirm).toBeDisabled(); - await dialog.getByLabel(/Type config-delete to confirm/).fill("config-delet"); - await expect(confirm).toBeDisabled(); - await dialog.getByLabel(/Type config-delete to confirm/).fill("config-delete"); - await confirm.click(); - await expect(page).toHaveURL(`${harness.webUrl}/`); - await expect.poll(async () => (await harness.api.listDots()).some((d) => d.id === dot.id)).toBe(false); -}); - -test("the settings page is usable from the keyboard alone, and fits a phone", async ({ page, harness }) => { - const dot = await harness.createDot("config-keys"); - await page.goto(settings(harness.webUrl, dot.id)); - // The arrow keys walk the three decisions of a row. - await row(page, "Read files").getByRole("radio", { name: "Allow" }).focus(); - await page.keyboard.press("ArrowRight"); - await expect(row(page, "Read files").getByRole("radio", { name: "Ask" })).toBeChecked(); - await page.keyboard.press("ArrowRight"); - await expect(row(page, "Read files").getByRole("radio", { name: "Deny" })).toBeChecked(); - await page.getByRole("button", { name: "Review and save" }).focus(); - await page.keyboard.press("Enter"); - const dialog = page.getByRole("dialog", { name: "Save these changes?" }); - await expect(dialog.getByRole("list", { name: "Changes" }).getByRole("listitem")).toHaveText(["Read filesfrom allow to deny"]); - await page.keyboard.press("Escape"); - await expect(dialog).toHaveCount(0); - - await page.setViewportSize({ width: 390, height: 844 }); - await page.goto(settings(harness.webUrl, dot.id)); - await expect(page.getByRole("heading", { name: "Permissions and tools" })).toBeVisible(); - expect(await page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth)).toBe(0); - await choose(page, "Run commands", "Ask"); - // The bar with the save button stays in reach while the long page scrolls. - await expect(page.getByRole("button", { name: "Review and save" })).toBeInViewport(); - expect(await page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth)).toBe(0); -}); - -for (const scheme of ["light", "dark"] as const) { - test(`the settings are readable in ${scheme}: the rows, the risk and change marks, the tools, the save bar`, async ({ page, harness }) => { - await page.emulateMedia({ colorScheme: scheme }); - const dot = await harness.createDot(`config-look-${scheme}`); - await page.goto(settings(harness.webUrl, dot.id)); - await choose(page, "Run commands", "Deny"); - await expect(page.getByRole("list", { name: "Tools of Run commands" })).toBeVisible(); - await page.getByRole("combobox", { name: "Model", exact: true }).fill("two words"); - await expect(page.getByText("model id must not contain whitespace")).toBeVisible(); - - const spots: Record = { - "a permission's description": "li:has(fieldset legend:text-is('Run commands')) > div p.text-sm", - "a decision that is not chosen": "li:has(legend:text-is('Read files')) label:has(input:not(:checked))", - "the decision that is chosen": "li:has(legend:text-is('Run commands')) label:has(input:checked)", - "the high risk mark": "li:has(legend:text-is('Run commands')) span:text-is('High risk')", - "the medium risk mark": "li:has(legend:text-is('Change files')) span:text-is('Medium risk')", - "the low risk mark": "li:has(legend:text-is('Read files')) span:text-is('Low risk')", - "the changed mark": "li:has(legend:text-is('Run commands')) span:text-is('Changed')", - "what a decision means": "li:has(legend:text-is('Run commands')) > p.text-xs", - "a field's problem": "p.text-danger", - "the save bar's text": "[aria-label=Save] p", - }; - for (const [name, selector] of Object.entries(spots)) { - expect((await lookOf(page, selector)).contrast, name).toBeGreaterThanOrEqual(4.5); - } - }); -} diff --git a/apps/web/e2e/fixtures.ts b/apps/web/e2e/fixtures.ts deleted file mode 100644 index 3a42aee5..00000000 --- a/apps/web/e2e/fixtures.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { test as base } from "@playwright/test"; -import { startHarness, type Harness } from "./harness.js"; - -interface WorkerFixtures { - harness: Harness; - /** - * A control plane of its own that has never been given a key and has no Dot: the host a person finds on the first - * run. Started only by a test that asks for it. - */ - unconfigured: Harness; - /** - * A control plane of its own with a key and no Dot, for the pages that count what needs the person across every Dot - * (the Inbox): on the shared one, the approvals, failures and errors of every other spec would be in the count. - */ - fresh: Harness; -} - -export const test = base.extend<{}, WorkerFixtures>({ - // One control plane and one web server per worker. - harness: [ - async ({}, use) => { - const harness = await startHarness(); - try { - await use(harness); - } finally { - await harness.close(); - } - }, - { scope: "worker", timeout: 120_000 }, - ], - unconfigured: [ - async ({}, use) => { - const harness = await startHarness({ withKey: false }); - try { - await use(harness); - } finally { - await harness.close(); - } - }, - { scope: "worker", timeout: 120_000 }, - ], - fresh: [ - async ({}, use) => { - const harness = await startHarness(); - try { - await use(harness); - } finally { - await harness.close(); - } - }, - { scope: "worker", timeout: 120_000 }, - ], -}); - -export { expect } from "@playwright/test"; diff --git a/apps/web/e2e/harness.ts b/apps/web/e2e/harness.ts deleted file mode 100644 index 0649bbc6..00000000 --- a/apps/web/e2e/harness.ts +++ /dev/null @@ -1,155 +0,0 @@ -/** - * The real control plane and the real web server, for the browser tests: `startServer` of apps/api in this process - * on a temporary INVISIBLE_DOTS_HOME (PGlite), with FakeDriver for the VM layer, so a test can drive what a Dot's - * computer does (`driver.guestOf(id)`); and the web client as the product ships and runs it: the built standalone - * server (`.next/standalone/apps/web/server.js` with its copied static files) started by the CLI's own `startWebServer`, - * with the reduced environment `invisible-dots server` gives it (`webEnvironment`), pointed at that control plane. - * The channels run for real too, on what the hub's own tests use: the Telegram adapter talks to a Bot API server of this - * process (`bots`) and the WhatsApp adapter to a connection a test plays (`whatsapp`). Only `npm run build --workspace - * @invisible-dots/web` has to have run before (the standalone build, not only `next build`). - */ -import { mkdtemp, rm } from "node:fs/promises"; -import { createServer } from "node:net"; -import { tmpdir } from "node:os"; -import { dirname, join, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; -import { startServer, type RunningServer } from "@invisible-dots/api"; -import { TelegramChannelType, WhatsAppChannelType } from "@invisible-dots/channels"; -import { FakeBotApi, FakeWhatsAppConnector } from "@invisible-dots/channels/testing"; -import { FakeDriver, waitUntilSettledReady } from "@invisible-dots/scheduler/testing"; -import { InvisibleDotsClient, type DotRecord } from "@invisible-dots/sdk"; -import { ENV, type DoctorCheck } from "@invisible-dots/shared"; -import { locateWebBuild, startWebServer, type WebServer } from "../../cli/src/web.js"; -import { healthyDoctor, ok } from "../../vm-manager/test/doctor-fakes.js"; - -const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../../.."); -const quiet = { debug() {}, info() {}, warn() {}, error() {} }; - -export interface Harness { - /** The web client, as the browser opens it. */ - webUrl: string; - /** INVISIBLE_DOTS_HOME of the control plane: what its health answer calls the data directory. */ - home: string; - /** The control plane behind it. */ - control: RunningServer; - driver: FakeDriver; - /** The Bot API the Telegram adapter polls: a test makes bots, writes to them as a person and reads what they sent. */ - bots: FakeBotApi; - /** The WhatsApp connection of every Dot, played by a test: `whatsapp.current` is the newest. */ - whatsapp: FakeWhatsAppConnector; - /** The control plane's SDK client, signed in with its token. */ - api: InvisibleDotsClient; - /** The API token of this control plane (the web server reads it itself; the tests call the API with it). */ - token: string; - /** - * The host the doctor looks at: a healthy one (QEMU, an accelerator, disk), so a test does not depend on the machine - * it runs on. A test changes what the images row says; the page's next check shows it. - */ - host: { images: DoctorCheck[] }; - /** A Dot whose computer is up and READY. */ - createDot(name: string): Promise; - /** The control plane stops answering (its server closes) while the web client goes on: what a person sees when the API dies. `close` still cleans up. */ - stopApi(): Promise; - close(): Promise; -} - -async function freePort(): Promise { - return new Promise((resolvePort, reject) => { - const probe = createServer(); - probe.once("error", reject); - probe.listen(0, "127.0.0.1", () => { - const address = probe.address(); - probe.close(() => (typeof address === "object" && address ? resolvePort(address.port) : reject(new Error("no port")))); - }); - }); -} - -export interface HarnessOptions { - /** - * Whether the control plane already holds an OpenRouter key (the default), as a host does once it has been set up. - * Without one, no Dot reaches READY, so this is for the pages a person sees before the first Dot. - */ - withKey?: boolean; -} - -export async function startHarness({ withKey = true }: HarnessOptions = {}): Promise { - const home = await mkdtemp(join(tmpdir(), "idots-e2e-")); - const driver = new FakeDriver(); - const bots = await FakeBotApi.start(); - const whatsapp = new FakeWhatsAppConnector(); - const host = { - images: [ok("golden-image", "golden image", "golden-1.qcow2 matches its manifest"), ok("runtime-image", "runtime ISO", "runtime-1.iso matches its manifest")], - }; - const control = await startServer({ - env: { INVISIBLE_DOTS_HOME: home }, - listen: "127.0.0.1:0", - logger: quiet, - driver, - channelTypes: [new TelegramChannelType({ apiRoot: bots.apiRoot, pollSeconds: 1 }), new WhatsAppChannelType({ connector: () => whatsapp, pauseMs: () => 0 })], - doctor: { ...healthyDoctor().deps, images: async () => host.images }, - // Quick polls, so a Dot is READY in milliseconds; the slow background loops are not needed. - scheduler: { - dispatchIntervalMs: 60_000, - idleCheckIntervalMs: 60_000, - lifecycle: { healthPollMs: 20, readyTimeoutMs: 15_000, pumpRetryMs: 20 }, - dispatcher: { retryDelayMs: 0 }, - }, - }); - - let web: WebServer; - let unexpectedExit: string | undefined; - try { - web = await startWebServer({ - build: await locateWebBuild(REPO_ROOT), - listen: { host: "127.0.0.1", port: await freePort() }, - apiUrl: control.url, - // What `invisible-dots server` has in its own environment: the web server gets the allowlisted part of it. - env: { ...process.env, [ENV.HOME]: home, [ENV.TOKEN]: control.token }, - onUnexpectedExit: (message) => (unexpectedExit = message), - }); - } catch (error) { - await control.close(); - await bots.close(); - await rm(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 }); - throw error; - } - const webUrl = web.url; - - let apiStopped = false; - const stopApi = async () => { - if (apiStopped) return; - apiStopped = true; - await control.close(); - }; - const close = async () => { - await web.stop(); - await stopApi(); - await bots.close(); - await rm(home, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 }); - }; - if (unexpectedExit !== undefined) { - await close(); - throw new Error(unexpectedExit); - } - const api = new InvisibleDotsClient({ baseUrl: control.url, token: control.token }); - // A Dot is not READY until the guest holds a key (the READY procedure pushes it); the fake guest accepts any. - if (withKey) await api.setOpenRouterKey("sk-or-e2e-0123456789abcdef"); - return { - webUrl, - home, - control, - driver, - bots, - whatsapp, - host, - api, - token: control.token, - async createDot(name) { - const dot = await api.createDot(`name: ${name}\nmodel:\n provider: openrouter\n id: test/model\ncomputer:\n idle_timeout: 15m\n`); - await waitUntilSettledReady(control.scheduler, driver, dot.id, name); - return dot; - }, - stopApi, - close, - }; -} diff --git a/apps/web/e2e/inbox.spec.ts b/apps/web/e2e/inbox.spec.ts deleted file mode 100644 index 9ccd0841..00000000 --- a/apps/web/e2e/inbox.spec.ts +++ /dev/null @@ -1,328 +0,0 @@ -import type { FakeGuest } from "@invisible-dots/scheduler/testing"; -import { newId, resolvePermission, type ApprovalRequestedData } from "@invisible-dots/shared"; -import type { Page } from "@playwright/test"; -import { expect, test } from "./fixtures.js"; -import { lookOf } from "./look.js"; - -type Ask = Pick & Partial>; - -/** The Dot asks for an approval, as its engine does on an `ask`. Returns the approval's id. */ -function ask(guest: FakeGuest, data: Ask): string { - const approval_id = newId("apr"); - guest.emit("approval.requested", { approval_id, reason: "to get the job done", ...data }); - return approval_id; -} - -const exec = (command: string): Ask => ({ tool: "exec", permission: "computer.exec", arguments: { command } }); -const navigate = (url: string): Ask => ({ tool: "browser_navigate", permission: "browser.navigate", arguments: { identity_id: "shop-abc123", url } }); - -const cards = (page: Page) => page.getByRole("article", { name: /^Wants to / }); -const railInbox = (page: Page) => page.getByRole("complementary", { name: "Navigation" }).getByRole("link", { name: /^Inbox/ }); - -/** What the guest was told about approvals, in order. */ -function answersHeard(guest: FakeGuest) { - return guest.inbound.flatMap((event) => (event.type === "approval.received" ? [event.data] : [])); -} - -// The Inbox counts across every Dot of the host: each test leaves it as it found it, with no approval waiting and no Dot. -test.afterEach(async ({ fresh }) => { - for (const approval of await fresh.api.listApprovals("pending")) await fresh.api.reject(approval.id); - for (const dot of await fresh.api.listDots()) await fresh.api.deleteDot(dot.id); - await expect.poll(async () => (await fresh.api.listDots()).length).toBe(0); -}); - -test("the Inbox lists what every Dot waits on, counts it in the rail and the title, and an answer is heard by the Dot and clears the count", async ({ page, fresh: harness }) => { - const a = await harness.createDot("inbox-a"); - const b = await harness.createDot("inbox-b"); - const guestA = harness.driver.guestOf(a.id); - const idA = ask(guestA, exec("make test")); - ask(harness.driver.guestOf(b.id), navigate("https://example.com/cart")); - - await page.goto(`${harness.webUrl}/inbox`); - await expect(page.getByRole("heading", { level: 1, name: "Inbox" })).toBeVisible(); - await expect(cards(page)).toHaveCount(2); - const first = page.getByRole("article", { name: "Wants to run a command" }); - await expect(first.getByText("inbox-a")).toBeVisible(); - await expect(first.getByText("make test", { exact: true })).toBeVisible(); - await expect(first.getByText("to get the job done")).toBeVisible(); - await expect(first.getByText("Run commands (high risk)")).toBeVisible(); - await expect(page.getByRole("article", { name: "Wants to open a page" }).getByText("inbox-b")).toBeVisible(); - await expect(railInbox(page).getByLabel("2 need you")).toBeVisible(); - await expect(page).toHaveTitle(/^\(2\) /); - - // Nothing below reloads the page: this marker would not survive it. - await page.evaluate(() => ((window as unknown as { marker: number }).marker = 7)); - await first.getByRole("button", { name: "Allow once" }).click(); - await expect(first.getByRole("status")).toHaveText("Allowed"); - await expect(railInbox(page).getByLabel("1 need you")).toBeVisible(); - await expect(page).toHaveTitle(/^\(1\) /); - expect((await harness.api.listApprovals("approved")).map((x) => x.id)).toEqual([idA]); - await expect.poll(() => answersHeard(guestA)).toEqual([{ approval_id: idA, decision: "approve" }]); - // The card is a receipt, still where it was; the other one is untouched. - await expect(cards(page)).toHaveCount(2); - expect(await page.evaluate(() => (window as unknown as { marker: number }).marker)).toBe(7); - - // Another Dot's answer from elsewhere (the CLI, a channel) reaches this page by itself. - const [other] = await harness.api.listApprovals("pending"); - await harness.api.reject(other!.id); - await expect(railInbox(page).getByLabel(/need you/)).toHaveCount(0); - await expect(page).toHaveTitle(/^(?!\()/); -}); - -test("Always allow says what it changes, changes the Dot's settings and its computer's, and is remembered as the answer", async ({ page, fresh: harness }) => { - const dot = await harness.createDot("inbox-always"); - const guest = harness.driver.guestOf(dot.id); - expect(guest.config?.permissions["automations"]).toBe("ask"); - const id = ask(guest, { tool: "cron", permission: "automations", arguments: { action: "add", name: "standup", message: "Say hello", every_seconds: 3600 } }); - - await page.goto(`${harness.webUrl}/inbox`); - const card = page.getByRole("article", { name: "Wants to manage an automation" }); - await expect(card.getByText("every hour")).toBeVisible(); - await card.getByRole("button", { name: "Always allow" }).click(); - const dialog = page.getByRole("dialog", { name: /Always allow .Automations.\?/ }); - await expect(dialog.getByText("Medium risk")).toBeVisible(); - // The tools of this permission, read from the Dot's own computer. - await expect(dialog.getByRole("list", { name: "Tools covered" }).getByRole("listitem")).toHaveText(["cron"]); - await expect(dialog.getByText(/automations: allow/)).toBeVisible(); - expect(resolvePermission((await harness.api.getDot(dot.id)).config, "automations")).toBe("ask"); - - await dialog.getByRole("button", { name: "Always allow" }).click(); - await expect(card.getByRole("status")).toContainText('The Dot will not ask for "Automations" again.'); - expect(resolvePermission((await harness.api.getDot(dot.id)).config, "automations")).toBe("allow"); - await expect.poll(() => guest.config?.permissions["automations"]).toBe("allow"); - // The guest hears the plain decision. - await expect.poll(() => answersHeard(guest)).toEqual([{ approval_id: id, decision: "approve" }]); -}); - -test("Keep asking in that question changes nothing, and Deny sends the note with it", async ({ page, fresh: harness }) => { - const dot = await harness.createDot("inbox-deny"); - const guest = harness.driver.guestOf(dot.id); - const id = ask(guest, exec("make deploy")); - await page.goto(`${harness.webUrl}/inbox`); - const card = page.getByRole("article", { name: "Wants to run a command" }); - const version = (await harness.api.getDot(dot.id)).config_version; - - await card.getByRole("button", { name: "Always allow" }).click(); - await page.getByRole("dialog").getByRole("button", { name: "Keep asking" }).click(); - await expect(page.getByRole("dialog")).toHaveCount(0); - // Nothing was answered and the Dot's settings were not written. - expect(await harness.api.listApprovals("pending")).toHaveLength(1); - expect((await harness.api.getDot(dot.id)).config_version).toBe(version); - - await card.getByRole("button", { name: "Add a note for the Dot" }).click(); - await card.getByLabel("Note for the Dot (optional)").fill("not before the review"); - await card.getByRole("button", { name: "Deny" }).click(); - await expect(card.getByRole("status")).toContainText("Denied"); - await expect(card.getByText("Your note: not before the review")).toBeVisible(); - await expect.poll(() => answersHeard(guest)).toEqual([{ approval_id: id, decision: "reject", note: "not before the review" }]); - - // The History keeps it, with the note, and the filters are in the address. - await page.getByRole("navigation", { name: "Inbox sections" }).getByRole("link", { name: "History" }).click(); - await expect(page).toHaveURL(/\/inbox\?tab=history$/); - const row = page.getByRole("table").getByRole("row", { name: /Asked to run a command/ }); - await expect(row.getByText("Answer: Denied")).toBeVisible(); - await expect(row.getByText("Note: not before the review")).toBeVisible(); - await expect(row.getByRole("link", { name: "inbox-deny" })).toBeVisible(); - - await page.getByLabel("Permission").selectOption("files.write"); - await expect(page).toHaveURL(/permission=files\.write/); - await expect(page.getByText("No approval has been answered under these filters.")).toBeVisible(); - await page.reload(); - await expect(page.getByLabel("Permission")).toHaveValue("files.write"); - await page.getByLabel("Permission").selectOption(""); - await expect(page.getByRole("table")).toBeVisible(); -}); - -test("the old addresses lead to the Inbox, a Dot's approvals to the Inbox filtered to that Dot", async ({ page, fresh: harness }) => { - const kept = await harness.createDot("inbox-kept"); - const other = await harness.createDot("inbox-other"); - ask(harness.driver.guestOf(kept.id), exec("ls")); - ask(harness.driver.guestOf(other.id), navigate("https://example.com")); - - await page.goto(`${harness.webUrl}/approvals`); - await expect(page).toHaveURL(`${harness.webUrl}/inbox`); - await expect(cards(page)).toHaveCount(2); - - await page.goto(`${harness.webUrl}/dots/${kept.id}/approvals`); - await expect(page).toHaveURL(`${harness.webUrl}/inbox?dot=${kept.id}`); - await expect(page.getByLabel("Dot", { exact: true })).toHaveValue(kept.id); - await expect(cards(page)).toHaveCount(1); - await expect(page.getByRole("article", { name: "Wants to run a command" })).toBeVisible(); - - // By name, as the control plane takes it. - await page.goto(`${harness.webUrl}/inbox?dot=inbox-other`); - await expect(page.getByLabel("Dot", { exact: true })).toHaveValue(other.id); - await expect(page.getByRole("article", { name: "Wants to open a page" })).toBeVisible(); - await expect(cards(page)).toHaveCount(1); - - // The Dot's own tab bar no longer has an Approvals tab. - await page.goto(`${harness.webUrl}/dots/${kept.id}/chat`); - await expect(page.getByRole("navigation", { name: "Dot sections" }).getByRole("link", { name: /Approvals/ })).toHaveCount(0); -}); - -test("the Inbox is answered from the keyboard alone: Tab to the list, j and k choose, a allows once, d denies, and a command waits for its confirmation", async ({ page, fresh: harness }) => { - const dot = await harness.createDot("inbox-keys"); - const guest = harness.driver.guestOf(dot.id); - const one = ask(guest, navigate("https://example.com/one")); - const two = ask(guest, navigate("https://example.com/two")); - const three = ask(guest, exec("make test")); - await page.goto(`${harness.webUrl}/inbox`); - await expect(cards(page)).toHaveCount(3); - const selected = page.locator("article[aria-current=true]"); - await expect(selected).toHaveAttribute("data-approval-id", one); - - // A key typed while the focus is elsewhere on the page answers nothing (WCAG 2.1.4): the keys are the list's. - await page.keyboard.press("d"); - await page.keyboard.press("a"); - expect(await harness.api.listApprovals("pending")).toHaveLength(3); - const list = page.getByRole("list", { name: "Approvals waiting for your answer" }); - while (!(await list.evaluate((element) => element === document.activeElement))) await page.keyboard.press("Tab"); - - await page.keyboard.press("a"); - await expect(selected.getByRole("status")).toHaveText("Allowed"); - await page.keyboard.press("j"); - await expect(selected).toHaveAttribute("data-approval-id", two); - await page.keyboard.press("d"); - await expect(selected.getByRole("status")).toContainText("Denied"); - - await page.keyboard.press("j"); - await expect(selected).toHaveAttribute("data-approval-id", three); - await page.keyboard.press("a"); - // A command is not allowed by a key press: the focus is on its Allow once button, and nothing is answered yet. - await expect(selected.getByRole("button", { name: "Allow once" })).toBeFocused(); - expect(await harness.api.listApprovals("pending")).toHaveLength(1); - await page.keyboard.press("Enter"); - await expect(selected.getByRole("status")).toHaveText("Allowed"); - expect((await harness.api.listApprovals("approved")).map((x) => x.id).sort()).toEqual([one, three].sort()); - expect((await harness.api.listApprovals("rejected")).map((x) => x.id)).toEqual([two]); -}); - -test("an approval answered somewhere else while this page was cut off says so, and is not a failure", async ({ page, fresh: harness }) => { - const dot = await harness.createDot("inbox-race"); - const id = ask(harness.driver.guestOf(dot.id), exec("pwd")); - await page.goto(`${harness.webUrl}/inbox`); - const card = page.getByRole("article", { name: "Wants to run a command" }); - await expect(card).toBeVisible(); - // This page stops hearing the stream, as one behind a sleeping laptop does; the CLI answers meanwhile. - await page.route("**/api/stream", (route) => route.abort()); - await page.reload(); - await expect(card).toBeVisible(); - await harness.api.reject(id); - - await card.getByRole("button", { name: "Allow once" }).click(); - await expect(card.getByRole("status")).toHaveText("Already answered, on another tab or another channel"); - await expect(page.getByRole("alert").filter({ hasText: "not recorded" })).toHaveCount(0); - expect((await harness.api.listApprovals("rejected")).map((x) => x.id)).toEqual([id]); -}); - -test("a task's approval is a card on the task, in its drawer too, and answering it lets the task go on", async ({ page, fresh: harness }) => { - const dot = await harness.createDot("inbox-task"); - const guest = harness.driver.guestOf(dot.id); - const onInbound = guest.onInbound; - guest.onInbound = (event, g) => { - if (event.type !== "task.created") return onInbound(event, g); - g.emit("agent.state", { state: "THINKING" }); - g.emit("task.started", { task_id: event.data.task_id }); - ask(g, { ...exec("make build"), task_id: event.data.task_id }); - }; - const task = await harness.api.createTask(dot.id, { description: "Build the site" }); - await page.goto(`${harness.webUrl}/dots/${dot.id}/tasks`); - const taskCard = page.getByRole("article", { name: "Build the site" }); - await expect(taskCard.getByText("Waiting for you").first()).toBeVisible(); - const approval = taskCard.getByRole("article", { name: "Wants to run a command" }); - await expect(approval.getByText("make build", { exact: true })).toBeVisible(); - await expect(approval.getByRole("link", { name: "From a task" })).toHaveAttribute("href", `/dots/${dot.id}/tasks/${task.id}`); - - await approval.getByRole("link", { name: "From a task" }).click(); - const drawer = page.getByRole("dialog", { name: "Build the site" }); - await expect(drawer.getByRole("article", { name: "Wants to run a command" }).getByText("make build", { exact: true })).toBeVisible(); - await page.keyboard.press("Escape"); - - await approval.getByRole("button", { name: "Allow once" }).click(); - await expect(approval.getByRole("status")).toHaveText("Allowed"); - await expect.poll(async () => (await harness.api.getTask(task.id)).status).toBe("COMPLETED"); - // The task finished and left Running; the Dot's own history holds the answer. - await expect(page.getByRole("region", { name: /^History/ }).getByRole("row", { name: /Build the site/ })).toBeVisible(); -}); - -test("a failed task is in the Inbox for a day, opens in its drawer, and leaves for good when dismissed", async ({ page, fresh: harness }) => { - const dot = await harness.createDot("inbox-failed"); - const guest = harness.driver.guestOf(dot.id); - const onInbound = guest.onInbound; - guest.onInbound = (event, g) => { - if (event.type !== "task.created") return onInbound(event, g); - g.emit("task.started", { task_id: event.data.task_id }); - g.emit("task.failed", { task_id: event.data.task_id, error: "the SMTP server refused the connection" }); - }; - const task = await harness.api.createTask(dot.id, { description: "Send the digest" }); - await expect.poll(async () => (await harness.api.getTask(task.id)).status).toBe("FAILED"); - - await page.goto(`${harness.webUrl}/inbox`); - const failed = page.getByRole("article", { name: "Failed: Send the digest" }); - await expect(failed.getByText("the SMTP server refused the connection")).toBeVisible(); - await expect(railInbox(page).getByLabel("1 need you")).toBeVisible(); - await failed.getByRole("link", { name: "Open task" }).click(); - await expect(page).toHaveURL(new RegExp(`/dots/${dot.id}/tasks/${task.id}$`)); - await page.keyboard.press("Escape"); - await railInbox(page).click(); - await failed.getByRole("button", { name: /^Dismiss/ }).click(); - await expect(failed).toHaveCount(0); - await expect(page.getByText("Nothing needs you")).toBeVisible(); - await expect(railInbox(page).getByLabel(/need you/)).toHaveCount(0); - await page.reload(); - await expect(page.getByText("Nothing needs you")).toBeVisible(); -}); - -test("an approval with a long command and a long file fits a phone, and its answers stay in reach", async ({ page, fresh: harness }) => { - const dot = await harness.createDot("inbox-phone"); - const guest = harness.driver.guestOf(dot.id); - const content = Array.from({ length: 120 }, (_, i) => `line ${i} of a file that is long enough to need the sideways scroll of its own, ${"x".repeat(80)}`).join("\n"); - ask(guest, { tool: "write_file", permission: "files.write", arguments: { path: "workspace-report-with-a-very-long-name-for-a-phone.md", content } }); - ask(guest, exec(`echo ${"very-long-argument-".repeat(20)}`)); - await page.setViewportSize({ width: 390, height: 844 }); - await page.goto(`${harness.webUrl}/inbox`); - await expect(cards(page)).toHaveCount(2); - const overflow = () => page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth); - expect(await overflow()).toBeLessThanOrEqual(0); - - // The long card is taller than the screen, and its answers are on the screen anyway. - const long = page.getByRole("article", { name: "Wants to write a file" }); - await long.scrollIntoViewIfNeeded(); - await expect(long.getByRole("button", { name: "Allow once" })).toBeInViewport(); - await expect(long.getByRole("button", { name: "Deny" })).toBeInViewport(); - await long.getByRole("button", { name: /Show the other/ }).click(); - expect(await overflow()).toBeLessThanOrEqual(0); - - await long.getByRole("button", { name: "Always allow" }).click(); - await expect(page.getByRole("dialog")).toBeVisible(); - expect(await overflow()).toBeLessThanOrEqual(0); -}); - -for (const scheme of ["light", "dark"] as const) { - test(`the approval card is readable: answers, diff and receipt keep their contrast (${scheme})`, async ({ page, fresh: harness }) => { - await page.emulateMedia({ colorScheme: scheme }); - const dot = await harness.createDot(`inbox-look-${scheme}`); - const guest = harness.driver.guestOf(dot.id); - ask(guest, { tool: "edit_file", permission: "files.write", arguments: { path: "/etc/motd", old_text: "welcome", new_text: "welcome back" } }); - ask(guest, navigate("https://example.com")); - await page.goto(`${harness.webUrl}/inbox`); - await expect(cards(page)).toHaveCount(2); - - // The first is destructive (a file outside the workspace), the second is not. - const parts: Record = { - "Allow once of a destructive card": "article:has-text('Wants to edit a file') button[data-action=allow]", - "Allow once of an ordinary card": "article:has-text('Wants to open a page') button[data-action=allow]", - "Deny": "button[data-action=deny]", - "Always allow": "button[data-action=always]", - "an added line": "[data-kind=add]", - "a removed line": "[data-kind=remove]", - "the title": "article:has-text('Wants to open a page') [data-slot=approval-card] p", - }; - for (const [name, selector] of Object.entries(parts)) { - expect((await lookOf(page, selector)).contrast, name).toBeGreaterThanOrEqual(4.5); - } - await page.getByRole("article", { name: "Wants to open a page" }).getByRole("button", { name: "Allow once" }).click(); - await expect(page.getByRole("article", { name: "Wants to open a page" }).getByRole("status")).toBeVisible(); - expect((await lookOf(page, "article:has-text('Wants to open a page') [role=status]")).contrast, "the receipt").toBeGreaterThanOrEqual(4.5); - }); -} diff --git a/apps/web/e2e/look.ts b/apps/web/e2e/look.ts deleted file mode 100644 index 769a2ba2..00000000 --- a/apps/web/e2e/look.ts +++ /dev/null @@ -1,47 +0,0 @@ -import type { Page } from "@playwright/test"; - -/** - * What the eye gets from the page, which the role queries cannot see: the colors the browser computed and the - * markers it drew. Read in the page, with the canvas turning any color syntax into the pixels it paints. - */ -export async function lookOf(page: Page, selector: string) { - return page.locator(selector).first().evaluate((element) => { - const context = document.createElement("canvas").getContext("2d", { willReadFrequently: true }); - if (!context) throw new Error("no canvas"); - const rgba = (color: string): [number, number, number, number] => { - context.clearRect(0, 0, 1, 1); - context.fillStyle = "#000"; - context.fillStyle = color; - context.fillRect(0, 0, 1, 1); - const [r = 0, g = 0, b = 0, a = 0] = context.getImageData(0, 0, 1, 1).data; - return [r, g, b, a / 255]; - }; - const luminance = ([r, g, b]: number[]) => { - const [lr = 0, lg = 0, lb = 0] = [r, g, b].map((v) => { - const s = (v ?? 0) / 255; - return s <= 0.03928 ? s / 12.92 : ((s + 0.055) / 1.055) ** 2.4; - }); - return 0.2126 * lr + 0.7152 * lg + 0.0722 * lb; - }; - // The surface under the text: the fills from the nearest opaque one down to the element's own, each painted over - // the one below (a button's `bg-input/30` is a fraction of a color, not a color). - const layers: Array<[number, number, number, number]> = []; - for (let node: Element | null = element; node; node = node.parentElement) { - const painted = rgba(getComputedStyle(node).backgroundColor); - if (painted[3] > 0) layers.push(painted); - if (painted[3] > 0.99) break; - } - let surface: [number, number, number, number] = rgba(getComputedStyle(document.body).backgroundColor); - for (const layer of layers.reverse()) { - const [r, g, b, alpha] = layer; - surface = [0, 1, 2].map((i) => [r, g, b][i]! * alpha + surface[i]! * (1 - alpha)).concat(1) as [number, number, number, number]; - } - const style = getComputedStyle(element); - const [high, low] = [luminance(rgba(style.color)), luminance(surface)].sort((a, b) => b - a); - return { - contrast: ((high ?? 0) + 0.05) / ((low ?? 0) + 0.05), - listStyle: style.listStyleType, - paddingLeft: style.paddingLeft, - }; - }); -} diff --git a/apps/web/e2e/offline.spec.ts b/apps/web/e2e/offline.spec.ts deleted file mode 100644 index b8ac15c7..00000000 --- a/apps/web/e2e/offline.spec.ts +++ /dev/null @@ -1,26 +0,0 @@ -import { expect, test } from "@playwright/test"; -import { startHarness } from "./harness.js"; - -test("the page says, at the top and as an alert, when the control plane stops answering, also on a phone", async ({ page }) => { - test.setTimeout(180_000); - // A control plane of its own: this one is stopped, and the others of the run must keep theirs. - const harness = await startHarness(); - try { - await page.goto(`${harness.webUrl}/`); - await expect(page.getByRole("complementary", { name: "Navigation" }).getByText("Live")).toBeVisible(); - const banner = page.getByRole("alert").filter({ hasText: "The control plane does not answer" }); - await expect(banner).toHaveCount(0); - - await harness.stopApi(); - await expect(banner).toBeVisible({ timeout: 30_000 }); - await expect(banner).toContainText("out of date"); - // It is above the page, not in the rail, so a phone (where the rail is a closed sheet) sees it too. - await page.setViewportSize({ width: 390, height: 844 }); - await expect(banner).toBeVisible(); - const box = await banner.boundingBox(); - expect(box!.y).toBeLessThan(120); - expect(await page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth)).toBeLessThanOrEqual(0); - } finally { - await harness.close(); - } -}); diff --git a/apps/web/e2e/settings.spec.ts b/apps/web/e2e/settings.spec.ts deleted file mode 100644 index 4a67de0f..00000000 --- a/apps/web/e2e/settings.spec.ts +++ /dev/null @@ -1,141 +0,0 @@ -import { join } from "node:path"; -import type { DoctorCheck } from "@invisible-dots/shared"; -import { expect, test } from "./fixtures.js"; - -const MISSING_IMAGE: DoctorCheck = { id: "golden-image", label: "golden image", status: "missing", detail: "none in the images folder", fix: "invisible-dots image build" }; - -test("on the first run Home lists what the host lacks with its commands, takes the key, and turns green when the last thing is done", async ({ page, unconfigured: harness }) => { - const healthy = harness.host.images; - harness.host.images = [MISSING_IMAGE, healthy[1]!]; - try { - await page.goto(`${harness.webUrl}/`); - await expect(page.getByRole("heading", { name: "No Dots yet" })).toBeVisible(); - const checklist = page.getByRole("region", { name: "Get this computer ready" }); - await expect(checklist.getByText(/2 things need attention/)).toBeVisible(); - await expect(checklist.getByText("golden image: Needs attention")).toBeVisible(); - await expect(checklist.getByText("OpenRouter key: Needs attention")).toBeVisible(); - await expect(checklist.getByText("invisible-dots image build")).toBeVisible(); - // The rail says it too, and links to where the key is entered. - await expect(page.getByRole("complementary", { name: "Navigation" }).getByRole("link", { name: "No OpenRouter key stored yet" })).toBeVisible(); - - // The image is built in a terminal; coming back and checking again shows it. - harness.host.images = healthy; - await checklist.getByRole("button", { name: "Check again" }).click(); - await expect(checklist.getByText("golden image: Ready")).toBeVisible(); - await expect(checklist.getByText(/1 thing needs attention/)).toBeVisible(); - - // The key: write-only, stored, and the page agrees without a reload. - const field = checklist.getByLabel("OpenRouter API key"); - await expect(field).toHaveAttribute("type", "password"); - await field.fill("sk-or-first-run-0123456789"); - await checklist.getByRole("button", { name: "Save key" }).click(); - // Not scoped to `checklist`: the region is named for its state and is "This computer is ready" a moment later. - await expect(page.getByText("Saved. No Dot's computer is running, so each gets the key when it starts.")).toBeVisible(); - await expect(page.getByRole("region", { name: "This computer is ready" })).toBeVisible(); - await expect(page.getByRole("complementary", { name: "Navigation" }).getByText("No OpenRouter key stored yet")).toHaveCount(0); - expect(await harness.api.health()).toMatchObject({ openrouter_configured: true }); - expect(await page.content()).not.toContain("sk-or-first-run-0123456789"); - - await page.getByRole("link", { name: "Create your first Dot" }).click(); - await expect(page).toHaveURL(/\/new$/); - await expect(page.getByRole("region", { name: "Before you create" }).getByText("OpenRouter key: Ready")).toBeVisible(); - } finally { - harness.host.images = healthy; - } -}); - -test("the Settings page is in the rail and shows the checks, the key, the appearance, the session and where the data is", async ({ page, harness }) => { - await page.goto(`${harness.webUrl}/`); - await page.getByRole("complementary", { name: "Navigation" }).getByRole("link", { name: "Settings" }).click(); - await expect(page).toHaveURL(/\/settings$/); - await expect(page).toHaveTitle(/^Settings/); - await expect(page.getByRole("complementary", { name: "Navigation" }).getByRole("link", { name: "Settings" })).toHaveAttribute("aria-current", "page"); - await expect(page.getByRole("heading", { level: 1, name: "Settings" })).toBeVisible(); - - const checks = page.getByRole("region", { name: "Host checks" }); - await expect(checks.getByText("Control plane: Ready")).toBeVisible(); - await expect(checks.getByText("QEMU: Ready")).toBeVisible(); - await expect(checks.getByText("Everything a Dot needs is in place.")).toBeVisible(); - await expect(page.getByRole("region", { name: "OpenRouter key" }).getByText("A key is stored.")).toBeVisible(); - - const about = page.getByRole("region", { name: "About" }); - await expect(about.getByText("embedded PostgreSQL", { exact: false })).toBeVisible(); - await expect(about.getByText(harness.home, { exact: true })).toBeVisible(); - await expect(about.getByText(join(harness.home, "logs"), { exact: true })).toBeVisible(); - await expect(about.getByText("invisible-dots logs ")).toBeVisible(); -}); - -test("a missing host check shows its command here too, and checking again follows the host", async ({ page, harness }) => { - const healthy = harness.host.images; - harness.host.images = [MISSING_IMAGE, healthy[1]!]; - try { - await page.goto(`${harness.webUrl}/settings`); - const checks = page.getByRole("region", { name: "Host checks" }); - await expect(checks.getByText("golden image: Needs attention")).toBeVisible(); - await expect(checks.getByText("invisible-dots image build")).toBeVisible(); - await expect(checks.getByText(/1 thing needs attention/)).toBeVisible(); - await expect(checks.getByRole("button", { name: "Copy the fix for golden image" })).toBeVisible(); - harness.host.images = healthy; - await checks.getByRole("button", { name: "Check again" }).click(); - await expect(checks.getByText("golden image: Ready")).toBeVisible(); - await expect(checks.getByText("invisible-dots image build")).toHaveCount(0); - } finally { - harness.host.images = healthy; - } -}); - -test("a new key is pushed to the running Dots, and a key that cannot be one is refused before it is sent", async ({ page, harness }) => { - const dot = await harness.createDot("settings-key"); - await page.goto(`${harness.webUrl}/settings`); - const key = page.getByRole("region", { name: "OpenRouter key" }); - const field = key.getByLabel("Replace the key"); - - await field.fill("has a space"); - await key.getByRole("button", { name: "Replace key" }).click(); - await expect(key.getByText(/must be printable ASCII without spaces/)).toBeVisible(); - expect(harness.driver.guestOf(dot.id).openrouterKey).toBe("sk-or-e2e-0123456789abcdef"); - - await field.fill("sk-or-replacement-0123456789"); - await key.getByRole("button", { name: "Replace key" }).click(); - await expect(key.getByText(/^Saved\. Pushed to \d+ running Dots?\.$/)).toBeVisible(); - await expect(field).toHaveValue(""); - expect(harness.driver.guestOf(dot.id).openrouterKey).toBe("sk-or-replacement-0123456789"); - expect(await page.content()).not.toContain("sk-or-replacement-0123456789"); -}); - -test("the theme is chosen on the page, applied at once and kept across a reload", async ({ page, harness }) => { - await page.emulateMedia({ colorScheme: "light" }); - await page.goto(`${harness.webUrl}/settings`); - const appearance = page.getByRole("region", { name: "Appearance" }); - await expect(appearance.getByRole("radio", { name: /System/ })).toBeChecked(); - await appearance.getByRole("radio", { name: /Dark/ }).check(); - await expect(page.locator("html")).toHaveAttribute("data-theme", "dark"); - await page.reload(); - await expect(page.locator("html")).toHaveAttribute("data-theme", "dark"); - await expect(page.getByRole("region", { name: "Appearance" }).getByRole("radio", { name: /Dark/ })).toBeChecked(); - await page.getByRole("region", { name: "Appearance" }).getByRole("radio", { name: /System/ }).check(); - await expect(page.locator("html")).toHaveAttribute("data-theme", "light"); -}); - -test("Settings and Home fit a phone", async ({ page, harness }) => { - await page.setViewportSize({ width: 390, height: 844 }); - for (const path of ["/settings", "/"]) { - await page.goto(`${harness.webUrl}${path}`); - await expect(page.getByRole("heading", { level: 1 })).toBeVisible(); - expect(await page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth)).toBe(0); - } -}); - -test("the settings page is usable from the keyboard alone", async ({ page, harness }) => { - await page.goto(`${harness.webUrl}/settings`); - const key = page.getByRole("region", { name: "OpenRouter key" }); - await key.getByLabel("Replace the key").focus(); - await page.keyboard.type("sk-or-by-keyboard-0123456789"); - await page.keyboard.press("Enter"); - await expect(key.getByText(/^Saved\./)).toBeVisible(); - const appearance = page.getByRole("region", { name: "Appearance" }); - await appearance.getByRole("radio", { name: /Light/ }).focus(); - await page.keyboard.press("ArrowRight"); - await expect(appearance.getByRole("radio", { name: /Dark/ })).toBeChecked(); - await expect(page.locator("html")).toHaveAttribute("data-theme", "dark"); -}); diff --git a/apps/web/e2e/shell.spec.ts b/apps/web/e2e/shell.spec.ts deleted file mode 100644 index f013c774..00000000 --- a/apps/web/e2e/shell.spec.ts +++ /dev/null @@ -1,158 +0,0 @@ -import { expect, test } from "./fixtures.js"; - -test("the rail lists the Dots and a Dot's page shows its header and its tabs", async ({ page, harness }) => { - const dot = await harness.createDot("shell-header"); - await page.goto(`${harness.webUrl}/`); - const dots = page.getByRole("navigation", { name: "Dots" }); - await dots.getByRole("link", { name: /shell-header/ }).click(); - - await expect(page).toHaveURL(new RegExp(`/dots/${dot.id}/chat$`)); - await expect(page.getByRole("heading", { level: 1, name: "shell-header" })).toBeVisible(); - await expect(page.getByRole("button", { name: /Show the whole goal/ })).toHaveCount(0); - await expect(dots.getByRole("link", { name: /shell-header/ })).toHaveAttribute("aria-current", "page"); - await expect(page.getByRole("img", { name: "Ready" }).first()).toBeVisible(); - await expect(page.getByText("$0.00")).toBeVisible(); - - const tabs = page.getByRole("navigation", { name: "Dot sections" }); - await expect(tabs.getByRole("link", { name: "Chat" })).toHaveAttribute("aria-current", "page"); - await tabs.getByRole("link", { name: "Tasks" }).click(); - await expect(page).toHaveURL(new RegExp(`/dots/${dot.id}/tasks$`)); - await expect(tabs.getByRole("link", { name: "Tasks" })).toHaveAttribute("aria-current", "page"); - await expect(page.getByRole("button", { name: "New task" })).toBeVisible(); -}); - -test("a Dot that waits for an approval shows it everywhere at once, live, and clears it when it is answered", async ({ page, harness }) => { - const dot = await harness.createDot("shell-asks"); - await page.goto(`${harness.webUrl}/`); - const link = page.getByRole("navigation", { name: "Dots" }).getByRole("link", { name: /shell-asks/ }); - await expect(link.getByRole("img", { name: "Ready" })).toBeVisible(); - const titleBefore = await page.title(); - - harness.driver.guestOf(dot.id).requestApproval(undefined); - - await expect(link.getByRole("img", { name: "Waiting for you" })).toBeVisible(); - await expect(link.getByLabel("1 waiting")).toBeVisible(); - await expect(page.getByRole("link", { name: /^Inbox/ }).getByLabel("1 need you")).toBeVisible(); - await expect(page).toHaveTitle(`(1) ${titleBefore}`); - - const [approval] = await harness.api.listApprovals("pending"); - await harness.api.approve(approval!.id); - - await expect(link.getByLabel("1 waiting")).toHaveCount(0); - await expect(page).toHaveTitle(titleBefore); -}); - -test("the computer pill stops and starts the computer", async ({ page, harness }) => { - const dot = await harness.createDot("shell-power"); - await page.goto(`${harness.webUrl}/dots/${dot.id}/chat`); - const pill = page.getByRole("button", { name: /^Computer: Running/ }); - await pill.click(); - page.once("dialog", (dialog) => void dialog.accept()); - await page.getByRole("menuitem", { name: "Stop" }).click(); - await expect(page.getByRole("button", { name: /^Computer: Stopped/ })).toBeVisible(); - await expect(page.getByRole("navigation", { name: "Dots" }).getByRole("link", { name: /shell-power/ }).getByRole("img", { name: "Computer stopped" })).toBeVisible(); - - await page.getByRole("button", { name: /^Computer: Stopped/ }).click(); - await page.getByRole("menuitem", { name: "Start" }).click(); - await expect(page.getByRole("button", { name: /^Computer: Running/ })).toBeVisible(); -}); - -test("an address with nothing behind it says so in the app's look, with the way back to Home", async ({ page, harness }) => { - const response = await page.goto(`${harness.webUrl}/no-such-page-here`); - expect(response?.status()).toBe(404); - await expect(page.getByRole("heading", { name: "There is nothing at this address" })).toBeVisible(); - // The app's own font, not the framework's default page. - expect(await page.evaluate(() => getComputedStyle(document.querySelector("h1")!).fontFamily)).toMatch(/Geist/); - await page.getByRole("link", { name: "Go to Home" }).click(); - await expect(page).toHaveURL(`${harness.webUrl}/`); -}); - -test("the theme follows the system, can be chosen, and is applied before the page is drawn", async ({ page, harness }) => { - await page.emulateMedia({ colorScheme: "dark" }); - await page.goto(`${harness.webUrl}/`); - await expect(page.locator("html")).toHaveAttribute("data-theme", "dark"); - const darkBackground = await page.evaluate(() => getComputedStyle(document.body).backgroundColor); - - await page.getByRole("button", { name: "Theme: System" }).click(); - await page.getByRole("menuitemradio", { name: "Light" }).click(); - await expect(page.locator("html")).toHaveAttribute("data-theme", "light"); - expect(await page.evaluate(() => getComputedStyle(document.body).backgroundColor)).not.toBe(darkBackground); - - // The choice survives a reload, against a system that says dark: the inline script has set it by the time the - // document is parsed, before any module has run. - await page.goto(`${harness.webUrl}/`, { waitUntil: "commit" }); - await page.waitForFunction(() => document.documentElement.dataset.theme === "light"); - await expect(page.getByRole("button", { name: "Theme: Light" })).toBeVisible(); - - await page.getByRole("button", { name: "Theme: Light" }).click(); - await page.getByRole("menuitemradio", { name: "System" }).click(); - await expect(page.locator("html")).toHaveAttribute("data-theme", "dark"); -}); - -test("on a phone the rail is a sheet behind the menu button and nothing scrolls sideways", async ({ page, harness }) => { - const dot = await harness.createDot("shell-phone"); - await page.setViewportSize({ width: 390, height: 800 }); - await page.goto(`${harness.webUrl}/dots/${dot.id}/chat`); - - await expect(page.getByRole("complementary", { name: "Navigation" })).toBeHidden(); - await expect(page.getByRole("heading", { level: 1, name: "shell-phone" })).toBeVisible(); - const overflow = await page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth); - expect(overflow).toBe(0); - - await page.getByRole("button", { name: "Open the menu" }).click(); - const sheet = page.getByRole("dialog"); - await expect(sheet.getByRole("link", { name: /shell-phone/ })).toBeVisible(); - await sheet.getByRole("link", { name: "Home", exact: true }).click(); - await expect(page).toHaveURL(`${harness.webUrl}/`); - await expect(sheet).toBeHidden(); -}); - -test("the whole shell works from the keyboard", async ({ page, harness }) => { - const dot = await harness.createDot("shell-keys"); - await page.goto(`${harness.webUrl}/dots/${dot.id}/chat`); - await page.getByRole("button", { name: /^Computer: Running/ }).focus(); - await page.keyboard.press("Enter"); - await expect(page.getByRole("menu")).toBeVisible(); - await page.keyboard.press("Escape"); - await expect(page.getByRole("menu")).toBeHidden(); - // The skip link is the first stop of a page that does not take the focus itself (the chat does, for its box). - await page.goto(`${harness.webUrl}/inbox`); - await expect(page.getByRole("heading", { name: "Inbox" })).toBeVisible(); - await page.keyboard.press("Tab"); - await expect(page.getByRole("link", { name: "Skip to content" })).toBeFocused(); - await page.keyboard.press("Enter"); - await expect(page).toHaveURL(/#main$/); -}); - -test("a focused control keeps an outline under forced colors, where box shadows are not drawn, and draws its ring as a shadow otherwise", async ({ page, harness }) => { - await page.goto(`${harness.webUrl}/new`); - await expect(page.getByRole("heading", { name: "Create a Dot" })).toBeVisible(); - const focused = async () => { - // The first button or field: the rail lists every Dot of the host before it, and a link is drawn with the browser's own focus outline. - for (let i = 0; i < 300; i++) { - await page.keyboard.press("Tab"); - const tag = await page.evaluate(() => document.activeElement?.tagName ?? ""); - if (["BUTTON", "INPUT", "TEXTAREA"].includes(tag)) break; - } - return page.evaluate(() => { - const style = getComputedStyle(document.activeElement!); - return { tag: document.activeElement!.tagName, outlineStyle: style.outlineStyle, outlineWidth: style.outlineWidth, outlineColor: style.outlineColor, boxShadow: style.boxShadow }; - }); - }; - - // Ordinary colors: the ring is a shadow, and there is no outline besides it. - const ordinary = await focused(); - expect(ordinary.boxShadow, JSON.stringify(ordinary)).not.toBe("none"); - expect(ordinary.outlineStyle, JSON.stringify(ordinary)).toBe("none"); - - // Forced colors: no shadow is drawn, so the indicator is the outline, in a system color. - await page.emulateMedia({ forcedColors: "active" }); - await page.reload(); - await expect(page.getByRole("heading", { name: "Create a Dot" })).toBeVisible(); - const forced = await focused(); - expect(forced.outlineStyle).toBe("solid"); - // The button's transition-all eases its outline in from the default width (medium, 3px) over 150 ms: it is read - // once settled, or a loaded machine reads it on the way. - await expect.poll(() => page.evaluate(() => getComputedStyle(document.activeElement!).outlineWidth)).toBe("2px"); - expect(forced.outlineColor).not.toMatch(/rgba\(0, 0, 0, 0\)|transparent/); -}); diff --git a/apps/web/e2e/skills.spec.ts b/apps/web/e2e/skills.spec.ts deleted file mode 100644 index d7180e56..00000000 --- a/apps/web/e2e/skills.spec.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { expect, test } from "./fixtures.js"; - -test("the Skills tab lists the built-in skills and the Dot's own, opens one as text, and follows one the Dot writes", async ({ page, harness }) => { - const dot = await harness.createDot("skills-tab"); - const guest = harness.driver.guestOf(dot.id); - await page.goto(`${harness.webUrl}/dots/${dot.id}/chat`); - await page.getByRole("navigation", { name: "Dot sections" }).getByRole("link", { name: "Skills" }).click(); - await expect(page).toHaveURL(new RegExp(`/dots/${dot.id}/skills$`)); - - const list = page.getByRole("list", { name: "The Dot's skills" }); - await expect(list.getByRole("link")).toHaveCount(1); - await expect(list.getByRole("link").first()).toContainText("invisible-playwright"); - await expect(list.getByRole("link").first()).toContainText("Built in"); - await list.getByRole("link").first().click(); - await expect(page).toHaveURL(new RegExp(`/dots/${dot.id}/skills\\?skill=invisible-playwright$`)); - await expect(page.getByRole("region", { name: "The skill invisible-playwright" }).getByRole("heading", { name: "The browser" })).toBeVisible(); - - // The Dot writes a skill of its own in a turn: the list has it once the turn ends. - guest.skills = [ - ...guest.skills, - { name: "shop-login", description: "Log in to the shop.", source: "dot", path: "/home/dot/skills/shop-login/SKILL.md", content: "---\nname: shop-login\ndescription: Log in to the shop.\n---\n\nClick Sign in.\n" }, - ]; - guest.emit("message.assistant", { text: "I wrote down how to log in." }); - await expect(list.getByRole("link").filter({ hasText: "shop-login" })).toContainText("Written by the Dot"); -}); - -test("a stopped computer says so on the Skills tab, and starts from there", async ({ page, harness }) => { - const dot = await harness.createDot("skills-stopped"); - await harness.api.stopComputer(dot.id); - await page.goto(`${harness.webUrl}/dots/${dot.id}/skills`); - await expect(page.getByRole("status").filter({ hasText: "Start the computer to see its skills" })).toBeVisible(); - await page.getByRole("button", { name: "Start the computer" }).click(); - await expect(page.getByRole("list", { name: "The Dot's skills" })).toBeVisible(); -}); diff --git a/apps/web/e2e/tasks.spec.ts b/apps/web/e2e/tasks.spec.ts deleted file mode 100644 index 0e63ba56..00000000 --- a/apps/web/e2e/tasks.spec.ts +++ /dev/null @@ -1,228 +0,0 @@ -import type { FakeGuest } from "@invisible-dots/scheduler/testing"; -import { TASK_LIST_LIMIT } from "@invisible-dots/shared"; -import { expect, test } from "./fixtures.js"; -import type { Harness } from "./harness.js"; -import { lookOf } from "./look.js"; - -/** A guest that takes a task up and keeps it: the test says what it reports and when it ends. */ -function holdTasks(guest: FakeGuest): void { - guest.onInbound = (event, g) => { - if (event.type !== "task.created") return; - g.emit("agent.state", { state: "THINKING" }); - g.emit("task.started", { task_id: event.data.task_id }); - }; -} - -async function taskNamed(harness: Harness, dotId: string, description: string) { - const task = (await harness.api.listTasks(dotId)).find((t) => t.description === description); - if (!task) throw new Error(`no task "${description}"`); - return task; -} - -test("a task is made in the dialog, its progress line follows the Dot live, and it ends in the history with its result", async ({ page, harness }) => { - const dot = await harness.createDot("tasks-live"); - const guest = harness.driver.guestOf(dot.id); - holdTasks(guest); - await page.goto(`${harness.webUrl}/dots/${dot.id}/tasks`); - await expect(page.getByText("No tasks yet")).toBeVisible(); - - await page.getByRole("button", { name: "New task" }).click(); - const dialog = page.getByRole("dialog", { name: "New task" }); - await dialog.getByLabel("What should it do?").fill("Find the cheapest fare to Lisbon"); - await dialog.getByRole("radio", { name: "High" }).check(); - await dialog.getByRole("button", { name: "Create task" }).click(); - - const card = page.getByRole("article", { name: "Find the cheapest fare to Lisbon" }); - await expect(card).toBeVisible(); - await expect(card.getByText(/Status: Running/)).toBeVisible(); - await expect(card.getByText(/Priority: High/)).toBeVisible(); - const task = await taskNamed(harness, dot.id, "Find the cheapest fare to Lisbon"); - expect(task.priority).toBe(10); - await expect(card.getByText(/has not reported anything yet/)).toBeVisible(); - - // Nothing below reloads the page: this marker would not survive it. - await page.evaluate(() => ((window as unknown as { marker: number }).marker = 7)); - - guest.emit("task.progress", { task_id: task.id, text: "Reading the first airline's fares", spent_usd: 0.05 }); - await expect(card.getByText("Reading the first airline's fares")).toBeVisible(); - await expect(card.getByText("$0.05")).toBeVisible(); - // The Dot's header says it too, wherever the person is in it. - await expect(page.getByRole("link", { name: /Reading the first airline's fares/ }).first()).toBeVisible(); - - guest.emit("task.progress", { task_id: task.id, text: "Comparing three airlines", spent_usd: 0.11 }); - await expect(card.getByText("Comparing three airlines")).toBeVisible(); - await expect(card.getByText("Reading the first airline's fares")).toHaveCount(0); - await expect(card.getByText("$0.11")).toBeVisible(); - - guest.emit("task.completed", { task_id: task.id, summary: "The cheapest fare is **EUR 41** on Tuesday.", spent_usd: 0.14 }); - guest.emit("agent.state", { state: "IDLE" }); - const history = page.getByRole("region", { name: /^History/ }); - const row = history.getByRole("row", { name: /Find the cheapest fare/ }); - await expect(row.getByText(/Status: Completed/)).toBeVisible(); - await expect(row.getByText("$0.14")).toBeVisible(); - await expect(row.getByText(/The cheapest fare is/)).toBeVisible(); - await expect(card).toHaveCount(0); - await expect(page.getByText("Nothing is running.")).toBeVisible(); - expect(await page.evaluate(() => (window as unknown as { marker: number }).marker)).toBe(7); - - // The drawer has its own address, and tells the story of the task. - await row.getByRole("link", { name: "Find the cheapest fare to Lisbon" }).click(); - await expect(page).toHaveURL(new RegExp(`/dots/${dot.id}/tasks/${task.id}$`)); - const drawer = page.getByRole("dialog", { name: "Find the cheapest fare to Lisbon" }); - await expect(drawer.getByRole("region", { name: "Result" }).locator("strong")).toHaveText("EUR 41"); - await expect(drawer.getByText("Reading the first airline's fares")).toBeVisible(); - await expect(drawer.getByText("Comparing three airlines")).toBeVisible(); - await expect(drawer.getByText("Task created")).toBeVisible(); - await expect(drawer.getByText("$0.14")).toBeVisible(); - - // Opened by its address on a fresh load, it is the same. - await page.reload(); - await expect(page.getByRole("dialog", { name: "Find the cheapest fare to Lisbon" }).getByText("Comparing three airlines")).toBeVisible(); - await page.keyboard.press("Escape"); - await expect(page).toHaveURL(new RegExp(`/dots/${dot.id}/tasks$`)); - await expect(page.getByRole("dialog")).toHaveCount(0); -}); - -test("a running task is cancelled after a question, and the history keeps it as cancelled", async ({ page, harness }) => { - const dot = await harness.createDot("tasks-cancel"); - holdTasks(harness.driver.guestOf(dot.id)); - const created = await harness.api.createTask(dot.id, { description: "Sort the whole archive" }); - await page.goto(`${harness.webUrl}/dots/${dot.id}/tasks`); - const card = page.getByRole("article", { name: "Sort the whole archive" }); - await expect(card.getByText(/Status: Running/)).toBeVisible(); - - await card.getByRole("button", { name: /^Cancel/ }).click(); - const dialog = page.getByRole("dialog", { name: "Cancel this task?" }); - await dialog.getByRole("button", { name: "Keep it" }).click(); - await expect(dialog).toHaveCount(0); - expect((await harness.api.getTask(created.id)).status).toBe("RUNNING"); - - await card.getByRole("button", { name: /^Cancel/ }).click(); - await page.getByRole("dialog", { name: "Cancel this task?" }).getByRole("button", { name: "Cancel task" }).click(); - const row = page.getByRole("region", { name: /^History/ }).getByRole("row", { name: /Sort the whole archive/ }); - await expect(row.getByText(/Status: Cancelled/)).toBeVisible(); - await expect(row.getByText("cancelled by the user")).toBeVisible(); - await expect(card).toHaveCount(0); - expect((await harness.api.getTask(created.id)).status).toBe("CANCELLED"); -}); - -test("a task that fails shows the reason as the Dot gave it, in the history and in the drawer", async ({ page, harness }) => { - const dot = await harness.createDot("tasks-fail"); - const guest = harness.driver.guestOf(dot.id); - holdTasks(guest); - const created = await harness.api.createTask(dot.id, { description: "Rebuild the index" }); - await page.goto(`${harness.webUrl}/dots/${dot.id}/tasks`); - await expect(page.getByRole("article", { name: "Rebuild the index" })).toBeVisible(); - - guest.emit("task.failed", { task_id: created.id, error: "the per-task cost cap of $1.00 was reached", spent_usd: 1.02 }); - guest.emit("agent.state", { state: "IDLE" }); - const row = page.getByRole("region", { name: /^History/ }).getByRole("row", { name: /Rebuild the index/ }); - await expect(row.getByText(/Status: Failed/)).toBeVisible(); - await expect(row.getByText("the per-task cost cap of $1.00 was reached")).toBeVisible(); - await expect(row.getByText("$1.02")).toBeVisible(); - - await page.getByRole("region", { name: /^History/ }).getByRole("button", { name: "Completed" }).click(); - await expect(page.getByText("No completed tasks.")).toBeVisible(); - await page.getByRole("button", { name: "Failed" }).click(); - await row.getByRole("link", { name: "Rebuild the index" }).click(); - await expect(page.getByRole("dialog").getByRole("alert").filter({ hasText: "cost cap of $1.00" })).toBeVisible(); -}); - -test("tasks that wait are in the queue in the order the Dot takes them, and one with a time is scheduled", async ({ page, harness }) => { - const dot = await harness.createDot("tasks-queue"); - holdTasks(harness.driver.guestOf(dot.id)); - await harness.api.createTask(dot.id, { description: "Occupy the Dot" }); - await page.goto(`${harness.webUrl}/dots/${dot.id}/tasks`); - await expect(page.getByRole("article", { name: "Occupy the Dot" })).toBeVisible(); - - await harness.api.createTask(dot.id, { description: "Tidy up", priority: -10 }); - await harness.api.createTask(dot.id, { description: "Answer the customer", priority: 100 }); - await harness.api.createTask(dot.id, { description: "Weekly report" }); - await harness.api.createTask(dot.id, { description: "Next week's plan", scheduled_at: new Date(Date.now() + 26 * 3_600_000).toISOString() }); - - const queue = page.getByRole("region", { name: /^Queue/ }); - await expect(queue.getByRole("article")).toHaveCount(3); - expect(await queue.getByRole("article").evaluateAll((items) => items.map((i) => i.getAttribute("aria-label")))).toEqual(["Answer the customer", "Weekly report", "Tidy up"]); - await expect(queue.getByRole("article", { name: "Answer the customer" }).getByText("Next", { exact: true })).toBeVisible(); - await expect(queue.getByRole("article", { name: "Answer the customer" }).getByText(/Priority: Urgent/)).toBeVisible(); - const scheduled = page.getByRole("region", { name: /^Scheduled/ }); - await expect(scheduled.getByRole("article", { name: "Next week's plan" }).getByText(/in 1d/)).toBeVisible(); -}); - -test("the Tasks page and its drawer fit a phone: no sideways scroll at 390 px", async ({ page, harness }) => { - const dot = await harness.createDot("tasks-phone"); - const guest = harness.driver.guestOf(dot.id); - holdTasks(guest); - const long = "Collect every price from the long list of airlines and write them into one very long spreadsheet with a column for each day"; - const created = await harness.api.createTask(dot.id, { description: long }); - await harness.api.createTask(dot.id, { description: "Another one waiting in line behind the first, with words enough to wrap" }); - await page.setViewportSize({ width: 390, height: 844 }); - await page.goto(`${harness.webUrl}/dots/${dot.id}/tasks`); - await expect(page.getByRole("article", { name: long })).toBeVisible(); - guest.emit("task.progress", { task_id: created.id, text: "An unbroken line of progress text that is long enough to wrap on a narrow screen several times over", spent_usd: 0.01 }); - await expect(page.getByText(/An unbroken line of progress/).first()).toBeVisible(); - const overflow = () => page.evaluate(() => document.documentElement.scrollWidth - document.documentElement.clientWidth); - expect(await overflow()).toBeLessThanOrEqual(0); - - await page.getByRole("link", { name: long }).first().click(); - await expect(page.getByRole("dialog", { name: long })).toBeVisible(); - expect(await overflow()).toBeLessThanOrEqual(0); -}); - -for (const scheme of ["light", "dark"] as const) { - test(`the Tasks page is drawn by its own styles, not the old stylesheet's: readable buttons, no list markers (${scheme})`, async ({ page, harness }) => { - await page.emulateMedia({ colorScheme: scheme }); - const dot = await harness.createDot(`tasks-look-${scheme}`); - const guest = harness.driver.guestOf(dot.id); - holdTasks(guest); - await harness.api.createTask(dot.id, { description: "Occupy the Dot" }); - await harness.api.createTask(dot.id, { description: "Waits in the queue" }); - await harness.api.createTask(dot.id, { description: "Waits for its time", scheduled_at: new Date(Date.now() + 26 * 3_600_000).toISOString() }); - const early = await harness.api.createTask(dot.id, { description: "Dropped early", scheduled_at: new Date(Date.now() + 27 * 3_600_000).toISOString() }); - await harness.api.cancelTask(early.id); - await page.goto(`${harness.webUrl}/dots/${dot.id}/tasks`); - await expect(page.getByRole("region", { name: /^History/ }).getByRole("row", { name: /Dropped early/ })).toBeVisible(); - await expect(page.getByRole("region", { name: /^Queue/ }).getByRole("article", { name: "Waits in the queue" })).toBeVisible(); - await expect(page.getByRole("region", { name: /^Scheduled/ }).getByRole("article", { name: "Waits for its time" })).toBeVisible(); - - for (const list of ["section[aria-labelledby=tasks-running] ul", "section[aria-labelledby=tasks-scheduled] ul", "section[aria-labelledby=tasks-queue] ol"]) { - const look = await lookOf(page, list); - expect(look.listStyle, list).toBe("none"); - expect(look.paddingLeft, list).toBe("0px"); - } - const buttons: Record = { - "the Cancel button": "section[aria-labelledby=tasks-running] button:has-text('Cancel')", - "a history filter": "section[aria-labelledby=tasks-history] button[aria-pressed=false]", - "the pressed history filter": "section[aria-labelledby=tasks-history] button[aria-pressed=true]", - "the new task button": "button:has-text('New task')", - }; - for (const [name, selector] of Object.entries(buttons)) { - expect((await lookOf(page, selector)).contrast, name).toBeGreaterThanOrEqual(4.5); - } - }); -} - -test("a Dot with more tasks than the host lists says so, and one with fewer does not", async ({ page, harness }) => { - const dot = await harness.createDot("tasks-many"); - holdTasks(harness.driver.guestOf(dot.id)); - const make = async (from: number, to: number) => { - for (let start = from; start < to; start += 20) { - await Promise.all(Array.from({ length: Math.min(20, to - start) }, (_, i) => harness.api.createTask(dot.id, { description: `Backlog ${start + i}` }))); - } - }; - await make(0, TASK_LIST_LIMIT - 1); - await page.goto(`${harness.webUrl}/dots/${dot.id}/tasks`); - await expect(page.getByRole("region", { name: /^Queue/ })).toBeVisible(); - await expect(page.getByRole("status").filter({ hasText: "tasks are listed" })).toHaveCount(0); - - await make(TASK_LIST_LIMIT - 1, TASK_LIST_LIMIT + 5); - await page.reload(); - const notice = page.getByRole("status").filter({ hasText: `The newest ${TASK_LIST_LIMIT} tasks are listed` }); - await expect(notice).toBeVisible(); - await expect(page.getByRole("region", { name: /^Queue/ }).getByRole("listitem")).toHaveCount(TASK_LIST_LIMIT); - - // The tasks past the newest page are reached: read on request, joined to the queue, and the notice is gone. - await notice.getByRole("button", { name: "Show older tasks" }).click(); - await expect(page.getByRole("region", { name: /^Queue/ }).getByRole("listitem")).toHaveCount(TASK_LIST_LIMIT + 4); - await expect(notice).toHaveCount(0); -}); diff --git a/apps/web/e2e/tsconfig.json b/apps/web/e2e/tsconfig.json deleted file mode 100644 index 8dd222bc..00000000 --- a/apps/web/e2e/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "../../../tsconfig.base.json", - "compilerOptions": { - "lib": ["ES2023", "DOM"] - }, - "include": [ - "./**/*.ts", - "../playwright.config.ts" - ] -} diff --git a/apps/web/next.config.ts b/apps/web/next.config.ts deleted file mode 100644 index 0d9ceee7..00000000 --- a/apps/web/next.config.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { dirname, resolve } from "node:path"; -import { fileURLToPath } from "node:url"; -import type { NextConfig } from "next"; - -// invisible_dots sends no telemetry, and nothing turns it on. Next's build and dev server create their telemetry -// after they load this file and read this variable first, ahead of any setting the person's own Next has stored, so -// every way of building the web client (setup, npm run build, next dev) sends nothing. -process.env.NEXT_TELEMETRY_DISABLED = "1"; - -// The npm workspace root: dependencies are hoisted there, and the shared -// package is linked from there as TypeScript source. -const workspaceRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); - -const config: NextConfig = { - // `invisible-dots server` runs the built web client as a child process: - // .next/standalone holds a server.js and the files it needs, no `next` CLI. - output: "standalone", - // The workspace packages publish their .ts sources, not compiled output. - transpilePackages: ["@invisible-dots/shared", "@invisible-dots/sdk"], - outputFileTracingRoot: workspaceRoot, - poweredByHeader: false, - reactStrictMode: true, - // The shared package is written for NodeNext and imports "./config.js" for - // config.ts. Turbopack only maps .js to .ts when the app's own tsconfig uses - // NodeNext resolution, which Next's own imports ("next/navigation") do not - // type-check under, so the build uses webpack with an extension alias. - webpack(webpackConfig: { resolve?: { extensionAlias?: Record } }) { - webpackConfig.resolve ??= {}; - webpackConfig.resolve.extensionAlias = { - ...webpackConfig.resolve.extensionAlias, - ".js": [".ts", ".tsx", ".js"], - }; - return webpackConfig; - }, -}; - -export default config; diff --git a/apps/web/package.json b/apps/web/package.json deleted file mode 100644 index 76f47e35..00000000 --- a/apps/web/package.json +++ /dev/null @@ -1,54 +0,0 @@ -{ - "name": "@invisible-dots/web", - "version": "0.0.0", - "private": true, - "description": "Next.js web client.", - "license": "MIT", - "type": "module", - "exports": { - ".": "./src/index.ts" - }, - "scripts": { - "dev": "next dev --webpack --hostname 127.0.0.1 --port 3000", - "build": "node scripts/build.mjs", - "test": "vitest run --root ../.. apps/web/", - "typecheck": "tsc -p tsconfig.json", - "test:e2e": "playwright test" - }, - "dependencies": { - "@fontsource/geist-mono": "5.3.0", - "@fontsource/geist-sans": "5.3.0", - "@invisible-dots/sdk": "0.0.0", - "@invisible-dots/shared": "0.0.0", - "class-variance-authority": "^0.7.1", - "clsx": "^2.1.1", - "lucide-react": "^1.52.0", - "next": "^16.3.8", - "qrcode": "1.5.4", - "radix-ui": "^1.7.0", - "react": "^19.3.0", - "react-dom": "^19.3.0", - "react-markdown": "^10.1.0", - "remark-gfm": "^4.0.1", - "sonner": "^2.0.8", - "tailwind-merge": "^3.7.0", - "use-stick-to-bottom": "^1.1.6" - }, - "devDependencies": { - "@invisible-dots/api": "0.0.0", - "@invisible-dots/channels": "0.0.0", - "@invisible-dots/scheduler": "0.0.0", - "@playwright/test": "^1.63.0", - "@tailwindcss/postcss": "^4.3.3", - "@testing-library/dom": "^10.4.2", - "@testing-library/react": "^16.3.3", - "@testing-library/user-event": "^14.6.7", - "@types/qrcode": "1.5.6", - "@types/react": "^19.3.0", - "@types/react-dom": "^19.3.0", - "jsdom": "^30.1.2", - "postcss": "^8.5.29", - "tailwindcss": "^4.3.3", - "tw-animate-css": "^1.4.0" - } -} diff --git a/apps/web/playwright.config.ts b/apps/web/playwright.config.ts deleted file mode 100644 index 04d96325..00000000 --- a/apps/web/playwright.config.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { defineConfig, devices } from "@playwright/test"; - -/** - * The browser tests of the web client. Each worker starts its own control plane and web server (e2e/harness.ts), so - * nothing here needs a server running beforehand, only `next build`. One worker: the specs share that one server and - * make their own Dots, named for the test. - */ -export default defineConfig({ - testDir: "e2e", - testMatch: "**/*.spec.ts", - timeout: 60_000, - expect: { timeout: 10_000 }, - workers: 1, - fullyParallel: false, - forbidOnly: Boolean(process.env.CI), - // The JUnit report is written on every run, not only under CI: .github/scripts/test-guard.mjs counts what ran - // against the floors file, and a local run must not leave an older report for it to read. - reporter: [["list"], ["junit", { outputFile: "playwright-report.xml" }]], - use: { trace: "retain-on-failure" }, - projects: [{ name: "chromium", use: { ...devices["Desktop Chrome"] } }], -}); diff --git a/apps/web/postcss.config.mjs b/apps/web/postcss.config.mjs deleted file mode 100644 index 173e83ca..00000000 --- a/apps/web/postcss.config.mjs +++ /dev/null @@ -1,6 +0,0 @@ -// Tailwind CSS 4 runs as a PostCSS plugin inside Next's webpack build. -export default { - plugins: { - "@tailwindcss/postcss": {}, - }, -}; diff --git a/apps/web/scripts/build.mjs b/apps/web/scripts/build.mjs deleted file mode 100644 index 10a8d7bc..00000000 --- a/apps/web/scripts/build.mjs +++ /dev/null @@ -1,16 +0,0 @@ -// `npm run build --workspace @invisible-dots/web`: `next build`, then standalone.mjs. It is a script of its own and -// not an `&&` in package.json so that `invisible-dots setup --all` can run exactly these steps with plain node: npm -// is a .cmd file on Windows, which Node starts only through a shell, and the CLI never starts a program through one. -import { spawnSync } from "node:child_process"; -import { createRequire } from "node:module"; -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; - -const app = dirname(dirname(fileURLToPath(import.meta.url))); -const next = createRequire(join(app, "package.json")).resolve("next/dist/bin/next"); - -for (const args of [[next, "build", "--webpack"], [join(app, "scripts", "standalone.mjs")]]) { - const result = spawnSync(process.execPath, args, { cwd: app, stdio: "inherit" }); - if (result.error) throw result.error; - if (result.status !== 0) process.exit(result.status ?? 1); -} diff --git a/apps/web/scripts/standalone.mjs b/apps/web/scripts/standalone.mjs deleted file mode 100644 index 05079694..00000000 --- a/apps/web/scripts/standalone.mjs +++ /dev/null @@ -1,23 +0,0 @@ -// Finishes `next build` for `invisible-dots server`, which runs the web client -// as a child process: Next's standalone output holds the server and what it -// traced, but not the browser files (`.next/static`) or `public/`, which its -// own documentation says to copy next to it. apps/cli/src/web.ts looks for -// the result at the paths below. -import { cpSync, existsSync, rmSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; - -const app = dirname(dirname(fileURLToPath(import.meta.url))); -// The workspace root is Next's tracing root (next.config.ts), so the app sits at its own path inside the standalone tree. -const standalone = join(app, ".next", "standalone", "apps", "web"); - -if (!existsSync(join(standalone, "server.js"))) { - throw new Error(`next build left no ${join(standalone, "server.js")}: is output "standalone" set in next.config.ts?`); -} -for (const [from, to] of [ - [join(app, ".next", "static"), join(standalone, ".next", "static")], - [join(app, "public"), join(standalone, "public")], -]) { - rmSync(to, { recursive: true, force: true }); - if (existsSync(from)) cpSync(from, to, { recursive: true }); -} diff --git a/apps/web/src/app/(app)/approvals/page.tsx b/apps/web/src/app/(app)/approvals/page.tsx deleted file mode 100644 index a181c88f..00000000 --- a/apps/web/src/app/(app)/approvals/page.tsx +++ /dev/null @@ -1,7 +0,0 @@ -import { redirect } from "next/navigation"; -import { inboxHref } from "../../../lib/inbox"; - -/** The approvals of every Dot are the Inbox's first tab. */ -export default function ApprovalsPage() { - redirect(inboxHref()); -} diff --git a/apps/web/src/app/(app)/dots/[id]/activity/page.tsx b/apps/web/src/app/(app)/dots/[id]/activity/page.tsx deleted file mode 100644 index 3307e4dd..00000000 --- a/apps/web/src/app/(app)/dots/[id]/activity/page.tsx +++ /dev/null @@ -1,11 +0,0 @@ -import type { Metadata } from "next"; -import { dotPageTitle } from "../../../../../lib/dot-title"; -import { ActivityView } from "../../../../../components/activity/ActivityView"; - -export async function generateMetadata({ params }: { params: Promise<{ id: string }> }): Promise { - return dotPageTitle((await params).id, "Activity"); -} - -export default function Page() { - return ; -} diff --git a/apps/web/src/app/(app)/dots/[id]/approvals/page.tsx b/apps/web/src/app/(app)/dots/[id]/approvals/page.tsx deleted file mode 100644 index c1e33cb4..00000000 --- a/apps/web/src/app/(app)/dots/[id]/approvals/page.tsx +++ /dev/null @@ -1,8 +0,0 @@ -import { redirect } from "next/navigation"; -import { inboxHref } from "../../../../../lib/inbox"; - -/** The approvals of one Dot are the Inbox filtered to it. */ -export default async function Page({ params }: { params: Promise<{ id: string }> }) { - const { id } = await params; - redirect(inboxHref({ dot: id })); -} diff --git a/apps/web/src/app/(app)/dots/[id]/channels/page.tsx b/apps/web/src/app/(app)/dots/[id]/channels/page.tsx deleted file mode 100644 index 4294784a..00000000 --- a/apps/web/src/app/(app)/dots/[id]/channels/page.tsx +++ /dev/null @@ -1,11 +0,0 @@ -import type { Metadata } from "next"; -import { dotPageTitle } from "../../../../../lib/dot-title"; -import { ChannelsView } from "../../../../../components/channels/ChannelsView"; - -export async function generateMetadata({ params }: { params: Promise<{ id: string }> }): Promise { - return dotPageTitle((await params).id, "Channels"); -} - -export default function Page() { - return ; -} diff --git a/apps/web/src/app/(app)/dots/[id]/chat/page.tsx b/apps/web/src/app/(app)/dots/[id]/chat/page.tsx deleted file mode 100644 index e6ee6e44..00000000 --- a/apps/web/src/app/(app)/dots/[id]/chat/page.tsx +++ /dev/null @@ -1,11 +0,0 @@ -import type { Metadata } from "next"; -import { dotPageTitle } from "../../../../../lib/dot-title"; -import { ChatView } from "../../../../../components/chat/ChatView"; - -export async function generateMetadata({ params }: { params: Promise<{ id: string }> }): Promise { - return dotPageTitle((await params).id, "Chat"); -} - -export default function Page() { - return ; -} diff --git a/apps/web/src/app/(app)/dots/[id]/computer/page.tsx b/apps/web/src/app/(app)/dots/[id]/computer/page.tsx deleted file mode 100644 index c67adaac..00000000 --- a/apps/web/src/app/(app)/dots/[id]/computer/page.tsx +++ /dev/null @@ -1,12 +0,0 @@ -import type { Metadata } from "next"; -import { dotPageTitle } from "../../../../../lib/dot-title"; -import { ComputerView } from "../../../../../components/computer/ComputerView"; -import { parseComputerQuery } from "../../../../../lib/computer-view"; - -export async function generateMetadata({ params }: { params: Promise<{ id: string }> }): Promise { - return dotPageTitle((await params).id, "Computer"); -} - -export default async function Page({ searchParams }: { searchParams: Promise> }) { - return ; -} diff --git a/apps/web/src/app/(app)/dots/[id]/layout.tsx b/apps/web/src/app/(app)/dots/[id]/layout.tsx deleted file mode 100644 index f5eb13f6..00000000 --- a/apps/web/src/app/(app)/dots/[id]/layout.tsx +++ /dev/null @@ -1,18 +0,0 @@ -import type { ReactNode } from "react"; -import { DotShell } from "../../../../components/DotShell"; -import { DotEventScope } from "../../../../components/events"; - -export default async function DotLayout({ - children, - params, -}: { - children: ReactNode; - params: Promise<{ id: string }>; -}) { - const { id } = await params; - return ( - - {children} - - ); -} diff --git a/apps/web/src/app/(app)/dots/[id]/page.tsx b/apps/web/src/app/(app)/dots/[id]/page.tsx deleted file mode 100644 index 5a41ad33..00000000 --- a/apps/web/src/app/(app)/dots/[id]/page.tsx +++ /dev/null @@ -1,6 +0,0 @@ -import { redirect } from "next/navigation"; - -export default async function DotIndex({ params }: { params: Promise<{ id: string }> }) { - const { id } = await params; - redirect(`/dots/${encodeURIComponent(id)}/chat`); -} diff --git a/apps/web/src/app/(app)/dots/[id]/settings/page.tsx b/apps/web/src/app/(app)/dots/[id]/settings/page.tsx deleted file mode 100644 index dfbe200b..00000000 --- a/apps/web/src/app/(app)/dots/[id]/settings/page.tsx +++ /dev/null @@ -1,11 +0,0 @@ -import type { Metadata } from "next"; -import { dotPageTitle } from "../../../../../lib/dot-title"; -import { DotSettings } from "../../../../../components/dot-settings/DotSettings"; - -export async function generateMetadata({ params }: { params: Promise<{ id: string }> }): Promise { - return dotPageTitle((await params).id, "Settings"); -} - -export default function Page() { - return ; -} diff --git a/apps/web/src/app/(app)/dots/[id]/skills/page.tsx b/apps/web/src/app/(app)/dots/[id]/skills/page.tsx deleted file mode 100644 index 6906fad8..00000000 --- a/apps/web/src/app/(app)/dots/[id]/skills/page.tsx +++ /dev/null @@ -1,12 +0,0 @@ -import type { Metadata } from "next"; -import { dotPageTitle } from "../../../../../lib/dot-title"; -import { SkillsView } from "../../../../../components/skills/SkillsView"; -import { parseSkillsQuery } from "../../../../../lib/skills-view"; - -export async function generateMetadata({ params }: { params: Promise<{ id: string }> }): Promise { - return dotPageTitle((await params).id, "Skills"); -} - -export default async function Page({ searchParams }: { searchParams: Promise> }) { - return ; -} diff --git a/apps/web/src/app/(app)/dots/[id]/tasks/[taskId]/page.tsx b/apps/web/src/app/(app)/dots/[id]/tasks/[taskId]/page.tsx deleted file mode 100644 index 2d2f8503..00000000 --- a/apps/web/src/app/(app)/dots/[id]/tasks/[taskId]/page.tsx +++ /dev/null @@ -1,12 +0,0 @@ -import type { Metadata } from "next"; -import { dotPageTitle } from "../../../../../../lib/dot-title"; -import { TaskDrawer } from "../../../../../../components/tasks/task-drawer"; - -export async function generateMetadata({ params }: { params: Promise<{ id: string }> }): Promise { - return dotPageTitle((await params).id, "Task"); -} - -export default async function Page({ params }: { params: Promise<{ taskId: string }> }) { - const { taskId } = await params; - return ; -} diff --git a/apps/web/src/app/(app)/dots/[id]/tasks/layout.tsx b/apps/web/src/app/(app)/dots/[id]/tasks/layout.tsx deleted file mode 100644 index 77d8d6f1..00000000 --- a/apps/web/src/app/(app)/dots/[id]/tasks/layout.tsx +++ /dev/null @@ -1,6 +0,0 @@ -import type { ReactNode } from "react"; -import { TasksView } from "../../../../../components/tasks/TasksView"; - -export default function Layout({ children }: { children: ReactNode }) { - return {children}; -} diff --git a/apps/web/src/app/(app)/dots/[id]/tasks/page.tsx b/apps/web/src/app/(app)/dots/[id]/tasks/page.tsx deleted file mode 100644 index a05ae260..00000000 --- a/apps/web/src/app/(app)/dots/[id]/tasks/page.tsx +++ /dev/null @@ -1,11 +0,0 @@ -import type { Metadata } from "next"; -import { dotPageTitle } from "../../../../../lib/dot-title"; - -export async function generateMetadata({ params }: { params: Promise<{ id: string }> }): Promise { - return dotPageTitle((await params).id, "Tasks"); -} - -/** The list is the layout's: this page is the address with no task open, and adds nothing to it. */ -export default function Page() { - return null; -} diff --git a/apps/web/src/app/(app)/dots/[id]/timeline/page.tsx b/apps/web/src/app/(app)/dots/[id]/timeline/page.tsx deleted file mode 100644 index 3be5110d..00000000 --- a/apps/web/src/app/(app)/dots/[id]/timeline/page.tsx +++ /dev/null @@ -1,7 +0,0 @@ -import { redirect } from "next/navigation"; - -/** The Timeline became the Activity page. */ -export default async function Page({ params }: { params: Promise<{ id: string }> }) { - const { id } = await params; - redirect(`/dots/${encodeURIComponent(id)}/activity`); -} diff --git a/apps/web/src/app/(app)/inbox/page.tsx b/apps/web/src/app/(app)/inbox/page.tsx deleted file mode 100644 index f196ba2c..00000000 --- a/apps/web/src/app/(app)/inbox/page.tsx +++ /dev/null @@ -1,9 +0,0 @@ -import type { Metadata } from "next"; -import { InboxView } from "../../../components/inbox/InboxView"; -import { parseInboxQuery } from "../../../lib/inbox"; - -export const metadata: Metadata = { title: "Inbox" }; - -export default async function InboxPage({ searchParams }: { searchParams: Promise> }) { - return ; -} diff --git a/apps/web/src/app/(app)/layout.tsx b/apps/web/src/app/(app)/layout.tsx deleted file mode 100644 index 29f155e8..00000000 --- a/apps/web/src/app/(app)/layout.tsx +++ /dev/null @@ -1,18 +0,0 @@ -import type { ReactNode } from "react"; -import { EventStreamProvider } from "../../components/events"; -import { AttentionProvider } from "../../components/shell/attention"; -import { AppShell } from "../../components/shell/AppShell"; - -/** - * Every page of the app: one live stream, what needs the person, and the frame with the rail. The page for an address - * with nothing behind it lives outside this group, so it shows without them. - */ -export default function AppLayout({ children }: { children: ReactNode }) { - return ( - - - {children} - - - ); -} diff --git a/apps/web/src/app/(app)/new/page.tsx b/apps/web/src/app/(app)/new/page.tsx deleted file mode 100644 index 102f1a8e..00000000 --- a/apps/web/src/app/(app)/new/page.tsx +++ /dev/null @@ -1,8 +0,0 @@ -import type { Metadata } from "next"; -import { NewDotPage } from "../../../components/new-dot/NewDotPage"; - -export const metadata: Metadata = { title: "Create a Dot" }; - -export default function New() { - return ; -} diff --git a/apps/web/src/app/(app)/page.tsx b/apps/web/src/app/(app)/page.tsx deleted file mode 100644 index 1fc87e13..00000000 --- a/apps/web/src/app/(app)/page.tsx +++ /dev/null @@ -1,8 +0,0 @@ -import type { Metadata } from "next"; -import { HomePage } from "../../components/home/HomePage"; - -export const metadata: Metadata = { title: "Dots" }; - -export default function Home() { - return ; -} diff --git a/apps/web/src/app/(app)/settings/page.tsx b/apps/web/src/app/(app)/settings/page.tsx deleted file mode 100644 index b9317f2a..00000000 --- a/apps/web/src/app/(app)/settings/page.tsx +++ /dev/null @@ -1,8 +0,0 @@ -import type { Metadata } from "next"; -import { HostSettings } from "../../../components/settings/HostSettings"; - -export const metadata: Metadata = { title: "Settings" }; - -export default function SettingsPage() { - return ; -} diff --git a/apps/web/src/app/api/[...path]/route.ts b/apps/web/src/app/api/[...path]/route.ts deleted file mode 100644 index 8d854407..00000000 --- a/apps/web/src/app/api/[...path]/route.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { - ProxyError, - apiBaseUrl, - checkRequestOrigin, - errorResponse, - forwardRequestHeaders, - forwardResponseHeaders, - loadApiToken, - upstreamUrl, -} from "../../../lib/proxy"; - -export const dynamic = "force-dynamic"; -export const runtime = "nodejs"; - -interface RouteContext { - params: Promise<{ path: string[] }>; -} - -async function proxy(request: Request, context: RouteContext): Promise { - const refusal = checkRequestOrigin({ - method: request.method, - host: request.headers.get("host"), - origin: request.headers.get("origin"), - secFetchSite: request.headers.get("sec-fetch-site"), - }); - if (refusal) return errorResponse(403, "forbidden", refusal); - - const { path } = await context.params; - let target: string; - let token: string; - try { - target = upstreamUrl(apiBaseUrl(), path, new URL(request.url).search); - token = await loadApiToken(); - } catch (error) { - if (error instanceof ProxyError) { - if (error.status >= 500) console.error(`[web proxy] ${error.message}`); - return errorResponse(error.status, error.code, error.message); - } - throw error; - } - const hasBody = request.method !== "GET" && request.method !== "HEAD"; - let upstream: Response; - try { - upstream = await fetch(target, { - method: request.method, - headers: forwardRequestHeaders(request.headers, token), - body: hasBody ? await request.arrayBuffer() : undefined, - // Closing the browser tab aborts the request, which ends an SSE stream upstream too. - signal: request.signal, - redirect: "manual", - cache: "no-store", - }); - } catch (error) { - if (request.signal.aborted) return new Response(null, { status: 499 }); - const reason = (error as Error & { cause?: { code?: string } }).cause?.code ?? (error as Error).message; - console.error(`[web proxy] ${request.method} ${target} failed: ${reason}`); - return errorResponse(502, "api_unreachable", `the control plane API at ${apiBaseUrl()} did not answer (${reason})`); - } - - return new Response(upstream.body, { - status: upstream.status, - statusText: upstream.statusText, - headers: forwardResponseHeaders(upstream.headers), - }); -} - -export const GET = proxy; -export const POST = proxy; -export const PUT = proxy; -export const PATCH = proxy; -export const DELETE = proxy; diff --git a/apps/web/src/app/globals.css b/apps/web/src/app/globals.css deleted file mode 100644 index 69b19bd5..00000000 --- a/apps/web/src/app/globals.css +++ /dev/null @@ -1,116 +0,0 @@ -/* - * The stylesheet of the web client: Tailwind 4 and the design tokens (tokens.css). Every screen is styled by - * utilities over those tokens; there is no other stylesheet. - */ -@import "tailwindcss"; -@import "tw-animate-css"; -@import "./tokens.css"; - -@custom-variant dark (&:is([data-theme="dark"] *)); - -@theme inline { - /* Only the tokens are colors: no stray palette to reach for. */ - --color-*: initial; - --color-black: #000000; - --color-white: #ffffff; - --color-background: var(--background); - --color-foreground: var(--foreground); - --color-card: var(--card); - --color-card-foreground: var(--card-foreground); - --color-popover: var(--popover); - --color-popover-foreground: var(--popover-foreground); - --color-muted: var(--muted); - --color-muted-foreground: var(--muted-foreground); - --color-secondary: var(--secondary); - --color-secondary-foreground: var(--secondary-foreground); - --color-accent: var(--accent); - --color-accent-foreground: var(--accent-foreground); - --color-border: var(--border); - --color-input: var(--input); - --color-ring: var(--ring); - --color-primary: var(--primary); - --color-primary-foreground: var(--primary-foreground); - --color-primary-hover: var(--primary-hover); - --color-ok: var(--ok); - --color-ok-soft: var(--ok-soft); - --color-warn: var(--warn); - --color-warn-soft: var(--warn-soft); - --color-danger: var(--danger); - --color-danger-soft: var(--danger-soft); - --color-info: var(--info); - --color-info-soft: var(--info-soft); - --color-attention: var(--attention); - --color-destructive: var(--destructive); - --color-destructive-foreground: var(--destructive-foreground); - - --radius-sm: calc(var(--radius) - 4px); - --radius-md: calc(var(--radius) - 2px); - --radius-lg: var(--radius); - --radius-xl: calc(var(--radius) + 4px); - - --font-sans: var(--font-ui); - --font-mono: var(--font-code); - - /* The scale is 12/13/14/16/20/24/30; text-ui is the 13. */ - --text-ui: 0.8125rem; - --text-ui--line-height: 1.25rem; - - --animate-breathe: breathe 2.4s ease-in-out infinite; - @keyframes breathe { - 0%, - 100% { - box-shadow: 0 0 0 0 color-mix(in srgb, var(--attention) 55%, transparent); - } - 50% { - box-shadow: 0 0 0 4px color-mix(in srgb, var(--attention) 0%, transparent); - } - } -} - -@layer base { - * { - border-color: var(--border); - outline-color: var(--ring); - } - - body { - background: var(--background); - color: var(--foreground); - font-size: 0.875rem; - line-height: 1.5; - } - - /* Motion is a courtesy: under this setting nothing animates. */ - @media (prefers-reduced-motion: reduce) { - *, - ::before, - ::after { - animation-duration: 0.01ms !important; - animation-iteration-count: 1 !important; - transition-duration: 0.01ms !important; - } - } -} - -/* A label that is going on now: the text carries a light that passes over it (both ends are text colors of the tokens). */ -@utility shimmer { - background-image: linear-gradient(90deg, var(--muted-foreground) 0%, var(--foreground) 50%, var(--muted-foreground) 100%); - background-size: 200% 100%; - background-clip: text; - -webkit-background-clip: text; - -webkit-text-fill-color: transparent; - animation: shimmer 2.2s linear infinite; - - /* Forced colors (Windows High Contrast) drop the gradient but keep a transparent fill: the label would vanish. It is drawn in the system text color, still. */ - @media (forced-colors: active) { - background-image: none; - -webkit-text-fill-color: currentColor; - animation: none; - } -} - -@keyframes shimmer { - to { - background-position: -200% 0; - } -} diff --git a/apps/web/src/app/layout.tsx b/apps/web/src/app/layout.tsx deleted file mode 100644 index 03a1470f..00000000 --- a/apps/web/src/app/layout.tsx +++ /dev/null @@ -1,40 +0,0 @@ -import type { Metadata, Viewport } from "next"; -import type { ReactNode } from "react"; -import { Toaster } from "../components/ui/sonner"; -import { THEME_BOOTSTRAP } from "../lib/theme"; -import "@fontsource/geist-sans/400.css"; -import "@fontsource/geist-sans/500.css"; -import "@fontsource/geist-sans/600.css"; -import "@fontsource/geist-mono/400.css"; -import "@fontsource/geist-mono/500.css"; -import "./globals.css"; - -export const metadata: Metadata = { - title: { default: "invisible_dots", template: "%s - invisible_dots" }, - description: "Control plane for Dots: agents that each own a Linux computer.", -}; - -export const viewport: Viewport = { - colorScheme: "light dark", -}; - -export default function RootLayout({ children }: { children: ReactNode }) { - return ( - // The theme script sets data-theme before the first paint, so the server's markup has none to agree with. - - - raw after'}); - expect(container.querySelector("img")).toBeNull(); - expect(container.querySelector("script")).toBeNull(); - expect(container.querySelector("b")).toBeNull(); - expect(container.textContent).toContain("before"); - expect(container.textContent).toContain("after"); - }); - - it("does not turn a javascript: link into a link", () => { - const { container } = render({"[click](javascript:alert(1))"}); - expect(container.querySelector("a")?.getAttribute("href") ?? "").not.toMatch(/^javascript:/i); - }); - - it("shows an image as a link to it, so that reading the message asks nobody for the picture", () => { - const { container } = render({"before ![a chart](https://tracker.example/pixel.png?q=secret) after"}); - expect(container.querySelector("img")).toBeNull(); - const link = container.querySelector("a")!; - expect(link.getAttribute("href")).toBe("https://tracker.example/pixel.png?q=secret"); - expect(link.textContent).toBe("Image: a chart"); - expect(link.getAttribute("rel")).toBe("noreferrer"); - }); - - it("puts a copy button on a fenced block, which copies its code and says so", async () => { - const writeText = vi.fn(async (_text: string) => {}); - vi.stubGlobal("navigator", { clipboard: { writeText } }); - const { container } = render({"Run:\n\n```sh\nls -la\necho done\n```\n\nand `inline` code"}); - expect(container.querySelectorAll("pre")).toHaveLength(1); - expect(screen.getAllByRole("button", { name: "Copy the code" })).toHaveLength(1); - await userEvent.click(screen.getByRole("button", { name: "Copy the code" })); - expect(writeText).toHaveBeenCalledWith("ls -la\necho done"); - expect(await screen.findByRole("button", { name: "Copied" })).toBeTruthy(); - }); - - it("keeps the code readable when the clipboard refuses", async () => { - vi.stubGlobal("navigator", { clipboard: { writeText: vi.fn(async () => Promise.reject(new Error("denied"))) } }); - const { container } = render({"```\nsecret = 1\n```"}); - await userEvent.click(screen.getByRole("button", { name: "Copy the code" })); - expect(container.querySelector("pre")?.textContent).toContain("secret = 1"); - expect(screen.queryByRole("button", { name: "Copied" })).toBeNull(); - }); -}); diff --git a/apps/web/test/mcp-servers.test.ts b/apps/web/test/mcp-servers.test.ts deleted file mode 100644 index 2ae50ccb..00000000 --- a/apps/web/test/mcp-servers.test.ts +++ /dev/null @@ -1,132 +0,0 @@ -import { parseDotConfig, resolvePermission } from "@invisible-dots/shared/browser"; -import { describe, expect, it } from "vitest"; -import { approvalBody, askTitle, isDestructive, permissionInfo, type ApprovalAsk } from "../src/lib/approval-view"; -import { viewEvent } from "../src/lib/events/view"; -import { configChanges, rebase, setMcpServer, setPermission } from "../src/lib/config-fields"; -import { EMPTY_MCP_SERVER_FORM, formOfServer, serverOfForm } from "../src/lib/mcp-server-form"; -import { presetOf, presetPermissions } from "../src/lib/permission-presets"; -import { permissionGroups } from "../src/lib/permission-table"; -import { fullConfig } from "./support/config"; - -const TIME = { command: "uvx", args: ["mcp-server-time"], env: {}, secrets: ["TIME_TOKEN"], timeout_s: 120, startup_timeout_s: 60 }; -const WEB = { url: "https://web.example/mcp", headers: { "X-Client": "dots" }, secrets: ["Authorization"], timeout_s: 30, startup_timeout_s: 90 }; - -describe("the form of an MCP server", () => { - it("is the entry it was made from, for a program and for a URL", () => { - for (const server of [TIME, WEB]) { - const back = serverOfForm(formOfServer(server)); - expect(back).toEqual({ ok: true, server }); - } - }); - - it("reads one argument, variable, header or secret a line, keeps the spaces inside one, and names a line it cannot read", () => { - const form = { ...EMPTY_MCP_SERVER_FORM, command: " npx ", args: "-y\n@scope/server --flag\n\n", env: "MODE=a=b\n", secrets: "API_KEY\n" }; - expect(serverOfForm(form)).toEqual({ ok: true, server: { command: "npx", args: ["-y", "@scope/server --flag"], env: { MODE: "a=b" }, secrets: ["API_KEY"], timeout_s: 120, startup_timeout_s: 60 } }); - expect(serverOfForm({ ...form, env: "MODE" })).toEqual({ ok: false, problem: '"MODE" is not NAME=value' }); - expect(serverOfForm({ ...EMPTY_MCP_SERVER_FORM, kind: "url", url: "https://a.example", headers: "no colon" })).toEqual({ ok: false, problem: '"no colon" is not Name: value' }); - }); -}); - -describe("an MCP server in the settings", () => { - const base = fullConfig({ mcp_servers: { time: TIME } }); - - it("is added and removed in the draft, and its permission goes with it", () => { - const added = setMcpServer(base, "web", WEB); - expect(parseDotConfig(added).mcp_servers).toEqual({ time: TIME, web: WEB }); - const allowed = setPermission(added, "mcp.web", "allow"); - expect(allowed.permissions).toEqual({ "mcp.web": "allow" }); - const removed = setMcpServer(allowed, "web", null); - expect(removed.permissions).toEqual({}); - expect(parseDotConfig(removed).mcp_servers).toEqual({ time: TIME }); - }); - - it("never writes the permission of a server the config does not declare, which the schema would refuse", () => { - expect(setPermission(base, "mcp.ghost", "allow").permissions).toEqual({}); - expect(setPermission(base, "mcp.time", "ask").permissions).toEqual({}); - expect(setPermission(base, "mcp.time", "deny").permissions).toEqual({ "mcp.time": "deny" }); - }); - - it("is a change the review lists, by what it runs, and its permission is another", () => { - const changes = configChanges(base, setPermission(setMcpServer(base, "web", WEB), "mcp.web", "allow")); - expect(changes.map((c) => [c.key, c.before, c.after])).toEqual([ - ["mcp_servers.web", "(not declared)", "https://web.example/mcp"], - ["permissions.mcp.web", "deny", "allow"], - ]); - // The same entry with its keys in another order is no change. - const reordered = { ...base, mcp_servers: { time: { startup_timeout_s: 60, timeout_s: 120, secrets: ["TIME_TOKEN"], env: {}, args: ["mcp-server-time"], command: "uvx" } } }; - expect(configChanges(base, reordered)).toEqual([]); - }); - - it("an edit put on top of a config changed meanwhile keeps the server someone else added and the one this edit added", () => { - const latest = setMcpServer(base, "web", WEB); - const mine = setMcpServer(base, "files", { command: "npx", args: ["-y", "files"], env: {}, secrets: [], timeout_s: 60, startup_timeout_s: 60 }); - const merged = rebase(base, mine, latest); - expect(Object.keys(merged.mcp_servers).sort()).toEqual(["files", "time", "web"]); - // Removing a server in the edit removes it from the newer config too, with its permission. - const dropped = rebase(latest, setMcpServer(latest, "time", null), setPermission(latest, "mcp.time", "allow")); - expect(dropped.mcp_servers).not.toHaveProperty("time"); - expect(dropped.permissions).toEqual({}); - }); - - it("a preset leaves the servers' permissions as they are, and they do not decide which preset the config is", () => { - const current = { "mcp.time": "deny" as const, "computer.exec": "ask" as const }; - expect(presetPermissions("autonomous", current)["mcp.time"]).toBe("deny"); - expect(presetPermissions("careful", current)).toMatchObject({ "mcp.time": "deny", "computer.exec": "ask", "files.write": "ask" }); - expect(presetOf({ "mcp.time": "allow" })).toBe("balanced"); - }); - - it("has a row under MCP servers with the risk of running commands, the default ask, and the tools of the server", () => { - const tools = [{ name: "mcp_time_now", permission: "mcp.time" as const, offered: true, description: "Now." }]; - const group = permissionGroups(base, base, tools).find((g) => g.id === "mcp"); - expect(group?.label).toBe("MCP servers"); - // The permission to add a server comes first, then one row per declared server. - expect(group?.rows.map((row) => [row.permission, row.decision])).toEqual([["mcp.servers.add", "ask"], ["mcp.time", "ask"]]); - expect(group?.rows.slice(1)).toEqual([ - { - permission: "mcp.time", - label: "MCP server time", - description: expect.stringContaining('"time"'), - risk: "high", - decision: "ask", - defaultDecision: "ask", - changed: false, - tools: [{ name: "mcp_time_now", offered: true }], - }, - ]); - expect(resolvePermission(base, "mcp.time")).toBe("ask"); - }); - - it("an approval of one of its tools is named after the server and is the kind to stop and think about", () => { - const ask = { permission: "mcp.time", tool: "mcp_time_now", arguments: {} } as unknown as ApprovalAsk; - expect(permissionInfo("mcp.time")).toMatchObject({ permission: "mcp.time", label: "MCP server time", risk: "high" }); - expect(isDestructive(ask)).toBe(true); - expect(permissionInfo("mcp.Bad")).toBeNull(); - }); -}); - -describe("a server the Dot adds itself", () => { - const ask = (args: Record) => ({ permission: "mcp.servers.add", tool: "add_mcp_server", arguments: args }) as unknown as ApprovalAsk; - - it("asks with what the server runs, the secrets the person will set and why, never more", () => { - const body = approvalBody(ask({ name: "clock", command: "uvx", args: ["--from", "git+https://github.com/o/r", "clock-mcp"], env: { TZ: "Europe/Rome" }, secrets: ["API_KEY"], reason: "to tell the time" })); - expect(body).toEqual({ - kind: "facts", - facts: [ - { label: "Name", value: "clock" }, - { label: "Runs, as the Dot's user", value: "uvx --from git+https://github.com/o/r clock-mcp" }, - { label: "Environment", value: "TZ=Europe/Rome" }, - { label: "Secrets you will set", value: "API_KEY" }, - { label: "Why", value: "to tell the time" }, - ], - }); - expect(approvalBody(ask({ name: "web", url: "https://w.example/mcp", reason: "r" }))).toMatchObject({ facts: [{ label: "Name" }, { label: "URL", value: "https://w.example/mcp" }, { label: "Why" }] }); - expect(askTitle(ask({}))).toBe("Wants to add an MCP server to its config"); - expect(permissionInfo("mcp.servers.add")).toMatchObject({ label: "Add MCP servers", risk: "high" }); - }); - - it("says in the activity what was added, and why one was not", () => { - const event = (type: string, data: Record) => viewEvent({ id: 1, dot_id: "d", type, source: "guest", data, created_at: "2026-10-11T00:00:00Z" } as never); - expect(event("mcp.server.added", { arguments: { name: "clock", command: "uvx", args: ["mcp-server-time"] } })).toMatchObject({ title: "Added an MCP server", detail: "clock - uvx mcp-server-time" }); - expect(event("mcp.server.refused", { name: "clock", reason: "the Dot's config denies mcp.servers.add" })).toMatchObject({ title: "An MCP server was not added", tone: "warn" }); - }); -}); diff --git a/apps/web/test/new-dot.test.tsx b/apps/web/test/new-dot.test.tsx deleted file mode 100644 index 697bf20e..00000000 --- a/apps/web/test/new-dot.test.tsx +++ /dev/null @@ -1,323 +0,0 @@ -// @vitest-environment jsdom -import { cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react"; -import userEvent from "@testing-library/user-event"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { EventStreamProvider } from "../src/components/events"; -import { NewDotPage } from "../src/components/new-dot/NewDotPage"; -import { AttentionProvider } from "../src/components/shell/attention"; -import { stubMatchMedia } from "./support/browser"; -import { dotRecord, FakeControlPlane } from "./support/control-plane"; - -const push = vi.hoisted(() => vi.fn()); -vi.mock("next/navigation", () => ({ usePathname: () => "/new", useRouter: () => ({ push, replace() {} }) })); - -let plane: FakeControlPlane; - -beforeEach(() => { - push.mockClear(); - plane = new FakeControlPlane(); - plane.install(); - stubMatchMedia(); -}); - -afterEach(() => { - cleanup(); - vi.unstubAllGlobals(); - vi.restoreAllMocks(); -}); - -async function renderPage() { - render( - - - - - , - ); - await screen.findByRole("heading", { name: "Create a Dot" }); - await waitFor(() => expect(plane.streamOpen).toBe(true)); -} - -async function fillIdentity(name = "fare-watch") { - await userEvent.type(screen.getByLabelText("Name"), name); -} - -describe("Create a Dot: the form", () => { - it("has the three steps, with the defaults of the schema", async () => { - await renderPage(); - for (const step of ["Identity", "Brain", "Computer and safety"]) expect(screen.getByRole("heading", { name: step })).toBeTruthy(); - expect((screen.getByLabelText("Model") as HTMLInputElement).value).toBe("z-ai/glm-5.3-flash"); - expect((screen.getByLabelText("Processors") as HTMLInputElement).value).toBe("2"); - expect((screen.getByLabelText("Memory") as HTMLInputElement).value).toBe("4"); - expect((screen.getByLabelText("Disk") as HTMLInputElement).value).toBe("40"); - expect((screen.getByLabelText("Sleep after") as HTMLSelectElement).value).toBe("15m"); - expect((screen.getByLabelText("Spending cap per task") as HTMLInputElement).value).toBe("1"); - expect((screen.getByRole("radio", { name: /Balanced/ }) as HTMLInputElement).checked).toBe(true); - }); - - it("gives the sliders the schema's bounds", async () => { - await renderPage(); - const bounds = (label: string) => { - const slider = screen.getByLabelText(label) as HTMLInputElement; - return [Number(slider.min), Number(slider.max)]; - }; - expect(bounds("Processors")).toEqual([1, 16]); - expect(bounds("Memory")).toEqual([2, 64]); - expect(bounds("Disk")).toEqual([20, 1024]); - }); - - it("creates the Dot with what the form shows, and opens its chat", async () => { - await renderPage(); - await fillIdentity(); - await userEvent.type(screen.getByLabelText(/Instructions/), "Write to fares.csv."); - fireEvent.change(screen.getByLabelText("Processors"), { target: { value: "4" } }); - fireEvent.change(screen.getByLabelText("Memory"), { target: { value: "8" } }); - await userEvent.selectOptions(screen.getByLabelText("Sleep after"), "30m"); - await userEvent.click(screen.getByRole("button", { name: "Create Dot" })); - - await waitFor(() => expect(push).toHaveBeenCalledWith("/dots/created-1/chat")); - expect(plane.created).toEqual([ - { - name: "fare-watch", - instructions: "Write to fares.csv.", - model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, - computer: { cpu: 4, memory: "8gb", disk: "40gb", idle_timeout: "30m" }, - permissions: {}, - limits: { max_cost_per_task_usd: 1 }, - }, - ]); - }); - - it("sends the preset that was chosen as the permissions", async () => { - await renderPage(); - await fillIdentity(); - await userEvent.click(screen.getByRole("radio", { name: /Careful/ })); - expect((screen.getByRole("radio", { name: /Careful/ }) as HTMLInputElement).checked).toBe(true); - expect((screen.getByRole("radio", { name: /Balanced/ }) as HTMLInputElement).checked).toBe(false); - await userEvent.click(screen.getByRole("button", { name: "Create Dot" })); - await waitFor(() => expect(plane.created).toHaveLength(1)); - expect((plane.created[0] as { permissions: unknown }).permissions).toEqual({ "files.write": "ask", "computer.exec": "ask" }); - }); - - it("sends the summary model, and the cost cap that was typed", async () => { - await renderPage(); - await fillIdentity(); - await userEvent.type(screen.getByLabelText(/Summary model/), "a/summary"); - const cap = screen.getByLabelText("Spending cap per task"); - await userEvent.clear(cap); - await userEvent.type(cap, "2.5"); - await userEvent.click(screen.getByRole("button", { name: "Create Dot" })); - await waitFor(() => expect(plane.created).toHaveLength(1)); - const sent = plane.created[0] as { models: unknown; limits: unknown }; - expect(sent.models).toEqual({ summary: "a/summary" }); - expect(sent.limits).toEqual({ max_cost_per_task_usd: 2.5 }); - }); - - it("takes a typed number only while it is inside the range, and shows the value in force when the field is left", async () => { - await renderPage(); - const exact = screen.getByLabelText("Memory, exact value") as HTMLInputElement; - const slider = screen.getByLabelText("Memory") as HTMLInputElement; - await userEvent.clear(exact); - await userEvent.type(exact, "999"); - // "9" was in range; "99" and "999" were not, so the value stays what the last good text made it. - expect(slider.value).toBe("9"); - await userEvent.tab(); - expect(exact.value).toBe("9"); - await userEvent.clear(exact); - await userEvent.type(exact, "16"); - expect(slider.value).toBe("16"); - fireEvent.change(slider, { target: { value: "32" } }); - expect(exact.value).toBe("32"); - }); -}); - -describe("Create a Dot: what is wrong", () => { - it("does not create a Dot with no name, asks for no goal, and lists what to fix", async () => { - await renderPage(); - expect(screen.queryByLabelText("Goal")).toBeNull(); - await userEvent.click(screen.getByRole("button", { name: "Create Dot" })); - expect(plane.created).toEqual([]); - const summary = await screen.findByRole("alert"); - expect(within(summary).getAllByRole("listitem")).toHaveLength(1); - expect(screen.getByLabelText("Name").getAttribute("aria-invalid")).toBe("true"); - await waitFor(() => expect(document.activeElement).toBe(summary.parentElement)); - }); - - it("says a name is not valid as it is typed, and not before the person has touched it", async () => { - await renderPage(); - expect(screen.getByLabelText("Name").getAttribute("aria-invalid")).toBeNull(); - await userEvent.type(screen.getByLabelText("Name"), "Fare Watch"); - const name = screen.getByLabelText("Name"); - expect(name.getAttribute("aria-invalid")).toBe("true"); - expect(document.getElementById(name.getAttribute("aria-describedby")!.split(" ").at(-1)!)!.textContent).toMatch(/lowercase letters, digits and '-'/); - await userEvent.clear(name); - await userEvent.type(name, "fare-watch"); - expect(name.getAttribute("aria-invalid")).toBeNull(); - }); - - it("refuses a name another Dot already has", async () => { - plane.dots = [dotRecord("d1", { name: "fares" })]; - await renderPage(); - await waitFor(() => expect(plane.requests.filter((r) => r === "GET /api/dots").length).toBeGreaterThan(0)); - await userEvent.type(screen.getByLabelText("Name"), "fares"); - await waitFor(() => expect(screen.getByLabelText("Name").getAttribute("aria-invalid")).toBe("true")); - expect(screen.getByText('A Dot named "fares" already exists')).toBeTruthy(); - }); - - it("shows what the control plane answered when it refuses, and stays on the page", async () => { - plane.failCreate = { status: 409, error: "name_taken", message: 'a Dot named "fare-watch" already exists', details: [{ path: ["name"], message: "taken meanwhile" }] }; - await renderPage(); - await fillIdentity(); - await userEvent.click(screen.getByRole("button", { name: "Create Dot" })); - const alert = await screen.findByRole("alert"); - expect(alert.textContent).toContain("The Dot was not created"); - expect(alert.textContent).toContain('a Dot named "fare-watch" already exists'); - expect(alert.textContent).toContain("taken meanwhile"); - expect(push).not.toHaveBeenCalled(); - // The button is back, so the person can try again. - expect((screen.getByRole("button", { name: "Create Dot" }) as HTMLButtonElement).disabled).toBe(false); - }); -}); - -/** The line of the checks panel for one item, as a screen reader hears it. */ -function checkLine(panel: HTMLElement, label: string): string { - const line = within(panel).getAllByRole("listitem").find((item) => item.textContent?.startsWith(label)); - if (!line) throw new Error(`no check named ${label}`); - return line.textContent ?? ""; -} - -describe("Create a Dot: before you create", () => { - /** The control plane's three rows and the host's three (the doctor's key row is not drawn). */ - const ROWS = 6; - - it("shows the checks of the control plane and of the host, all ready when it has a key", async () => { - await renderPage(); - const panel = await screen.findByRole("region", { name: "Before you create" }); - await waitFor(() => expect(within(panel).getAllByRole("listitem")).toHaveLength(ROWS)); - expect(checkLine(panel, "Control plane")).toBe("Control plane: ReadyAnswering, version 9.9.9."); - expect(checkLine(panel, "Database")).toBe("Database: ReadyAnswering."); - expect(checkLine(panel, "OpenRouter key")).toBe("OpenRouter key: ReadyStored."); - expect(checkLine(panel, "QEMU")).toBe("QEMU: Ready8.2.2"); - expect(checkLine(panel, "golden image")).toBe("golden image: Readygolden-1.qcow2 matches its manifest"); - }); - - it("says that no key is stored, and still lets the Dot be created", async () => { - plane.keyConfigured = false; - await renderPage(); - const panel = await screen.findByRole("region", { name: "Before you create" }); - await waitFor(() => expect(within(panel).getAllByRole("listitem")).toHaveLength(ROWS)); - expect(checkLine(panel, "OpenRouter key")).toMatch(/^OpenRouter key: Needs attention.*cannot answer until one is/); - expect(checkLine(panel, "Control plane")).toContain(": Ready"); - await fillIdentity(); - await userEvent.click(screen.getByRole("button", { name: "Create Dot" })); - await waitFor(() => expect(push).toHaveBeenCalled()); - }); - - it("names what the host lacks and the command that fixes it, and still lets the Dot be created", async () => { - plane.doctor = [ - { id: "qemu", label: "QEMU", status: "missing", detail: "not found on PATH", fix: "invisible-dots setup" }, - { id: "golden-image", label: "golden image", status: "missing", detail: "none in the images folder", fix: "invisible-dots image build" }, - ]; - await renderPage(); - const panel = await screen.findByRole("region", { name: "Before you create" }); - await waitFor(() => expect(within(panel).getAllByRole("listitem")).toHaveLength(5)); - expect(checkLine(panel, "QEMU")).toBe("QEMU: Needs attentionnot found on PATHFix: invisible-dots setup"); - expect(checkLine(panel, "golden image")).toContain("Fix: invisible-dots image build"); - expect(checkLine(panel, "Control plane")).toContain(": Ready"); - await fillIdentity(); - await userEvent.click(screen.getByRole("button", { name: "Create Dot" })); - await waitFor(() => expect(push).toHaveBeenCalled()); - }); - - it("says that the host was not checked when the report cannot be made, and keeps the control plane's rows", async () => { - plane.failDoctor = 500; - await renderPage(); - const panel = await screen.findByRole("region", { name: "Before you create" }); - await waitFor(() => expect(within(panel).getAllByRole("listitem")).toHaveLength(4)); - expect(checkLine(panel, "This computer")).toBe("This computer: Not checkedNot checked: the doctor could not run"); - expect(checkLine(panel, "Control plane")).toContain(": Ready"); - }); - - it("says when the control plane does not answer, and checks again on request", async () => { - plane.healthy = false; - plane.failDoctor = 503; - await renderPage(); - const panel = await screen.findByRole("region", { name: "Before you create" }); - await waitFor(() => expect(within(panel).getAllByRole("listitem")).toHaveLength(4)); - expect(checkLine(panel, "Control plane")).toContain(": Needs attention"); - expect(checkLine(panel, "Database")).toContain(": Not checked"); - expect(checkLine(panel, "This computer")).toContain(": Not checked"); - plane.healthy = true; - plane.failDoctor = null; - await userEvent.click(within(panel).getByRole("button", { name: "Check again" })); - await waitFor(() => expect(checkLine(panel, "Control plane")).toContain(": Ready")); - expect(checkLine(panel, "Database")).toContain(": Ready"); - expect(checkLine(panel, "QEMU")).toContain(": Ready"); - }); -}); - -describe("Create a Dot: the YAML editor", () => { - const yamlBox = () => screen.getByRole("textbox", { name: "Configuration (YAML)" }) as HTMLTextAreaElement; - - it("shows the form's config, and creates the Dot from the text as edited", async () => { - await renderPage(); - await fillIdentity(); - await userEvent.click(screen.getByRole("button", { name: "Advanced YAML" })); - expect(yamlBox().value).toContain("name: fare-watch"); - expect(yamlBox().value).toContain("cpu: 2"); - expect(screen.getByText("The configuration is valid.")).toBeTruthy(); - fireEvent.change(yamlBox(), { target: { value: yamlBox().value.replace("cpu: 2", "cpu: 6") } }); - await userEvent.click(screen.getByRole("button", { name: "Create Dot" })); - await waitFor(() => expect(plane.created).toHaveLength(1)); - expect(typeof plane.created[0]).toBe("string"); - expect(plane.created[0]).toContain("cpu: 6"); - expect(push).toHaveBeenCalledWith("/dots/created-1/chat"); - }); - - it("lists what is wrong with the YAML as it is typed, and refuses to create from it", async () => { - await renderPage(); - await userEvent.click(screen.getByRole("button", { name: "Advanced YAML" })); - // An empty form is already invalid: it has no name. - const problems = screen.getByRole("list", { name: "Problems in the YAML" }); - expect(within(problems).getAllByRole("listitem").length).toBeGreaterThanOrEqual(1); - await userEvent.click(screen.getByRole("button", { name: "Create Dot" })); - expect(plane.created).toEqual([]); - }); - - it("brings the edits back into the form", async () => { - await renderPage(); - await fillIdentity(); - await userEvent.click(screen.getByRole("button", { name: "Advanced YAML" })); - fireEvent.change(yamlBox(), { target: { value: yamlBox().value.replace("cpu: 2", "cpu: 6").replace('memory: "4gb"', 'memory: "12gb"') } }); - await userEvent.click(screen.getByRole("button", { name: "Form" })); - expect((screen.getByLabelText("Processors") as HTMLInputElement).value).toBe("6"); - expect((screen.getByLabelText("Memory") as HTMLInputElement).value).toBe("12"); - expect((screen.getByLabelText("Name") as HTMLInputElement).value).toBe("fare-watch"); - }); - - it("stays in the YAML, and says why, when the text sets what the form cannot show", async () => { - await renderPage(); - await fillIdentity(); - await userEvent.click(screen.getByRole("button", { name: "Advanced YAML" })); - fireEvent.change(yamlBox(), { target: { value: yamlBox().value.replace("limits:\n", "limits:\n max_steps_per_task: 10\n") } }); - await userEvent.click(screen.getByRole("button", { name: "Form" })); - expect(screen.getByText("The form cannot show this")).toBeTruthy(); - expect(screen.getByText(/no controls for/)).toBeTruthy(); - expect(yamlBox().value).toContain("max_steps_per_task: 10"); - // Editing the text again takes the message away. - fireEvent.change(yamlBox(), { target: { value: yamlBox().value.replace(" max_steps_per_task: 10\n", "") } }); - expect(screen.queryByText("The form cannot show this")).toBeNull(); - }); - - it("says that permissions set in the YAML match no preset", async () => { - await renderPage(); - await fillIdentity(); - await userEvent.click(screen.getByRole("button", { name: "Advanced YAML" })); - fireEvent.change(yamlBox(), { target: { value: yamlBox().value.replace("permissions: {}", "permissions:\n computer.exec: deny") } }); - await userEvent.click(screen.getByRole("button", { name: "Form" })); - expect(screen.getByText(/match none of these/)).toBeTruthy(); - for (const preset of ["Careful", "Balanced", "Autonomous"]) expect((screen.getByRole("radio", { name: new RegExp(preset) }) as HTMLInputElement).checked).toBe(false); - await userEvent.click(screen.getByRole("radio", { name: /Autonomous/ })); - expect(screen.queryByText(/match none of these/)).toBeNull(); - }); -}); diff --git a/apps/web/test/offline.test.ts b/apps/web/test/offline.test.ts deleted file mode 100644 index 12e6f5e9..00000000 --- a/apps/web/test/offline.test.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { offlineOf } from "../src/lib/offline"; - -describe("when the page can no longer hear the control plane", () => { - it("says nothing while the stream is up or starting and the health check answers", () => { - expect(offlineOf({ stream: "open", streamDetail: "", apiError: null })).toBeNull(); - expect(offlineOf({ stream: "connecting", streamDetail: "", apiError: null })).toBeNull(); - }); - - it("says the control plane does not answer while the stream reconnects, with why, and that the page connects by itself", () => { - const offline = offlineOf({ stream: "reconnecting", streamDetail: "fetch failed", apiError: null })!; - expect(offline.title).toBe("The control plane does not answer"); - expect(offline.detail).toContain("fetch failed"); - expect(offline.detail).toContain("out of date"); - expect(offline.detail).toContain("connects again by itself"); - }); - - it("says it too when only the health check fails, with what it said", () => { - const offline = offlineOf({ stream: "open", streamDetail: "", apiError: new Error("the control plane answered 503") })!; - expect(offline.title).toBe("The control plane does not answer"); - expect(offline.detail).toContain("the control plane answered 503"); - }); - - it("tells a stream that ended for good apart: the page does not connect again by itself", () => { - const offline = offlineOf({ stream: "closed", streamDetail: "the origin was refused (403)", apiError: null })!; - expect(offline.title).toBe("Live updates have stopped"); - expect(offline.detail).toContain("the origin was refused (403)"); - expect(offline.detail).toContain("reload"); - expect(offline.detail).not.toContain("by itself"); - }); -}); diff --git a/apps/web/test/parent.test.ts b/apps/web/test/parent.test.ts deleted file mode 100644 index d3ad79c7..00000000 --- a/apps/web/test/parent.test.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { exitWhenParentGone, parentPidFrom } from "../src/lib/parent.js"; - -beforeEach(() => { - vi.useFakeTimers(); -}); - -afterEach(() => { - vi.useRealTimers(); -}); - -describe("exitWhenParentGone", () => { - it("does nothing while the parent exists, and calls onGone once when it is gone", () => { - let alive = true; - const asked: number[] = []; - const onGone = vi.fn(); - exitWhenParentGone(4242, { - exists: (pid) => { - asked.push(pid); - return alive; - }, - onGone, - intervalMs: 1000, - }); - vi.advanceTimersByTime(3000); - expect(onGone).not.toHaveBeenCalled(); - expect(asked).toEqual([4242, 4242, 4242]); - - alive = false; - vi.advanceTimersByTime(5000); - expect(onGone).toHaveBeenCalledTimes(1); - expect(asked).toHaveLength(4); - }); - - it("stops looking when told to", () => { - const onGone = vi.fn(); - const stop = exitWhenParentGone(4242, { exists: () => false, onGone, intervalMs: 1000 }); - stop(); - vi.advanceTimersByTime(5000); - expect(onGone).not.toHaveBeenCalled(); - }); -}); - -describe("parentPidFrom", () => { - it("reads a positive integer and nothing else", () => { - expect(parentPidFrom("4242")).toBe(4242); - for (const bad of [undefined, "", "0", "-1", "1.5", "12abc", " 7", "1e3", "99999999999999999999"]) { - expect(parentPidFrom(bad)).toBeUndefined(); - } - }); -}); diff --git a/apps/web/test/permission-presets.test.ts b/apps/web/test/permission-presets.test.ts deleted file mode 100644 index 09455047..00000000 --- a/apps/web/test/permission-presets.test.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { PERMISSIONS, parseDotConfig, resolvePermission, toRuntimeConfig } from "@invisible-dots/shared"; -import { describe, expect, it } from "vitest"; -import { PRESET_IDS, PRESETS, presetOf, presetPermissions } from "../src/lib/permission-presets"; - -const BASE = { name: "p", model: { provider: "openrouter", id: "a/b" } }; - -function resolved(preset: (typeof PRESET_IDS)[number]) { - return toRuntimeConfig(parseDotConfig({ ...BASE, permissions: presetPermissions(preset) })).permissions; -} - -describe("the permission presets", () => { - it("are accepted by the schema, which refuses a permission it does not know", () => { - for (const id of PRESET_IDS) expect(() => parseDotConfig({ ...BASE, permissions: presetPermissions(id) })).not.toThrow(); - }); - - it("Balanced is the shipped defaults and writes nothing", () => { - expect(presetPermissions("balanced")).toEqual({}); - const defaults = parseDotConfig(BASE); - for (const permission of PERMISSIONS) expect(resolved("balanced")[permission]).toBe(resolvePermission(defaults, permission)); - }); - - it("Careful asks before commands and file changes and leaves every other permission at its default", () => { - const careful = resolved("careful"); - const defaults = resolved("balanced"); - expect(careful["computer.exec"]).toBe("ask"); - expect(careful["files.write"]).toBe("ask"); - for (const permission of PERMISSIONS) { - if (permission !== "computer.exec" && permission !== "files.write") expect(careful[permission]).toBe(defaults[permission]); - } - }); - - it("Autonomous allows everything except deleting a browser identity, which still asks", () => { - const autonomous = resolved("autonomous"); - for (const permission of PERMISSIONS) { - expect(autonomous[permission]).toBe(permission === "browser.identity.delete" ? "ask" : "allow"); - } - }); - - it("hand out copies, so editing one never changes the preset", () => { - const copy = presetPermissions("careful"); - copy["files.write"] = "deny"; - expect(PRESETS.careful.permissions["files.write"]).toBe("ask"); - }); -}); - -describe("presetOf", () => { - it("names the preset a permission map amounts to", () => { - for (const id of PRESET_IDS) expect(presetOf(presetPermissions(id))).toBe(id); - }); - - it("counts an explicit value equal to the default as the default", () => { - expect(presetOf({ "computer.exec": "allow", "browser.identity.delete": "ask" })).toBe("balanced"); - }); - - it("is null for permissions that match no preset", () => { - expect(presetOf({ "computer.exec": "deny" })).toBeNull(); - // Half of Careful is not Careful. - expect(presetOf({ "files.write": "ask" })).toBeNull(); - }); -}); diff --git a/apps/web/test/playwright-config.test.ts b/apps/web/test/playwright-config.test.ts deleted file mode 100644 index 8741b2e6..00000000 --- a/apps/web/test/playwright-config.test.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { readFileSync } from "node:fs"; -import { afterEach, describe, expect, it, vi } from "vitest"; - -afterEach(() => vi.unstubAllEnvs()); - -describe("the Playwright config", () => { - for (const ci of ["", "1"]) { - it(`writes the JUnit report that test-guard reads, ${ci ? "under CI" : "outside CI"}, so a local run never leaves a stale one`, async () => { - vi.stubEnv("CI", ci); - vi.resetModules(); - const { default: config } = await import("../playwright.config"); - expect(config.reporter).toContainEqual(["junit", { outputFile: "playwright-report.xml" }]); - }); - } -}); - -describe("the browser tests' harness", () => { - it("runs the web client as the product does: the standalone server through the CLI's own start, not `next start` with the whole environment", () => { - const harness = readFileSync(new URL("../e2e/harness.ts", import.meta.url), "utf8"); - expect(harness).toContain("startWebServer"); - expect(harness).toContain("locateWebBuild"); - expect(harness).not.toContain("next/dist/bin/next"); - expect(harness).not.toContain("...process.env, INVISIBLE_DOTS_HOME"); - }); -}); diff --git a/apps/web/test/preflight.test.ts b/apps/web/test/preflight.test.ts deleted file mode 100644 index 72807fbd..00000000 --- a/apps/web/test/preflight.test.ts +++ /dev/null @@ -1,72 +0,0 @@ -import type { DoctorCheck } from "@invisible-dots/shared/browser"; -import { describe, expect, it } from "vitest"; -import { isReady, notReadyCount, preflightItems } from "../src/lib/preflight"; - -const HEALTHY = { status: "ok", database: "ok", version: "1.2.3", openrouter_configured: true, database_kind: "pglite", data_dir: "/home/me/.invisible-dots", logs_dir: "/home/me/.invisible-dots/logs" } as const; -const row = (id: DoctorCheck["id"], status: DoctorCheck["status"], detail = "fine", fix?: string): DoctorCheck => ({ id, label: id, status, detail, ...(fix ? { fix } : {}) }); -const HOST_OK = { checks: [row("node", "ok"), row("qemu", "ok"), row("openrouter", "ok", "stored")] }; - -describe("preflightItems", () => { - it("is all ok when the control plane answers, holds a key and the host is ready", () => { - const items = preflightItems({ health: { health: HEALTHY }, host: HOST_OK }); - expect(items.map((item) => [item.id, item.state])).toEqual([["api", "ok"], ["database", "ok"], ["key", "ok"], ["node", "ok"], ["qemu", "ok"]]); - expect(items[0]!.detail).toContain("1.2.3"); - }); - - it("fails the key item, and only it, when no key is stored", () => { - const items = preflightItems({ health: { health: { ...HEALTHY, openrouter_configured: false } }, host: HOST_OK }); - expect(items.slice(0, 3).map((item) => item.state)).toEqual(["ok", "ok", "failed"]); - expect(items[2]!.detail).toMatch(/cannot answer until one is/); - }); - - it("says the key once: the doctor's own row of it is not shown", () => { - const items = preflightItems({ health: { health: HEALTHY }, host: HOST_OK }); - expect(items.map((item) => item.id)).not.toContain("openrouter"); - }); - - it("fails a host check that is missing or failed and carries the command that fixes it, as the doctor words it", () => { - const items = preflightItems({ - health: { health: HEALTHY }, - host: { checks: [row("qemu", "missing", "not found on PATH", "invisible-dots setup"), row("disk", "failed", "3.0 GiB free", "free some space"), row("node", "ok", "24.1.0", "ignored")] }, - }); - const host = items.slice(3); - expect(host.map((item) => [item.id, item.state, item.detail, item.fix])).toEqual([ - ["qemu", "failed", "not found on PATH", "invisible-dots setup"], - ["disk", "failed", "3.0 GiB free", "free some space"], - ["node", "ok", "24.1.0", undefined], - ]); - }); - - it("fails the control plane with the reason, and says the rest was not checked, when it does not answer", () => { - const items = preflightItems({ health: { error: "cannot reach the API" }, host: { error: "cannot reach the API" } }); - expect(items.map((item) => item.state)).toEqual(["failed", "unknown", "unknown", "unknown"]); - expect(items[0]!.detail).toBe("cannot reach the API"); - expect(items[3]).toMatchObject({ id: "host", detail: "Not checked: the control plane does not answer." }); - }); - - it("keeps the control plane's rows and says why the host report is missing when only that fails", () => { - const items = preflightItems({ health: { health: HEALTHY }, host: { error: "the doctor could not run" } }); - expect(items.slice(0, 3).every((item) => item.state === "ok")).toBe(true); - expect(items[3]).toMatchObject({ id: "host", state: "unknown", detail: "Not checked: the doctor could not run" }); - }); -}); - -describe("isReady and notReadyCount", () => { - it("is ready when every item is ok", () => { - const items = preflightItems({ health: { health: HEALTHY }, host: HOST_OK }); - expect(isReady(items)).toBe(true); - expect(notReadyCount(items)).toBe(0); - }); - - it("is not ready with a failed item, and counts it", () => { - const items = preflightItems({ health: { health: { ...HEALTHY, openrouter_configured: false } }, host: { checks: [row("qemu", "missing", "not found", "invisible-dots setup")] } }); - expect(isReady(items)).toBe(false); - expect(notReadyCount(items)).toBe(2); - }); - - it("does not count a check that was not made as ready", () => { - const items = preflightItems({ health: { error: "down" }, host: { error: "down" } }); - expect(isReady(items)).toBe(false); - expect(notReadyCount(items)).toBe(4); - }); -}); diff --git a/apps/web/test/proxy.test.ts b/apps/web/test/proxy.test.ts deleted file mode 100644 index c5dd02af..00000000 --- a/apps/web/test/proxy.test.ts +++ /dev/null @@ -1,266 +0,0 @@ -import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; -import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; -import type { AddressInfo } from "node:net"; -import { tmpdir } from "node:os"; -import { dirname, join } from "node:path"; -import { InvisibleDotsClient } from "@invisible-dots/sdk"; -import { hostPaths } from "@invisible-dots/shared/paths"; -import { afterAll, beforeAll, beforeEach, describe, expect, it } from "vitest"; -import { DELETE, GET, PATCH, POST, PUT } from "../src/app/api/[...path]/route"; -import { - ProxyError, - apiBaseUrl, - checkRequestOrigin, - forwardRequestHeaders, - forwardResponseHeaders, - loadApiToken, - upstreamUrl, -} from "../src/lib/proxy"; - -const TOKEN = "test-token-0123456789"; - -describe("upstreamUrl", () => { - it("maps proxy segments under /api and keeps the query", () => { - expect(upstreamUrl("http://h:1", ["dots", "a b", "tasks"], "?x=1")).toBe("http://h:1/api/dots/a%20b/tasks?x=1"); - }); - - it("re-encodes a decoded slash so it cannot add a path level", () => { - expect(upstreamUrl("http://h:1", ["dots", "a/b"], "")).toBe("http://h:1/api/dots/a%2Fb"); - }); - - it("refuses dot segments and empty paths", () => { - expect(() => upstreamUrl("http://h:1", ["..", "health"], "")).toThrow(ProxyError); - expect(() => upstreamUrl("http://h:1", ["dots", "."], "")).toThrow(ProxyError); - expect(() => upstreamUrl("http://h:1", [], "")).toThrow(ProxyError); - }); -}); - -describe("apiBaseUrl", () => { - it("defaults to the contract's listen address and trims a trailing slash", () => { - expect(apiBaseUrl({})).toBe("http://127.0.0.1:8787"); - expect(apiBaseUrl({ INVISIBLE_DOTS_URL: "http://10.0.0.2:9000/" })).toBe("http://10.0.0.2:9000"); - }); -}); - -describe("loadApiToken", () => { - let home: string; - beforeEach(async () => { - home = await mkdtemp(join(tmpdir(), "idots-web-token-")); - }); - afterAll(async () => { - await rm(home, { recursive: true, force: true }); - }); - const writeToken = async (text: string) => { - const path = hostPaths({ INVISIBLE_DOTS_HOME: home }).apiTokenPath; - await mkdir(dirname(path), { recursive: true }); - await writeFile(path, text); - }; - - it("prefers INVISIBLE_DOTS_TOKEN", async () => { - expect(await loadApiToken({ INVISIBLE_DOTS_TOKEN: ` ${TOKEN} \n`, INVISIBLE_DOTS_HOME: home })).toBe(TOKEN); - }); - - it("reads the first line of api.token in INVISIBLE_DOTS_HOME, as the server and the command do", async () => { - // A note or an old token on a later line is ignored by every reader, not only some. - await writeToken(`${TOKEN}\nthe old one was rotated on Monday\n`); - expect(await loadApiToken({ INVISIBLE_DOTS_HOME: home })).toBe(TOKEN); - }); - - it("explains a missing, empty or too short token", async () => { - await expect(loadApiToken({ INVISIBLE_DOTS_HOME: home })).rejects.toMatchObject({ status: 500, code: "web_token_missing" }); - await writeToken(" \n"); - await expect(loadApiToken({ INVISIBLE_DOTS_HOME: home })).rejects.toThrow(/is empty/); - await writeToken("short\n"); - await expect(loadApiToken({ INVISIBLE_DOTS_HOME: home })).rejects.toThrow(/shorter than 16/); - }); -}); - -describe("checkRequestOrigin", () => { - const base = { method: "GET", host: "127.0.0.1:3000", origin: null, secFetchSite: null }; - - it("lets loopback hosts through", () => { - expect(checkRequestOrigin(base, {})).toBeNull(); - expect(checkRequestOrigin({ ...base, host: "localhost:3000" }, {})).toBeNull(); - expect(checkRequestOrigin({ ...base, host: "[::1]:3000" }, {})).toBeNull(); - }); - - it("refuses other host names unless allowed, which stops DNS rebinding", () => { - expect(checkRequestOrigin({ ...base, host: "evil.example:3000" }, {})).toMatch(/not allowed/); - expect( - checkRequestOrigin({ ...base, host: "dots.lan:3000" }, { INVISIBLE_DOTS_WEB_ALLOWED_HOSTS: "other, dots.lan" }), - ).toBeNull(); - }); - - it("refuses cross-site requests and foreign origins on writes", () => { - expect(checkRequestOrigin({ ...base, secFetchSite: "cross-site" }, {})).toMatch(/cross-site/); - expect(checkRequestOrigin({ ...base, method: "POST", origin: "http://evil.example" }, {})).toMatch(/does not match/); - expect(checkRequestOrigin({ ...base, method: "POST", origin: "http://127.0.0.1:3000" }, {})).toBeNull(); - }); -}); - -describe("header forwarding", () => { - it("drops browser credentials and adds the bearer token", () => { - const out = forwardRequestHeaders( - new Headers({ cookie: "a=b", authorization: "Bearer stolen", "content-type": "application/json", "last-event-id": "9" }), - "tok", - ); - expect(out.get("cookie")).toBeNull(); - expect(out.get("authorization")).toBe("Bearer tok"); - expect(out.get("last-event-id")).toBe("9"); - }); - - it("passes on how a Dot's file is served, so the browser neither sniffs nor runs it", () => { - const out = forwardResponseHeaders( - new Headers({ - "content-type": "text/plain; charset=utf-8", - "content-disposition": 'inline; filename="a.html"', - "x-content-type-options": "nosniff", - "content-security-policy": "default-src 'none'; sandbox", - "set-cookie": "x=1", - }), - ); - expect(out.get("content-type")).toBe("text/plain; charset=utf-8"); - expect(out.get("content-disposition")).toBe('inline; filename="a.html"'); - expect(out.get("x-content-type-options")).toBe("nosniff"); - expect(out.get("content-security-policy")).toBe("default-src 'none'; sandbox"); - expect(out.get("set-cookie")).toBeNull(); - }); - - it("marks event streams as not transformable", () => { - const out = forwardResponseHeaders(new Headers({ "content-type": "text/event-stream", "content-length": "5" })); - expect(out.get("cache-control")).toBe("no-cache, no-transform"); - expect(out.get("content-length")).toBeNull(); - }); -}); - -describe("proxy route against a fake API server", () => { - let server: Server; - let base: string; - const seen: { method: string; url: string; auth: string | undefined; body: string }[] = []; - const saved = { url: process.env.INVISIBLE_DOTS_URL, token: process.env.INVISIBLE_DOTS_TOKEN }; - - beforeAll(async () => { - server = createServer((req: IncomingMessage, res: ServerResponse) => { - let body = ""; - req.on("data", (chunk: Buffer) => (body += chunk.toString("utf8"))); - req.on("end", () => { - seen.push({ method: req.method ?? "", url: req.url ?? "", auth: req.headers.authorization, body }); - if (req.url === "/api/stream?dot_id=d1") { - res.writeHead(200, { "content-type": "text/event-stream" }); - res.write('id: 1\ndata: {"id":1,"type":"dot.created","data":{}}\n\n'); - res.end(); - return; - } - if (req.url === "/api/dots/d1/browser-identities") { - res.writeHead(409, { "content-type": "application/json" }); - res.end(JSON.stringify({ error: "computer_stopped", message: "the computer is stopped" })); - return; - } - if (req.method === "DELETE") { - res.writeHead(204); - res.end(); - return; - } - res.writeHead(200, { "content-type": "application/json" }); - res.end(JSON.stringify({ ok: true })); - }); - }); - await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); - base = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; - process.env.INVISIBLE_DOTS_URL = base; - process.env.INVISIBLE_DOTS_TOKEN = TOKEN; - }); - - afterAll(async () => { - process.env.INVISIBLE_DOTS_URL = saved.url; - process.env.INVISIBLE_DOTS_TOKEN = saved.token; - if (saved.url === undefined) delete process.env.INVISIBLE_DOTS_URL; - if (saved.token === undefined) delete process.env.INVISIBLE_DOTS_TOKEN; - await new Promise((resolve) => server.close(() => resolve())); - }); - - beforeEach(() => { - seen.length = 0; - }); - - const context = (...path: string[]) => ({ params: Promise.resolve({ path }) }); - const local = (path: string, init?: RequestInit) => - new Request(`http://127.0.0.1:3000/api/${path}`, { - ...init, - headers: { host: "127.0.0.1:3000", ...(init?.headers as Record | undefined) }, - }); - - it("forwards a POST with the token and the JSON body", async () => { - const response = await POST( - local("dots", { method: "POST", body: '{"config":"name: x"}', headers: { "content-type": "application/json" } }), - context("dots"), - ); - expect(response.status).toBe(200); - expect(await response.json()).toEqual({ ok: true }); - expect(seen).toEqual([{ method: "POST", url: "/api/dots", auth: `Bearer ${TOKEN}`, body: '{"config":"name: x"}' }]); - }); - - it("passes API errors through unchanged", async () => { - const response = await GET(local("dots/d1/browser-identities"), context("dots", "d1", "browser-identities")); - expect(response.status).toBe(409); - expect(await response.json()).toEqual({ error: "computer_stopped", message: "the computer is stopped" }); - }); - - it("streams server-sent events", async () => { - const response = await GET(local("stream?dot_id=d1"), context("stream")); - expect(response.headers.get("content-type")).toBe("text/event-stream"); - expect(response.headers.get("cache-control")).toBe("no-cache, no-transform"); - expect(await response.text()).toContain('"type":"dot.created"'); - }); - - it("answers 204 without a body", async () => { - const response = await DELETE(local("dots/d1", { method: "DELETE" }), context("dots", "d1")); - expect(response.status).toBe(204); - expect(seen[0]).toMatchObject({ method: "DELETE", url: "/api/dots/d1" }); - }); - - it("refuses a foreign origin before contacting the API", async () => { - const response = await POST( - local("dots", { method: "POST", body: "{}", headers: { origin: "http://evil.example" } }), - context("dots"), - ); - expect(response.status).toBe(403); - expect(seen).toEqual([]); - }); - - it("serves the SDK unchanged: same-origin /api paths reach the API with the token", async () => { - const handlers: Record = { GET, POST, DELETE }; - const client = new InvisibleDotsClient({ - baseUrl: "", - fetch: async (input, init) => { - const url = new URL(String(input), "http://127.0.0.1:3000"); - const segments = url.pathname.replace(/^\/api\//, "").split("/").map(decodeURIComponent); - const headers = new Headers(init?.headers); - headers.set("host", "127.0.0.1:3000"); - const request = new Request(url, { ...init, headers }); - const handler = handlers[request.method]; - if (!handler) throw new Error(`no handler for ${request.method}`); - return handler(request, context(...segments)); - }, - }); - await client.deleteDot("d1"); - await expect(client.listIdentities("d1")).rejects.toMatchObject({ status: 409, code: "computer_stopped" }); - expect(seen.map((s) => [s.method, s.url, s.auth])).toEqual([ - ["DELETE", "/api/dots/d1", `Bearer ${TOKEN}`], - ["GET", "/api/dots/d1/browser-identities", `Bearer ${TOKEN}`], - ]); - }); - - it("answers 502 with a clear message when the API is down", async () => { - process.env.INVISIBLE_DOTS_URL = "http://127.0.0.1:1"; - try { - const response = await GET(local("health"), context("health")); - expect(response.status).toBe(502); - const body = (await response.json()) as { error: string; message: string }; - expect(body.error).toBe("api_unreachable"); - expect(body.message).toContain("http://127.0.0.1:1"); - } finally { - process.env.INVISIBLE_DOTS_URL = base; - } - }); -}); diff --git a/apps/web/test/screen-view.test.tsx b/apps/web/test/screen-view.test.tsx deleted file mode 100644 index 4d885662..00000000 --- a/apps/web/test/screen-view.test.tsx +++ /dev/null @@ -1,32 +0,0 @@ -// @vitest-environment jsdom -import { cleanup, render, screen } from "@testing-library/react"; -import { afterEach, describe, expect, it } from "vitest"; -import { ScreenView } from "../src/components/computer/screen-view"; -import { contrast, fromHex } from "./support/contrast"; -import { themes } from "./support/tokens"; - -afterEach(cleanup); - -/** The tokens a badge is painted with: its `bg-` and `text-` classes, which must be color tokens of both themes. */ -function badgeTokens(badge: HTMLElement): { background: string; text: string } { - const classes = [...badge.classList]; - const named = (prefix: string) => classes.find((c) => c.startsWith(prefix) && c.slice(prefix.length) in themes.light); - const background = named("bg-"); - const text = named("text-"); - if (!background || !text) throw new Error(`the badge is not painted with tokens: ${badge.className}`); - return { background: background.slice(3), text: text.slice(5) }; -} - -describe("the badges over the screen", () => { - const badges: [string, () => HTMLElement][] = [ - ["LIVE", () => screen.getByText("LIVE")], - ["the stale warning", () => screen.getByRole("status")], - ]; - - it.each(badges)("%s is painted with two tokens that read at AA (4.5:1) in the light and the dark theme", (name, find) => { - const now = Date.parse("2026-01-01T00:00:10Z"); - render(); - const { background, text } = badgeTokens(find()); - for (const theme of Object.values(themes)) expect(contrast(fromHex(theme[text]!), fromHex(theme[background]!))).toBeGreaterThanOrEqual(4.5); - }); -}); diff --git a/apps/web/test/settings.test.tsx b/apps/web/test/settings.test.tsx deleted file mode 100644 index 611dc5e3..00000000 --- a/apps/web/test/settings.test.tsx +++ /dev/null @@ -1,209 +0,0 @@ -// @vitest-environment jsdom -import { act, cleanup, fireEvent, render, screen, waitFor, within } from "@testing-library/react"; -import userEvent from "@testing-library/user-event"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { EventStreamProvider } from "../src/components/events"; -import { HostSettings } from "../src/components/settings/HostSettings"; -import { AttentionProvider, useHealth } from "../src/components/shell/attention"; -import { THEME_KEY } from "../src/lib/theme"; -import { stubMatchMedia } from "./support/browser"; -import { FakeControlPlane } from "./support/control-plane"; - -vi.mock("next/navigation", () => ({ usePathname: () => "/settings", useRouter: () => ({ push() {}, replace() {} }) })); - -let plane: FakeControlPlane; - -beforeEach(() => { - plane = new FakeControlPlane(); - plane.install(); - stubMatchMedia(); - document.documentElement.removeAttribute("data-theme"); - localStorage.clear(); -}); - -afterEach(() => { - cleanup(); - vi.unstubAllGlobals(); - vi.restoreAllMocks(); -}); - -/** What the shell's rail reads from the same health answer, to see that the page and the rail agree. */ -function RailKey() { - const health = useHealth(); - return

{health.data ? String(health.data.openrouter_configured) : "unknown"}

; -} - -async function renderSettings() { - render( - - - - - - , - ); - await screen.findByRole("heading", { name: "Settings" }); -} - -const section = (name: string) => screen.getByRole("region", { name }); - -/** The line of the checks for one item, as a screen reader hears it. */ -function checkLine(label: string): string { - const line = within(section("Host checks")).getAllByRole("listitem").find((item) => item.textContent?.startsWith(label)); - if (!line) throw new Error(`no check named ${label}`); - return line.textContent ?? ""; -} - -describe("Settings: host checks", () => { - it("lists the checks of the control plane and of the host, and says when everything is in place", async () => { - await renderSettings(); - await waitFor(() => expect(within(section("Host checks")).getAllByRole("listitem")).toHaveLength(6)); - expect(checkLine("Control plane")).toBe("Control plane: ReadyAnswering, version 9.9.9."); - expect(checkLine("QEMU")).toBe("QEMU: Ready8.2.2"); - expect(within(section("Host checks")).getByText("Everything a Dot needs is in place.")).toBeTruthy(); - }); - - it("shows what is missing with the command that fixes it, a button that copies it, and how many things need attention", async () => { - plane.doctor = [ - { id: "qemu", label: "QEMU", status: "missing", detail: "not found on PATH", fix: "invisible-dots setup" }, - { id: "golden-image", label: "golden image", status: "missing", detail: "none in the images folder", fix: "invisible-dots image build" }, - ]; - const writeText = vi.fn(async () => {}); - vi.stubGlobal("navigator", { clipboard: { writeText } }); - await renderSettings(); - await waitFor(() => expect(within(section("Host checks")).getAllByRole("listitem")).toHaveLength(5)); - expect(checkLine("QEMU")).toBe("QEMU: Needs attentionnot found on PATHFix: invisible-dots setup"); - expect(within(section("Host checks")).getByText(/2 things need attention/)).toBeTruthy(); - - await userEvent.click(within(section("Host checks")).getByRole("button", { name: "Copy the fix for golden image" })); - expect(writeText).toHaveBeenCalledWith("invisible-dots image build"); - expect(await within(section("Host checks")).findByRole("button", { name: "Copied" })).toBeTruthy(); - }); - - it("checks again on request, and by itself when the person comes back to the window while something is missing", async () => { - plane.doctor = [{ id: "qemu", label: "QEMU", status: "missing", detail: "not found on PATH", fix: "invisible-dots setup" }]; - await renderSettings(); - await waitFor(() => expect(checkLine("QEMU")).toContain("Needs attention")); - const asked = () => plane.requests.filter((request) => request === "GET /api/doctor").length; - expect(asked()).toBe(1); - - // The person ran the command in a terminal and returns to this window. - plane.doctor = [{ id: "qemu", label: "QEMU", status: "ok", detail: "8.2.2" }]; - act(() => { - fireEvent.focus(window); - }); - await waitFor(() => expect(checkLine("QEMU")).toBe("QEMU: Ready8.2.2")); - expect(asked()).toBe(2); - - // Nothing is missing any more, so coming back asks nothing. - act(() => { - fireEvent.focus(window); - }); - await new Promise((resolve) => setTimeout(resolve, 50)); - expect(asked()).toBe(2); - - await userEvent.click(within(section("Host checks")).getByRole("button", { name: "Check again" })); - await waitFor(() => expect(asked()).toBe(3)); - }); - - it("says that the host was not checked when the report cannot be made", async () => { - plane.failDoctor = 500; - await renderSettings(); - await waitFor(() => expect(checkLine("This computer")).toBe("This computer: Not checkedNot checked: the doctor could not run")); - }); -}); - -describe("Settings: the OpenRouter key", () => { - const keyField = () => screen.getByLabelText(/^(OpenRouter API key|Replace the key)$/) as HTMLInputElement; - - it("is a write-only field, and says whether a key is stored from the health answer", async () => { - plane.keyConfigured = false; - await renderSettings(); - await waitFor(() => expect(within(section("OpenRouter key")).getByText("No key is stored.")).toBeTruthy()); - expect(keyField().type).toBe("password"); - expect(keyField().autocomplete).toBe("off"); - expect((screen.getByRole("button", { name: "Save key" }) as HTMLButtonElement).disabled).toBe(true); - }); - - it("saves the key, says how many running Dots got it, empties the field and tells the rest of the page", async () => { - plane.keyConfigured = false; - plane.keyPushedTo = 2; - await renderSettings(); - await waitFor(() => expect(screen.getByTestId("rail-key").textContent).toBe("false")); - await userEvent.type(keyField(), " sk-or-v1-abcdef "); - await userEvent.click(screen.getByRole("button", { name: "Save key" })); - - expect(await screen.findByText("Saved. Pushed to 2 running Dots.")).toBeTruthy(); - expect(plane.savedKeys).toEqual(["sk-or-v1-abcdef"]); - // The field is emptied and the key is nowhere on the page: it is write-only. - expect(keyField().value).toBe(""); - expect(document.body.textContent).not.toContain("sk-or-v1-abcdef"); - // The shared health answer was asked again, so the rail, the checklist and this form agree at once. - await waitFor(() => expect(screen.getByTestId("rail-key").textContent).toBe("true")); - await waitFor(() => expect(within(section("OpenRouter key")).getByText("A key is stored.")).toBeTruthy()); - await waitFor(() => expect(checkLine("OpenRouter key")).toBe("OpenRouter key: ReadyStored.")); - expect(screen.getByRole("button", { name: "Replace key" })).toBeTruthy(); - }); - - it("says nothing was pushed when no computer is running", async () => { - plane.keyConfigured = false; - await renderSettings(); - await userEvent.type(keyField(), "sk-or-first"); - await userEvent.click(screen.getByRole("button", { name: "Save key" })); - expect(await screen.findByText("Saved. No Dot's computer is running, so each gets the key when it starts.")).toBeTruthy(); - }); - - it("refuses a key with a space inside before it is sent", async () => { - await renderSettings(); - await userEvent.type(keyField(), "sk or key"); - await userEvent.click(screen.getByRole("button", { name: "Replace key" })); - expect(await screen.findByText(/must be printable ASCII without spaces/)).toBeTruthy(); - expect(keyField().getAttribute("aria-invalid")).toBe("true"); - expect(plane.savedKeys).toEqual([]); - // Typing again clears the complaint. - await userEvent.type(keyField(), "x"); - expect(keyField().getAttribute("aria-invalid")).toBeNull(); - }); - - it("shows the control plane's refusal and keeps what was typed", async () => { - plane.failKey = { status: 503, error: "database_unavailable", message: "the database did not answer" }; - await renderSettings(); - await userEvent.type(keyField(), "sk-or-keep"); - await userEvent.click(screen.getByRole("button", { name: "Replace key" })); - const alert = await screen.findByRole("alert"); - expect(alert.textContent).toContain("Could not save the key"); - expect(alert.textContent).toContain("the database did not answer"); - expect(keyField().value).toBe("sk-or-keep"); - expect(screen.queryByText(/^Saved\./)).toBeNull(); - }); -}); - -describe("Settings: appearance and about", () => { - it("chooses the theme, applies it at once and keeps it in this browser", async () => { - await renderSettings(); - expect((screen.getByRole("radio", { name: /System/ }) as HTMLInputElement).checked).toBe(true); - await userEvent.click(screen.getByRole("radio", { name: /Dark/ })); - expect(document.documentElement.dataset.theme).toBe("dark"); - expect(localStorage.getItem(THEME_KEY)).toBe("dark"); - expect((screen.getByRole("radio", { name: /Dark/ }) as HTMLInputElement).checked).toBe(true); - await userEvent.click(screen.getByRole("radio", { name: /Light/ })); - expect(document.documentElement.dataset.theme).toBe("light"); - }); - - it("lists the version, the database and where the data and the logs are", async () => { - await renderSettings(); - const about = await screen.findByRole("region", { name: "About" }); - await waitFor(() => expect(within(about).getByText("9.9.9")).toBeTruthy()); - expect(within(about).getByText("Embedded PostgreSQL (PGlite), inside the server")).toBeTruthy(); - expect(within(about).getByText("/home/me/.invisible-dots")).toBeTruthy(); - expect(within(about).getByText("/home/me/.invisible-dots/logs")).toBeTruthy(); - expect(within(about).getByText("invisible-dots logs ")).toBeTruthy(); - }); - - it("says when the control plane does not answer", async () => { - plane.healthy = false; - await renderSettings(); - const about = await screen.findByRole("region", { name: "About" }); - expect((await within(about).findByRole("alert")).textContent).toContain("The control plane does not answer"); - }); -}); diff --git a/apps/web/test/shell.test.tsx b/apps/web/test/shell.test.tsx deleted file mode 100644 index 46fbc9a4..00000000 --- a/apps/web/test/shell.test.tsx +++ /dev/null @@ -1,319 +0,0 @@ -// @vitest-environment jsdom -import { act, cleanup, render, screen, waitFor, within } from "@testing-library/react"; -import userEvent from "@testing-library/user-event"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import AppLayout from "../src/app/(app)/layout"; -import { FAVICON_ATTENTION } from "../src/lib/favicon"; -import { stubMatchMedia, stubResizeObserver } from "./support/browser"; -import { approvalRecord, channelRecord, dotRecord, FakeControlPlane, taskRecord } from "./support/control-plane"; - -let pathname = "/"; -vi.mock("next/navigation", () => ({ usePathname: () => pathname, useRouter: () => ({ push() {}, replace() {} }) })); - -let plane: FakeControlPlane; - -beforeEach(() => { - pathname = "/"; - window.localStorage.clear(); - document.title = "Dots - invisible_dots"; - plane = new FakeControlPlane(); - plane.install(); - stubMatchMedia(); - stubResizeObserver(); -}); - -afterEach(() => { - cleanup(); - vi.unstubAllGlobals(); -}); - -function renderShell() { - return render( - -

content

-
, - ); -} - -const rail = () => screen.getByRole("complementary", { name: "Navigation" }); - -describe("the rail", () => { - it("lists every Dot with the ring that says what it is doing, and the approvals that wait", async () => { - plane.dots = [ - dotRecord("quiet", { name: "quiet" }), - dotRecord("stopped", { name: "stopped", computer_state: "STOPPED" }), - dotRecord("broken", { name: "broken", status: "ERROR", error: "the disk is full", computer_state: "ERROR" }), - dotRecord("asking", { name: "asking", status: "WAITING_APPROVAL" }), - ]; - plane.approvals = [approvalRecord("a1", "asking"), approvalRecord("a2", "asking"), approvalRecord("a3", "quiet", { status: "approved" })]; - renderShell(); - - const dots = within(await screen.findByRole("navigation", { name: "Dots" })); - await waitFor(() => expect(dots.getAllByRole("link", { name: (name) => name !== "New Dot" })).toHaveLength(4)); - const ringOf = (name: string) => dots.getByRole("link", { name: new RegExp(name) }).querySelector("[data-ring]")?.getAttribute("data-ring"); - expect(ringOf("quiet")).toBe("ready"); - expect(ringOf("stopped")).toBe("stopped"); - expect(ringOf("broken")).toBe("error"); - expect(ringOf("asking")).toBe("waiting"); - expect(within(dots.getByRole("link", { name: /asking/ })).getByLabelText("2 waiting")).toBeTruthy(); - expect(within(dots.getByRole("link", { name: /quiet/ })).queryByLabelText(/waiting$/)).toBeNull(); - expect(dots.getByRole("link", { name: /asking/ }).getAttribute("href")).toBe("/dots/asking/chat"); - // The Inbox entry carries everything that needs the person: the two approvals and the Dot in ERROR. - expect(within(screen.getByRole("link", { name: /^Inbox/ })).getByLabelText("3 need you")).toBeTruthy(); - expect(screen.getByRole("link", { name: /^Inbox/ }).getAttribute("href")).toBe("/inbox"); - expect(screen.queryByRole("link", { name: /^Approvals/ })).toBeNull(); - }); - - it("marks the page that is open, the Dot's page included", async () => { - plane.dots = [dotRecord("one"), dotRecord("two")]; - pathname = "/dots/two/tasks"; - renderShell(); - const dots = within(await screen.findByRole("navigation", { name: "Dots" })); - await waitFor(() => expect(dots.getAllByRole("link", { name: (name) => name !== "New Dot" })).toHaveLength(2)); - expect(dots.getByRole("link", { name: /two/ }).getAttribute("aria-current")).toBe("page"); - expect(dots.getByRole("link", { name: /one/ }).getAttribute("aria-current")).toBeNull(); - expect(screen.getByRole("link", { name: "Home" }).getAttribute("aria-current")).toBeNull(); - }); - - it("says so when there are no Dots, and when the list cannot be loaded, and offers a retry", async () => { - renderShell(); - expect(await screen.findByText("No Dots yet.")).toBeTruthy(); - cleanup(); - - plane.dots = []; - const failing = vi.fn(() => Promise.resolve(Response.json({ error: "down", message: "down" }, { status: 503 }))); - vi.stubGlobal("fetch", failing); - renderShell(); - const alert = await screen.findByText(/Could not load the Dots/); - expect(alert).toBeTruthy(); - const calls = failing.mock.calls.length; - await userEvent.click(screen.getByRole("button", { name: "Retry" })); - await waitFor(() => expect(failing.mock.calls.length).toBeGreaterThan(calls)); - }); - - it("shows the API's state and version, and says that no key is stored yet", async () => { - plane.keyConfigured = false; - renderShell(); - await waitFor(() => expect(within(rail()).getByText("API: ok")).toBeTruthy()); - expect(within(rail()).getByText("v9.9.9")).toBeTruthy(); - expect(within(rail()).getByText("No OpenRouter key stored yet")).toBeTruthy(); - }); - - it("sends the person from the missing key to the settings page where it is entered", async () => { - plane.keyConfigured = false; - renderShell(); - const warning = await within(rail()).findByRole("link", { name: "No OpenRouter key stored yet" }); - expect(warning.getAttribute("href")).toBe("/settings"); - }); - - it("has a Settings page in the main navigation, marked as the current page while it is open", async () => { - pathname = "/settings"; - renderShell(); - const link = await within(rail()).findByRole("link", { name: "Settings" }); - expect(link.getAttribute("href")).toBe("/settings"); - expect(link.getAttribute("aria-current")).toBe("page"); - expect(within(rail()).getByRole("link", { name: "Home" }).getAttribute("aria-current")).toBeNull(); - }); - - it("learns that a key was stored from the next health answer, without a reload of the page", async () => { - plane.keyConfigured = false; - // Faked before the shell starts, so that its 30 second timer is the faked one. The shell asks every 30 seconds; - // a saved key asks at once through the shared resource (settings.test.tsx). - vi.useFakeTimers({ shouldAdvanceTime: true }); - try { - renderShell(); - await within(rail()).findByText("No OpenRouter key stored yet"); - plane.keyConfigured = true; - await act(async () => { - await vi.advanceTimersByTimeAsync(30_000); - }); - await waitFor(() => expect(within(rail()).queryByText("No OpenRouter key stored yet")).toBeNull()); - } finally { - vi.useRealTimers(); - } - }); - - it("says when the API cannot be reached", async () => { - plane.healthy = false; - renderShell(); - await waitFor(() => expect(within(rail()).getByText("API: unreachable")).toBeTruthy()); - }); - - it("says at the top of the page, as an alert, that the control plane does not answer, and takes it back when it does", async () => { - renderShell(); - await waitFor(() => expect(within(rail()).getByText("Live")).toBeTruthy()); - expect(screen.queryByRole("alert")).toBeNull(); - - plane.streamDown = true; - act(() => plane.dropStream()); - const alert = await screen.findByRole("alert", undefined, { timeout: 5000 }); - expect(within(alert).getByText("The control plane does not answer")).toBeTruthy(); - // First that the stream ended, then what the next attempt was answered with. - await waitFor(() => expect(screen.getByRole("alert").textContent).toContain("the control plane is down"), { timeout: 5000 }); - // It is above the page, not in the rail, so it shows under 768 px too. - expect(rail().contains(alert)).toBe(false); - expect(within(rail()).getByText("Reconnecting")).toBeTruthy(); - - plane.streamDown = false; - await waitFor(() => expect(screen.queryByRole("alert")).toBeNull(), { timeout: 8000 }); - expect(within(rail()).getByText("Live")).toBeTruthy(); - }, 20_000); - - it("says it when the health check fails, though the stream is up", async () => { - plane.healthy = false; - renderShell(); - const alert = await screen.findByRole("alert"); - expect(within(alert).getByText("The control plane does not answer")).toBeTruthy(); - }); - - it("shows whether live updates arrive", async () => { - renderShell(); - await waitFor(() => expect(within(rail()).getByText("Live")).toBeTruthy()); - }); - - it("offers a way to create a Dot, which is the create page", async () => { - renderShell(); - expect(screen.getByRole("link", { name: "New Dot" }).getAttribute("href")).toBe("/new"); - }); - -}); - -describe("what the live stream changes in the rail", () => { - it("counts a new approval at once, and drops it when it is answered", async () => { - plane.dots = [dotRecord("d1")]; - renderShell(); - await waitFor(() => expect(plane.streamOpen).toBe(true)); - await screen.findByRole("link", { name: /d1/ }); - expect(screen.queryByLabelText("1 waiting")).toBeNull(); - - plane.approvals = [approvalRecord("a1", "d1")]; - act(() => plane.push("d1", "approval.requested", { approval_id: "a1" })); - const dots = within(screen.getByRole("navigation", { name: "Dots" })); - await waitFor(() => expect(within(dots.getByRole("link", { name: /d1/ })).getByLabelText("1 waiting")).toBeTruthy()); - expect(within(dots.getByRole("link", { name: /d1/ })).getByRole("img", { name: "Waiting for you" })).toBeTruthy(); - - plane.approvals = []; - act(() => plane.push("d1", "approval.resolved", { approval_id: "a1", decision: "approve" })); - await waitFor(() => expect(screen.queryByLabelText("1 waiting")).toBeNull()); - }); - - it("shows the ring of a Dot that is working from the agent's own events", async () => { - plane.dots = [dotRecord("d1")]; - renderShell(); - await waitFor(() => expect(plane.streamOpen).toBe(true)); - const link = await screen.findByRole("link", { name: /d1/ }); - act(() => plane.push("d1", "agent.state", { state: "THINKING" })); - await waitFor(() => expect(link.querySelector("[data-ring]")?.getAttribute("data-ring")).toBe("working")); - act(() => plane.push("d1", "agent.state", { state: "IDLE" })); - await waitFor(() => expect(link.querySelector("[data-ring]")?.getAttribute("data-ring")).toBe("ready")); - }); - - it("marks a reply that arrives while another page is open, and clears the mark when the Dot is opened", async () => { - plane.dots = [dotRecord("d1"), dotRecord("d2")]; - const view = renderShell(); - await waitFor(() => expect(plane.streamOpen).toBe(true)); - await screen.findByRole("link", { name: /d1/ }); - act(() => plane.push("d1", "message.assistant", { text: "done" })); - await waitFor(() => expect(screen.getByRole("img", { name: "New reply" })).toBeTruthy()); - expect(within(screen.getByRole("link", { name: /d2/ })).queryByRole("img", { name: "New reply" })).toBeNull(); - - pathname = "/dots/d1/chat"; - view.rerender( - -

content

-
, - ); - await waitFor(() => expect(screen.queryByRole("img", { name: "New reply" })).toBeNull()); - }); - - it("puts the count of what needs the person in the tab title, and takes it out again", async () => { - plane.dots = [dotRecord("d1"), dotRecord("d2", { status: "ERROR", error: "x", computer_state: "ERROR" })]; - plane.approvals = [approvalRecord("a1", "d1")]; - renderShell(); - await waitFor(() => expect(document.title).toBe("(2) Dots - invisible_dots")); - expect(document.head.querySelector('link[rel="icon"]')?.getAttribute("href")).toContain(encodeURIComponent(FAVICON_ATTENTION)); - - plane.approvals = []; - plane.dots = [dotRecord("d1"), dotRecord("d2")]; - await waitFor(() => expect(plane.streamOpen).toBe(true)); - act(() => { - plane.push("d1", "approval.resolved", { approval_id: "a1", decision: "approve" }); - plane.push("d2", "dot.updated"); - }); - await waitFor(() => expect(document.title).toBe("Dots - invisible_dots")); - expect(document.head.querySelector('link[rel="icon"]')?.getAttribute("href")).not.toContain(encodeURIComponent(FAVICON_ATTENTION)); - }); - - it("counts a task that failed in the last day, and not one that failed earlier or one that was dismissed", async () => { - plane.dots = [dotRecord("d1")]; - const hoursAgo = (hours: number) => new Date(Date.now() - hours * 3_600_000).toISOString(); - plane.tasks = [ - taskRecord("recent", { dot_id: "d1", status: "FAILED", error: "boom", finished_at: hoursAgo(2) }), - taskRecord("old", { dot_id: "d1", status: "FAILED", error: "boom", finished_at: hoursAgo(30) }), - taskRecord("fine", { dot_id: "d1", status: "COMPLETED", finished_at: hoursAgo(1) }), - taskRecord("dismissed", { dot_id: "d1", status: "FAILED", error: "boom", finished_at: hoursAgo(3) }), - ]; - window.localStorage.setItem("idots.dismissed-tasks", JSON.stringify(["dismissed"])); - renderShell(); - expect(await within(await screen.findByRole("link", { name: /^Inbox/ })).findByLabelText("1 need you")).toBeTruthy(); - await waitFor(() => expect(document.title).toBe("(1) Dots - invisible_dots")); - }); - - it("reads the tasks again when one fails", async () => { - plane.dots = [dotRecord("d1"), dotRecord("d2")]; - renderShell(); - await waitFor(() => expect(plane.streamOpen).toBe(true)); - await screen.findByRole("link", { name: /d2/ }); - expect(screen.queryByLabelText(/need you/)).toBeNull(); - - plane.tasks = [taskRecord("t1", { dot_id: "d2", status: "FAILED", error: "no luck", finished_at: new Date().toISOString() })]; - act(() => plane.push("d2", "task.failed", { task_id: "t1", error: "no luck" })); - await waitFor(() => expect(within(screen.getByRole("link", { name: /^Inbox/ })).getByLabelText("1 need you")).toBeTruthy()); - }); - - it("marks the Dot whose channel needs linking again, counts it for the Inbox, and clears both when it is linked", async () => { - plane.dots = [dotRecord("d1", { name: "fares" }), dotRecord("d2", { name: "mailer" })]; - plane.channels = { d2: [channelRecord("telegram", { status: "needs_relink", status_detail: "revoked" })] }; - renderShell(); - const dots = within(await screen.findByRole("navigation", { name: "Dots" })); - expect(await dots.findByLabelText("Telegram needs linking again")).toBeTruthy(); - expect(within(dots.getByRole("link", { name: /mailer/ })).getByLabelText("Telegram needs linking again")).toBeTruthy(); - expect(within(dots.getByRole("link", { name: /fares/ })).queryByLabelText(/needs linking again/)).toBeNull(); - expect(within(screen.getByRole("link", { name: /^Inbox/ })).getByLabelText("1 need you")).toBeTruthy(); - await waitFor(() => expect(document.title).toBe("(1) Dots - invisible_dots")); - - // The person gives a new token: the host says so, and the shell reads the channels again. - plane.channels = { d2: [channelRecord("telegram")] }; - act(() => plane.push("d2", "channel.status", { kind: "telegram", status: "connected" })); - await waitFor(() => expect(screen.queryByLabelText(/needs linking again/)).toBeNull()); - await waitFor(() => expect(screen.queryByLabelText(/need you/)).toBeNull()); - }); - - it("puts the prefix back when the page writes a new title", async () => { - plane.dots = [dotRecord("d1", { status: "ERROR", error: "x", computer_state: "ERROR" })]; - renderShell(); - await waitFor(() => expect(document.title).toBe("(1) Dots - invisible_dots")); - act(() => { - document.title = "Chat - invisible_dots"; - }); - await waitFor(() => expect(document.title).toBe("(1) Chat - invisible_dots")); - }); -}); - -describe("the menu on a narrow screen", () => { - it("opens the same rail as a sheet, and closes it when a page is chosen", async () => { - plane.dots = [dotRecord("d1")]; - renderShell(); - await userEvent.click(screen.getByRole("button", { name: "Open the menu" })); - const sheet = await screen.findByRole("dialog"); - // It opens on the page the person is on, and no status of the rail's foot takes the focus (and opens its tooltip). - await waitFor(() => expect(document.activeElement?.getAttribute("aria-current")).toBe("page")); - expect(sheet.contains(document.activeElement)).toBe(true); - expect(screen.queryByRole("tooltip")).toBeNull(); - const link = await within(sheet).findByRole("link", { name: /d1/ }); - // jsdom cannot navigate: the click is still heard by the page, which is what closes the sheet. - document.addEventListener("click", (event) => event.preventDefault(), { once: true }); - await userEvent.click(link); - await waitFor(() => expect(screen.queryByRole("dialog")).toBeNull()); - }); -}); diff --git a/apps/web/test/skills.test.tsx b/apps/web/test/skills.test.tsx deleted file mode 100644 index 165a90d3..00000000 --- a/apps/web/test/skills.test.tsx +++ /dev/null @@ -1,107 +0,0 @@ -// @vitest-environment jsdom -import { act, cleanup, render, screen, waitFor, within } from "@testing-library/react"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { DotShell } from "../src/components/DotShell"; -import { DotEventScope, EventStreamProvider } from "../src/components/events"; -import { AttentionProvider } from "../src/components/shell/attention"; -import { SkillsView } from "../src/components/skills/SkillsView"; -import { parseSkillsQuery, skillBody, skillsHref } from "../src/lib/skills-view"; -import { stubMatchMedia } from "./support/browser"; -import { dotRecord, FakeControlPlane } from "./support/control-plane"; - -vi.mock("next/navigation", () => ({ usePathname: () => "/dots/d1/skills", useRouter: () => ({ push() {}, replace() {} }) })); - -let plane: FakeControlPlane; - -beforeEach(() => { - plane = new FakeControlPlane(); - plane.dots = [dotRecord("d1", { name: "fares" })]; - plane.install(); - stubMatchMedia(); -}); - -afterEach(() => { - cleanup(); - vi.unstubAllGlobals(); -}); - -const own = { - name: "shop-login", - description: "Log in to the shop.", - source: "dot" as const, - path: "/home/dot/skills/shop-login/SKILL.md", - content: "---\nname: shop-login\ndescription: Log in to the shop.\n---\n\nClick **Sign in**, then type the email.\n", -}; - -async function renderSkills(params: Record = {}) { - render( - - - - - - - - - , - ); - await screen.findByRole("heading", { level: 1, hidden: true }); - await waitFor(() => expect(plane.streamOpen).toBe(true)); -} - -describe("the address of the Skills page", () => { - it("holds the open skill, and the shortest address for none", () => { - expect(parseSkillsQuery({})).toEqual({ skill: null }); - expect(parseSkillsQuery({ skill: ["shop-login", "x"] })).toEqual({ skill: "shop-login" }); - expect(skillsHref("my dot")).toBe("/dots/my%20dot/skills"); - expect(skillsHref("d1", "shop-login")).toBe("/dots/d1/skills?skill=shop-login"); - }); - - it("shows a skill's file without its frontmatter", () => { - expect(skillBody(own.content)).toBe("Click **Sign in**, then type the email.\n"); - expect(skillBody("no frontmatter\n")).toBe("no frontmatter\n"); - }); -}); - -describe("the Skills page", () => { - it("lists the built-in skills and the Dot's own, saying which is which, and asks to pick one", async () => { - plane.skills = [...plane.skills!, own]; - await renderSkills(); - const list = await screen.findByRole("list", { name: "The Dot's skills" }); - const rows = within(list).getAllByRole("link"); - expect(rows.map((row) => row.getAttribute("href"))).toEqual(["/dots/d1/skills?skill=invisible-playwright", "/dots/d1/skills?skill=shop-login"]); - expect(rows[0]!.textContent).toContain("Built in"); - expect(rows[1]!.textContent).toContain("Written by the Dot"); - expect(rows[1]!.textContent).toContain("Log in to the shop."); - expect(screen.getByText("Pick a skill to read it.")).toBeTruthy(); - }); - - it("shows the open skill as Markdown, with where it is on the computer, and says so of a name it does not have", async () => { - plane.skills = [...plane.skills!, own]; - await renderSkills({ skill: "shop-login" }); - const open = await screen.findByRole("region", { name: "The skill shop-login" }); - expect(within(open).getByText("/home/dot/skills/shop-login/SKILL.md")).toBeTruthy(); - expect(within(open).getByText("Sign in").tagName).toBe("STRONG"); - expect(open.textContent).not.toContain("description:"); - expect(within(screen.getByRole("list", { name: "The Dot's skills" })).getByRole("link", { current: "page" }).textContent).toContain("shop-login"); - cleanup(); - - await renderSkills({ skill: "gone" }); - expect(await screen.findByText("The Dot has no skill named gone.")).toBeTruthy(); - }); - - it("reads the list again when a turn ends, which may have written a skill", async () => { - await renderSkills(); - await screen.findByRole("list", { name: "The Dot's skills" }); - plane.skills = [...plane.skills!, own]; - act(() => plane.push("d1", "message.assistant", { text: "I wrote down how to log in." })); - expect(await screen.findByText("shop-login")).toBeTruthy(); - }); - - it("asks nothing of a stopped computer and offers to start it", async () => { - plane.dots = [dotRecord("d1", { name: "fares", computer_state: "STOPPED" })]; - await renderSkills(); - expect((await screen.findByRole("status")).textContent).toContain("Start the computer to see its skills"); - expect(plane.requests.filter((r) => /\/skills$/.test(r))).toEqual([]); - }); -}); diff --git a/apps/web/test/support/browser.ts b/apps/web/test/support/browser.ts deleted file mode 100644 index c5133caa..00000000 --- a/apps/web/test/support/browser.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { vi } from "vitest"; - -/** - * jsdom has no `matchMedia`; this one answers every query the same: "no" by default (light mode, a narrow window), or - * "yes" when `matches` is true (a wide window; dark mode too, which no test of a wide window looks at). It never changes. - */ -export function stubMatchMedia(matches = false): void { - vi.stubGlobal( - "matchMedia", - vi.fn(() => ({ matches, addEventListener() {}, removeEventListener() {} })), - ); -} - -/** jsdom has no layout, so no `ResizeObserver`: this one never reports a size, which leaves a scrolling thread where it is. */ -export function stubResizeObserver(): void { - vi.stubGlobal( - "ResizeObserver", - class { - observe() {} - unobserve() {} - disconnect() {} - }, - ); -} - -/** - * jsdom has no object URLs: each one made here is a distinct address that the test can see being made and revoked. - * It is set on `URL` itself, which lasts for the test file (every file has its own jsdom). - */ -export function stubObjectUrls(): { created: string[]; revoked: string[] } { - const record = { created: [] as string[], revoked: [] as string[] }; - let next = 0; - URL.createObjectURL = () => { - const url = `blob:test/${++next}`; - record.created.push(url); - return url; - }; - URL.revokeObjectURL = (url: string) => { - record.revoked.push(url); - }; - return record; -} diff --git a/apps/web/test/support/config.ts b/apps/web/test/support/config.ts deleted file mode 100644 index bda2a697..00000000 --- a/apps/web/test/support/config.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { parseDotConfig, type DotConfig } from "@invisible-dots/shared/browser"; - -/** A whole Dot config as the API stores it: every default applied, the way a real record's config always is. */ -export function fullConfig(change: Record = {}): DotConfig { - return parseDotConfig({ - name: "fare-watch", - model: { provider: "openrouter", id: "z-ai/glm-5.3-flash" }, - ...change, - }); -} diff --git a/apps/web/test/support/contrast.ts b/apps/web/test/support/contrast.ts deleted file mode 100644 index 26eaaaa6..00000000 --- a/apps/web/test/support/contrast.ts +++ /dev/null @@ -1,41 +0,0 @@ -/** WCAG 2.x contrast, the one measure the color tests of the web client share (tokens, avatars). Colors are [r, g, b], 0 to 255. */ -export type Rgb = readonly [number, number, number]; - -export function fromHex(hex: string): Rgb { - return [parseInt(hex.slice(1, 3), 16), parseInt(hex.slice(3, 5), 16), parseInt(hex.slice(5, 7), 16)]; -} - -/** CSS `hsl(h s l)`: hue in degrees, saturation and lightness in 0 to 1. */ -export function fromHsl(hue: number, saturation: number, lightness: number): Rgb { - const chroma = (1 - Math.abs(2 * lightness - 1)) * saturation; - const x = chroma * (1 - Math.abs(((hue / 60) % 2) - 1)); - const m = lightness - chroma / 2; - const sector: Array<[number, number, number]> = [ - [chroma, x, 0], - [x, chroma, 0], - [0, chroma, x], - [0, x, chroma], - [x, 0, chroma], - [chroma, 0, x], - ]; - const [r, g, b] = sector[Math.floor(hue / 60) % 6]!; - return [(r + m) * 255, (g + m) * 255, (b + m) * 255]; -} - -/** `top` painted over `bottom` at `alpha` (what `bg-input/30` is on the surface under it). */ -export function over(top: Rgb, alpha: number, bottom: Rgb): Rgb { - return [0, 1, 2].map((i) => top[i]! * alpha + bottom[i]! * (1 - alpha)) as unknown as Rgb; -} - -export function luminance([r, g, b]: Rgb): number { - const channel = (value: number) => { - const v = value / 255; - return v <= 0.03928 ? v / 12.92 : ((v + 0.055) / 1.055) ** 2.4; - }; - return 0.2126 * channel(r) + 0.7152 * channel(g) + 0.0722 * channel(b); -} - -export function contrast(a: Rgb, b: Rgb): number { - const [light, dark] = [luminance(a), luminance(b)].sort((x, y) => y - x); - return (light! + 0.05) / (dark! + 0.05); -} diff --git a/apps/web/test/support/control-plane.ts b/apps/web/test/support/control-plane.ts deleted file mode 100644 index e502cb87..00000000 --- a/apps/web/test/support/control-plane.ts +++ /dev/null @@ -1,661 +0,0 @@ -/** - * A stand-in for the control plane as the browser sees it: the routes the shell reads and the computer actions it - * sends, answered from memory, and `/api/stream` as a live SSE body the test pushes events into. `install()` puts - * it behind the global `fetch`, which is where the web client's SDK looks. - */ -import { COMPUTER_STOPPED, CONVERSATION_LIST_LIMIT, TASK_LIST_LIMIT, computerIsUp, MAX_EVENT_PAGE, type ApprovalRecord, type BrowserIdentity, type ChannelKind, type ChannelLinkFrame, type ChannelRecord, type ChannelSettings, type ComputerAnswer, type DoctorCheck, type DotConfig, type DotSummary, type McpServerStatus, type Skill, type StoredEvent, type SystemAnswer, type ToolInfo, mcpServerNames } from "@invisible-dots/shared/browser"; -import type { TaskRecord } from "@invisible-dots/sdk"; -import { vi } from "vitest"; - -export function dotRecord(id: string, change: Partial = {}): DotSummary { - return { - id, - name: id, - // The API parses every config with the schema's defaults, so these two always exist on a real record. - config: { permissions: {} } as unknown as DotConfig, - status: "READY", - error: null, - created_at: "2026-01-01T00:00:00Z", - updated_at: "2026-01-01T00:00:00Z", - computer_state: "RUNNING", - config_version: 1, - ...change, - }; -} - -export function approvalRecord(id: string, dotId: string, change: Partial = {}): ApprovalRecord { - return { - id, - dot_id: dotId, - task_id: null, - tool: "exec", - permission: "computer.exec", - arguments: {}, - reason: "needs a command", - status: "pending", - note: null, - created_at: "2026-01-01T00:00:00Z", - resolved_at: null, - ...change, - }; -} - -export function channelRecord(kind: ChannelKind, change: Partial = {}): ChannelRecord { - const settings: ChannelSettings = { approvals: true, notify_tasks: true, show_arguments: false }; - return { - kind, - enabled: true, - status: "connected", - status_detail: null, - account: kind === "telegram" ? "fake_bot" : "15550001111", - settings, - peers: [], - created_at: "2026-01-01T00:00:00Z", - ...change, - }; -} - -export function taskRecord(id: string, change: Partial = {}): TaskRecord { - return { - id, - dot_id: "d1", - description: `task ${id}`, - priority: 0, - status: "PENDING", - created_at: "2026-01-01T00:00:00Z", - scheduled_at: null, - started_at: null, - finished_at: null, - summary: null, - error: null, - spent_usd: 0, - ...change, - }; -} - -export class FakeControlPlane { - dots: DotSummary[] = []; - approvals: ApprovalRecord[] = []; - tasks: TaskRecord[] = []; - /** The stored event log, as `GET /api/dots/:id/events` pages through it; `push` and `store` add to it. */ - events: StoredEvent[] = []; - /** What each `GET .../events` asked for, so a test can see that a page cut by type was asked for by type. */ - eventQueries: Array<{ after: number; before?: number; limit: number; types: string[] | null; tools: string[] | null; taskId: string | null; order: string | null }> = []; - /** What the shell asked for to know the agent's state when a page opened: the newest `agent.state` or `agent.started`. */ - agentQueries: Array<{ after: number; before?: number; limit: number; types: string[] | null; tools: string[] | null; taskId: string | null; order: string | null }> = []; - /** What each `GET .../tasks` asked for. */ - taskQueries: Array<{ limit: number | null; before: string | null }> = []; - /** While set, `GET /api/stream` answers 503, as a control plane that does not answer does. */ - streamDown = false; - /** What each `GET /api/approvals` asked for. */ - approvalQueries: Array<{ status: string[] | null; limit: number | null; order: string | null; before: string | null; dot_id: string | null }> = []; - /** The body of every `POST /api/dots/:id/tasks`, as the browser sent it. */ - createdTasks: unknown[] = []; - /** Answer `POST /api/dots/:id/tasks` with this error instead of 201. */ - failCreateTask: { status: number; error: string; message: string } | null = null; - /** Answer `POST /api/tasks/:id/cancel` with this error instead of cancelling. */ - failCancel: { status: number; error: string; message: string } | null = null; - /** Every `POST /api/approvals/:id/approve|reject` the browser sent, in order, with the body it carried. */ - answers: Array<{ id: string; decision: "approve" | "reject"; body: { note?: string; always?: true } }> = []; - /** Answer approval answers with this error instead of recording them. */ - failAnswer: { status: number; error: string; message: string } | null = null; - /** The skills `GET /api/dots/:id/skills` answers with; null answers 409 computer_stopped, as a stopped computer does. */ - skills: Skill[] | null = [ - { - name: "invisible-playwright", - description: "Use the Dot's browser for any task on a website.", - source: "builtin", - path: "/opt/invisible-dots/engine/skills/invisible-playwright/SKILL.md", - content: "---\nname: invisible-playwright\ndescription: Use the Dot's browser for any task on a website.\n---\n\n# The browser\n\nOpen an **identity** first.\n", - }, - ]; - /** The tool table `GET /api/dots/:id/tools` answers with; null answers 409 computer_stopped, as a stopped computer does. */ - tools: ToolInfo[] | null = [ - { name: "exec", permission: "computer.exec", offered: true, description: "Run a shell command." }, - { name: "exec_session", permission: "computer.exec", offered: true, description: "Use a command session." }, - { name: "read_file", permission: "files.read", offered: true, description: "Read a file." }, - ]; - /** The MCP servers' states `GET /api/dots/:id/tools` answers with, beside the tools. */ - mcpServers: McpServerStatus[] = []; - /** The MCP secrets that are set, as `/`; never their values, as the host answers. */ - readonly mcpSecretsSet = new Set(); - /** Every `PUT` (a value) and `DELETE` (null) of an MCP secret, as the browser sent it. */ - readonly mcpSecretWrites: Array<{ server: string; name: string; value: string | null }> = []; - /** The channels `GET /api/dots/:id/channels` lists, by Dot id. */ - channels: Record = {}; - /** The kinds of channel the server can run, as `GET .../channels` says. */ - channelsAvailable: ChannelKind[] = ["telegram"]; - /** Every change of a channel as "METHOD kind sub body", in order (a token is never written here). */ - channelActions: string[] = []; - /** Answer the channel routes that change something with this error instead of doing it. */ - failChannel: { status: number; error: string; message: string } | null = null; - /** Answer `GET .../channels` with this error instead of the list. */ - failChannels: { status: number; error: string; message: string } | null = null; - /** The code `POST .../pairing` makes next, and how long it lasts. */ - pairing = { code: "K7M2QX9P", expiresInMs: 600_000 }; - /** The token of every `PUT .../channels/telegram`, as the browser sent it. */ - sentTokens: string[] = []; - /** The body of every `PATCH /api/dots/:id`, as the browser sent it. */ - updates: Array<{ config: unknown; expected_config_version?: number }> = []; - /** Answer `PATCH /api/dots/:id` with this error instead of saving. */ - failUpdate: { status: number; error: string; message: string } | null = null; - /** The ids of the Dots the browser deleted (`DELETE /api/dots/:id`), in order. */ - deleted: string[] = []; - /** Answer `DELETE /api/dots/:id` with this error instead of accepting it. */ - failDelete: { status: number; error: string; message: string } | null = null; - /** The most events one `GET .../events` page holds (the real route's is 1000). */ - eventPage = MAX_EVENT_PAGE; - /** Answer `GET .../events` with this status instead of the log. */ - failEvents: number | null = null; - /** The text of every `POST /api/dots/:id/messages`, as the browser sent it. */ - sentMessages: string[] = []; - /** Answer `POST /api/dots/:id/messages` with this error instead of 201. */ - failSend: { status: number; error: string; message: string } | null = null; - /** What `POST /api/dots/:id/messages` says of the delivery. */ - delivery: "delivered" | "queued" = "delivered"; - /** While set, `POST /api/dots/:id/messages` waits for it to resolve before it answers. */ - holdSend: Promise | null = null; - /** What each `GET .../messages` asked for. */ - messageQueries: Array<{ limit: number | null; order: string | null; before: number | null }> = []; - /** The identities `GET .../browser-identities` lists. */ - identities: BrowserIdentity[] = []; - /** Answer `GET .../computer/screenshot` and `.../frame` with this error instead of a picture. */ - failPicture: { status: number; error: string; message: string } | null = null; - /** Spend today, as `GET /api/dots/:id/usage?since=` answers it, and in total, as it answers without `since`. */ - spentUsd = 0; - spentTotalUsd = 0; - /** The body of every `POST .../browser-identities`, as the browser sent it. */ - createdIdentities: Array<{ name: string; proxy?: string }> = []; - /** The ids of the identities the browser closed (`POST .../close`) and deleted, in order. */ - closedIdentities: string[] = []; - deletedIdentities: string[] = []; - /** Answer an identity's create, close or delete with this error instead of doing it. */ - failIdentityAction: { status: number; error: string; message: string } | null = null; - /** Answer `GET .../browser-identities` with this error instead of the list. */ - failIdentities: { status: number; error: string; message: string } | null = null; - /** The files of the Dot's computer by absolute path; a folder exists where a file is under it (and home always). */ - files = new Map(); - /** Answer the files routes with this error instead of the folder or the file. */ - failFiles: { status: number; error: string; message: string } | null = null; - /** What `GET .../computer` says of the guest while it runs. */ - system: SystemAnswer | null = null; - computerImages: { golden_image: string | null; runtime_image: string | null } = { golden_image: "golden-1.qcow2", runtime_image: "runtime-1.iso" }; - /** `ready` of `GET .../computer`. */ - ready = true; - keyConfigured = true; - healthy = true; - /** The value of every `PUT /api/secrets/openrouter`, as the browser sent it. */ - savedKeys: string[] = []; - /** How many running Dots `PUT /api/secrets/openrouter` says it pushed the key to. */ - keyPushedTo = 0; - /** Answer `PUT /api/secrets/openrouter` with this error instead of storing the key. */ - failKey: { status: number; error: string; message: string } | null = null; - /** What `GET /api/doctor` reports, in the contract's order (the real one has nine rows, the last the key). */ - doctor: DoctorCheck[] = [ - { id: "node", label: "Node.js", status: "ok", detail: "24.1.0" }, - { id: "qemu", label: "QEMU", status: "ok", detail: "8.2.2" }, - { id: "golden-image", label: "golden image", status: "ok", detail: "golden-1.qcow2 matches its manifest" }, - { id: "openrouter", label: "OpenRouter key", status: "ok", detail: "stored" }, - ]; - /** Answer `GET /api/doctor` with this status instead of the report. */ - failDoctor: number | null = null; - computerLastError: string | null = null; - /** `stop_reason` and `next_automation_at` of `GET .../computer`: what the host records, which is there while the computer is off. */ - computerStopReason: ComputerAnswer["stop_reason"] = null; - nextAutomationAt: string | null = null; - /** Every request as "METHOD path", in order. */ - requests: string[] = []; - /** Answer a computer action with this error status instead of 202. */ - failComputerAction: number | null = null; - /** The `config` of every `POST /api/dots`, as the browser sent it (YAML text or an object). */ - created: unknown[] = []; - /** Answer `POST /api/dots` with this error instead of 201. */ - failCreate: { status: number; error: string; message: string; details?: unknown } | null = null; - #stream: ReadableStreamDefaultController | null = null; - /** The WhatsApp link streams that are open, by Dot id. */ - #links = new Map>(); - #nextEventId = 1; - - /** `fetch` as the browser would have it: the routes of this class and nothing else. */ - fetch = (input: RequestInfo | URL, init?: RequestInit): Promise => this.#answer(String(input), init); - - install(): void { - vi.stubGlobal("fetch", vi.fn(this.fetch)); - } - - /** Put a file on the Dot's computer. */ - putFile(path: string, content: string | Uint8Array, mtime = "2026-03-10T12:00:00Z"): void { - this.files.set(path, { content: typeof content === "string" ? new TextEncoder().encode(content) : content, mtime }); - } - - /** Put an event in the stored log without telling the stream: it happened before the page opened. */ - store(dotId: string, type: string, data: Record = {}, createdAt = new Date().toISOString()): StoredEvent { - const event = { id: this.#nextEventId++, dot_id: dotId, type, data, source: "guest", guest_seq: null, created_at: createdAt } as StoredEvent; - this.events.push(event); - return event; - } - - /** Push a live event to everything listening on the stream (it is stored too, as the real one is). */ - push(dotId: string, type: string, data: Record = {}): void { - const event = this.store(dotId, type, data); - this.#stream?.enqueue(new TextEncoder().encode(`id: ${event.id}\ndata: ${JSON.stringify(event)}\n\n`)); - } - - /** The connection of the live stream ends without a last event: the control plane stopped, or the network dropped. */ - dropStream(): void { - const stream = this.#stream; - this.#stream = null; - stream?.close(); - } - - get streamOpen(): boolean { - return this.#stream !== null; - } - - /** Whether a page follows the WhatsApp link of this Dot (`GET .../channels/whatsapp/qr` is open). */ - linkOpen(dotId: string): boolean { - return this.#links.has(dotId); - } - - /** The connection of the WhatsApp link ends without a last frame: the server went away. */ - dropLink(dotId: string): void { - const link = this.#links.get(dotId); - if (!link) throw new Error(`nothing follows the WhatsApp link of ${dotId}`); - this.#links.delete(dotId); - link.close(); - } - - /** A frame of the WhatsApp link, to the page that follows it; a last frame (`linked`, `failed`) ends the stream, as the host's does. */ - linkFrame(dotId: string, frame: ChannelLinkFrame): void { - const link = this.#links.get(dotId); - if (!link) throw new Error(`nothing follows the WhatsApp link of ${dotId}`); - link.enqueue(new TextEncoder().encode(`data: ${JSON.stringify(frame)}\n\n`)); - if (frame.state === "linked" || frame.state === "failed") { - this.#links.delete(dotId); - link.close(); - } - } - - async #answer(url: string, init?: RequestInit): Promise { - const { pathname, searchParams } = new URL(url, "http://web.test"); - const method = init?.method ?? "GET"; - this.requests.push(`${method} ${pathname}`); - const json = (body: unknown, status = 200) => Response.json(body, { status }); - - if (pathname === "/api/stream") { - if (this.streamDown) return json({ error: "down", message: "the control plane is down" }, 503); - return new Response( - new ReadableStream({ - start: (controller) => { - this.#stream = controller; - init?.signal?.addEventListener("abort", () => { - this.#stream = null; - try { - controller.close(); - } catch { - // already closed - } - }); - }, - }), - { headers: { "content-type": "text/event-stream" } }, - ); - } - if (pathname === "/session" && method === "DELETE") return new Response(null, { status: 204 }); - if (pathname === "/api/doctor") { - if (this.failDoctor) return json({ error: "broken", message: "the doctor could not run" }, this.failDoctor); - return json({ ok: this.doctor.every((check) => check.status === "ok"), checks: this.doctor }); - } - if (pathname === "/api/health") { - return this.healthy - ? json({ - status: "ok", - database: "ok", - version: "9.9.9", - openrouter_configured: this.keyConfigured, - database_kind: "pglite", - data_dir: "/home/me/.invisible-dots", - logs_dir: "/home/me/.invisible-dots/logs", - }) - : json({ error: "down", message: "down" }, 503); - } - if (pathname === "/api/secrets/openrouter" && method === "PUT") { - if (this.failKey) return json({ error: this.failKey.error, message: this.failKey.message }, this.failKey.status); - this.savedKeys.push((JSON.parse(String(init?.body)) as { value: string }).value); - this.keyConfigured = true; - return json({ pushed: this.keyPushedTo }); - } - if (pathname === "/api/dots" && method === "POST") { - const { config } = JSON.parse(String(init?.body)) as { config: unknown }; - this.created.push(config); - if (this.failCreate) return json({ error: this.failCreate.error, message: this.failCreate.message, details: this.failCreate.details }, this.failCreate.status); - return json(dotRecord("created-1", { name: "created", status: "CREATING", computer_state: null }), 201); - } - if (pathname === "/api/dots") return json({ dots: this.dots }); - if (pathname === "/api/approvals") { - const status = searchParams.get("status")?.split(",") ?? null; - const limit = searchParams.get("limit") === null ? null : Number(searchParams.get("limit")); - const order = searchParams.get("order"); - const before = searchParams.get("before"); - const dotId = searchParams.get("dot_id"); - this.approvalQueries.push({ status, limit, order, before, dot_id: dotId }); - // The real route: oldest first, or with order=desc the newest by last change, a cursor being the id of the last row seen. - const changed = (a: ApprovalRecord) => Date.parse(a.resolved_at ?? a.created_at); - const listed = this.approvals.filter((a) => (!status || status.includes(a.status)) && (dotId === null || a.dot_id === dotId)); - const ordered = order === "desc" ? [...listed].sort((a, b) => changed(b) - changed(a) || b.id.localeCompare(a.id)) : listed; - const from = before === null ? 0 : ordered.findIndex((a) => a.id === before) + 1 || ordered.length; - return json({ approvals: ordered.slice(from, from + (limit ?? 500)) }); - } - const answer = /^\/api\/approvals\/([^/]+)\/(approve|reject)$/.exec(pathname); - if (answer && method === "POST") { - const id = decodeURIComponent(answer[1]!); - const decision = answer[2] as "approve" | "reject"; - const body = JSON.parse(String(init?.body ?? "{}")) as { note?: string; always?: true }; - this.answers.push({ id, decision, body }); - if (this.failAnswer) return json({ error: this.failAnswer.error, message: this.failAnswer.message }, this.failAnswer.status); - const record = this.approvals.find((a) => a.id === id); - if (!record) return json({ error: "not_found", message: `no approval ${id}` }, 404); - if (record.status !== "pending") return json({ error: "already_resolved", message: `approval ${id} is already ${record.status}` }, 409); - Object.assign(record, { status: decision === "approve" ? "approved" : "rejected", note: body.note ?? null, resolved_at: new Date().toISOString() }); - // The host logs the answer and publishes it: the page hears it as it would from the real one. - this.push(record.dot_id, "approval.resolved", { approval_id: id, decision, ...(record.task_id ? { task_id: record.task_id } : {}), ...(body.note ? { note: body.note } : {}), ...(body.always ? { always: true } : {}) }); - return json(record); - } - if (pathname === "/api/tasks" && method === "GET") { - // The real route: the tasks of every Dot that ended in `status` after `finished_after`, the latest end first. - const status = searchParams.get("status"); - const after = Date.parse(searchParams.get("finished_after") ?? ""); - const ended = this.tasks.filter((t) => t.status === status && t.finished_at !== null && Date.parse(t.finished_at) > after); - return json({ tasks: ended.sort((a, b) => Date.parse(b.finished_at!) - Date.parse(a.finished_at!)) }); - } - if (pathname === "/api/channels" && method === "GET") { - // The real route: every channel of every Dot, each with its Dot's id. - if (this.failChannels) return json({ error: this.failChannels.error, message: this.failChannels.message }, this.failChannels.status); - return json({ channels: Object.entries(this.channels).flatMap(([dotId, list]) => list.map((channel) => ({ dot_id: dotId, ...channel }))) }); - } - const task = /^\/api\/tasks\/([^/]+)(\/cancel)?$/.exec(pathname); - if (task) { - const record = this.tasks.find((t) => t.id === decodeURIComponent(task[1]!)); - if (!record) return json({ error: "not_found", message: "no such task" }, 404); - if (!task[2]) return json(record); - if (this.failCancel) return json({ error: this.failCancel.error, message: this.failCancel.message }, this.failCancel.status); - Object.assign(record, { status: "CANCELLED", error: "cancelled by the user", finished_at: new Date().toISOString() }); - return json(record); - } - const dot = /^\/api\/dots\/([^/]+)(?:\/(.*))?$/.exec(pathname); - if (dot) { - // The real routes take a Dot's id or its name (requireDot). - const address = decodeURIComponent(dot[1]!); - const record = this.dots.find((d) => d.id === address) ?? this.dots.find((d) => d.name === address); - if (!record) return json({ error: "not_found", message: "no such Dot" }, 404); - const rest = dot[2] ?? ""; - if (rest === "" && method === "PATCH") { - const body = JSON.parse(String(init?.body)) as { config: DotConfig; expected_config_version?: number }; - this.updates.push(body); - if (this.failUpdate) return json({ error: this.failUpdate.error, message: this.failUpdate.message }, this.failUpdate.status); - if (body.expected_config_version !== undefined && body.expected_config_version !== record.config_version) { - return json({ error: "dot_changed", message: `Dot ${record.name} changed after you read it` }, 409); - } - record.config = body.config; - record.config_version += 1; - this.push(record.id, "dot.updated", { name: record.name }); - return json(record); - } - if (rest === "" && method === "DELETE") { - if (this.failDelete) return json({ error: this.failDelete.error, message: this.failDelete.message }, this.failDelete.status); - this.deleted.push(record.id); - return json({ accepted: true }, 202); - } - if (rest === "") return json(record); - if (rest === "tasks" && method === "GET") { - // The real route: the newest created first, at most TASK_LIST_LIMIT a page, `before` the id of the last task of the page before. - const limit = Math.min(Number(searchParams.get("limit") ?? TASK_LIST_LIMIT), TASK_LIST_LIMIT); - const before = searchParams.get("before"); - this.taskQueries.push({ limit: searchParams.has("limit") ? limit : null, before }); - const newest = this.tasks.filter((t) => t.dot_id === record.id).sort((a, b) => Date.parse(b.created_at) - Date.parse(a.created_at) || (a.id < b.id ? 1 : -1)); - const from = before === null ? 0 : newest.findIndex((t) => t.id === before) + 1 || newest.length; - return json({ tasks: newest.slice(from, from + limit) }); - } - if (rest === "tasks" && method === "POST") { - const body = JSON.parse(String(init?.body)) as { description: string; priority?: number; scheduled_at?: string }; - this.createdTasks.push(body); - if (this.failCreateTask) return json({ error: this.failCreateTask.error, message: this.failCreateTask.message }, this.failCreateTask.status); - const created = taskRecord(`created-${this.createdTasks.length}`, { - dot_id: record.id, - description: body.description, - priority: body.priority ?? 0, - scheduled_at: body.scheduled_at ?? null, - created_at: new Date().toISOString(), - }); - this.tasks.push(created); - return json(created, 201); - } - if (rest === "events") { - if (this.failEvents) return json({ error: "broken", message: "the event log is not available" }, this.failEvents); - const after = Number(searchParams.get("after") ?? 0); - const limit = Math.min(Number(searchParams.get("limit") ?? 500), this.eventPage); - // The real route's filters: `types` (a comma-separated list) and `task_id` (data.task_id). - const types = searchParams.get("types")?.split(","); - const tools = searchParams.get("tools")?.split(","); - const taskId = searchParams.get("task_id"); - const order = searchParams.get("order"); - const before = searchParams.has("before") ? Number(searchParams.get("before")) : undefined; - if (before !== undefined && order !== "desc") return json({ error: "invalid_request", message: "before pages a list in order=desc" }, 400); - const query = { after, ...(before === undefined ? {} : { before }), limit, types: types ?? null, tools: tools ?? null, taskId, order }; - // The shell's one read of the agent's newest state is kept apart, so a test of what a view reads sees its own requests. - (types?.length === 2 && types.includes("agent.state") && types.includes("agent.started") ? this.agentQueries : this.eventQueries).push(query); - // `tools` narrows tool.called only, `order=desc` is the newest first and the limit keeps the newest. - const kept = this.events.filter( - (e) => - e.dot_id === record.id && - e.id > after && - (before === undefined || e.id < before) && - (!types || types.includes(e.type)) && - (!tools || e.type !== "tool.called" || tools.includes(String(e.data.tool))) && - (taskId === null || e.data.task_id === taskId), - ); - return json({ events: (order === "desc" ? [...kept].reverse() : kept).slice(0, limit) }); - } - if (rest === "messages" && method === "GET") { - // The host logs the person's side as `user.message`, which the shared event type list does not hold. - const type = (e: StoredEvent) => e.type as string; - const limit = searchParams.has("limit") ? Number(searchParams.get("limit")) : null; - const order = searchParams.get("order"); - const before = searchParams.has("before") ? Number(searchParams.get("before")) : null; - if (before !== null && order !== "desc") return json({ error: "invalid_request", message: "before pages a list in order=desc" }, 400); - this.messageQueries.push({ limit, order, before }); - // The real route: oldest first, or with order=desc the newest first, a cursor being the event id of the oldest message seen; a page holds at most CONVERSATION_LIST_LIMIT. - const kept = this.events.filter((e) => e.dot_id === record.id && (type(e) === "user.message" || type(e) === "message.assistant") && (before === null || e.id < before)); - const messages = (order === "desc" ? [...kept].reverse() : kept) - .slice(0, Math.min(limit ?? CONVERSATION_LIST_LIMIT, CONVERSATION_LIST_LIMIT)) - .map((e) => ({ event_id: e.id, role: type(e) === "user.message" ? "user" : "assistant", text: String(e.data.text ?? ""), in_reply_to: null, ...(type(e) === "user.message" && e.data.origin ? { origin: e.data.origin } : {}), created_at: e.created_at })); - return json({ messages }); - } - if (rest === "messages" && method === "POST") { - const { text } = JSON.parse(String(init?.body)) as { text: string }; - this.sentMessages.push(text); - if (this.holdSend) await this.holdSend; - if (this.failSend) return json({ error: this.failSend.error, message: this.failSend.message }, this.failSend.status); - const stored = this.store(record.id, "user.message", { message_id: `msg-${this.sentMessages.length}`, text }); - this.#stream?.enqueue(new TextEncoder().encode(`id: ${stored.id}\ndata: ${JSON.stringify(stored)}\n\n`)); - return json({ message_id: `msg-${this.sentMessages.length}`, event_id: stored.id, delivery: this.delivery }, 201); - } - if (rest === "computer/screenshot") { - if (this.failPicture) return json({ error: this.failPicture.error, message: this.failPicture.message }, this.failPicture.status); - return new Response(Uint8Array.from([0x89, 0x50, 0x4e, 0x47]), { headers: { "content-type": "image/png" } }); - } - if (rest === "browser-identities" && method === "GET") { - if (this.failIdentities) return json({ error: this.failIdentities.error, message: this.failIdentities.message }, this.failIdentities.status); - return json({ identities: this.identities }); - } - if (rest === "browser-identities" && method === "POST") { - const body = JSON.parse(String(init?.body)) as { name: string; proxy?: string }; - this.createdIdentities.push(body); - if (this.failIdentityAction) return json({ error: this.failIdentityAction.error, message: this.failIdentityAction.message }, this.failIdentityAction.status); - const created: BrowserIdentity = { id: `${body.name}-x${this.createdIdentities.length}`, name: body.name, status: "available", createdAt: new Date().toISOString(), lastUsedAt: null, profilePath: "/home/dot/browsers/x", hasProxy: Boolean(body.proxy) }; - this.identities.push(created); - this.push(record.id, "browser.identity.created", { identity_id: created.id, name: created.name }); - return json(created, 201); - } - const identity = /^browser-identities\/([^/]+)(?:\/(close))?$/.exec(rest); - if (identity && (identity[2] ? method === "POST" : method === "DELETE")) { - const id = decodeURIComponent(identity[1]!); - if (this.failIdentityAction) return json({ error: this.failIdentityAction.error, message: this.failIdentityAction.message }, this.failIdentityAction.status); - const found = this.identities.find((i) => i.id === id); - if (!found) return json({ error: "not_found", message: `no browser identity "${id}"` }, 404); - if (identity[2]) { - this.closedIdentities.push(id); - found.status = "available"; - this.push(record.id, "browser.identity.closed", { identity_id: id, name: found.name }); - } else { - this.deletedIdentities.push(id); - this.identities = this.identities.filter((i) => i.id !== id); - this.push(record.id, "browser.identity.deleted", { identity_id: id, name: found.name }); - } - return new Response(null, { status: 204 }); - } - if (/^browser-identities\/[^/]+\/frame$/.test(rest)) { - if (this.failPicture) return json({ error: this.failPicture.error, message: this.failPicture.message }, this.failPicture.status); - return new Response(Uint8Array.from([0xff, 0xd8, 0xff, 0xd9]), { headers: { "content-type": "image/jpeg" } }); - } - if (rest === "channels" && method === "GET") { - if (this.failChannels) return json({ error: this.failChannels.error, message: this.failChannels.message }, this.failChannels.status); - return json({ channels: this.channels[record.id] ?? [], available: this.channelsAvailable }); - } - const channel = /^channels\/(telegram|whatsapp)(?:\/(pairing|link|qr|peers\/([^/]+)))?$/.exec(rest); - if (channel) { - const kind = channel[1] as ChannelKind; - const sub = channel[2] ?? ""; - const list = (this.channels[record.id] ??= []); - const found = list.find((c) => c.kind === kind); - if (sub === "qr" && method === "GET") { - return new Response( - new ReadableStream({ - start: (controller) => { - this.#links.set(record.id, controller); - init?.signal?.addEventListener("abort", () => { - if (this.#links.get(record.id) === controller) this.#links.delete(record.id); - try { - controller.close(); - } catch { - // already closed - } - }); - }, - }), - { headers: { "content-type": "text/event-stream" } }, - ); - } - const body = init?.body ? (JSON.parse(String(init.body)) as Record) : {}; - const isToken = sub === "" && method === "PUT"; - this.channelActions.push(`${method} ${kind}${sub ? ` ${sub}` : ""}${init?.body && !isToken ? ` ${JSON.stringify(body)}` : ""}`); - if (this.failChannel) return json({ error: this.failChannel.error, message: this.failChannel.message }, this.failChannel.status); - const announce = (next: ChannelRecord) => this.push(record.id, "channel.status", { kind, status: next.status }); - if (isToken && kind === "telegram") { - this.sentTokens.push(String(body.token)); - const next = found ?? channelRecord("telegram"); - if (!found) list.push(next); - Object.assign(next, { status: "connected", status_detail: null }); - announce(next); - return json(next, found ? 200 : 201); - } - if (sub === "link" && method === "POST") { - const next = found ?? channelRecord(kind, { status: "connecting", account: null }); - if (!found) list.push(next); - // As the host does: a new link clears the account of the old one and keeps the people paired. - Object.assign(next, { status: "connecting", status_detail: null, account: null }); - return json(next, 202); - } - if (!found) return json({ error: "not_found", message: `Dot has no ${kind} channel` }, 404); - if (sub === "" && method === "PATCH") { - if (body.settings) Object.assign(found.settings, body.settings); - if (typeof body.enabled === "boolean") { - const changed = found.enabled !== body.enabled; - found.enabled = body.enabled; - // The host announces what the person did to a channel, which no status says. - if (changed) this.push(record.id, "channel.changed", { kind, change: body.enabled ? "resumed" : "paused" }); - } - return json(found); - } - if (sub === "" && method === "DELETE") { - this.channels[record.id] = list.filter((c) => c !== found); - this.push(record.id, "channel.changed", { kind, change: "removed" }); - return new Response(null, { status: 204 }); - } - if (sub === "pairing" && method === "POST") { - const message = kind === "telegram" ? `/start ${this.pairing.code}` : `pair ${this.pairing.code}`; - const deepLink = kind === "telegram" ? `https://t.me/${found.account}?start=${this.pairing.code}` : `https://wa.me/${found.account}?text=${encodeURIComponent(message)}`; - return json({ code: this.pairing.code, deep_link: deepLink, message, expires_at: new Date(Date.now() + this.pairing.expiresInMs).toISOString() }, 201); - } - if (channel[3] && method === "DELETE") { - const peer = decodeURIComponent(channel[3]); - if (!found.peers.some((p) => p.peer_id === peer)) return json({ error: "not_found", message: `no peer ${peer}` }, 404); - found.peers = found.peers.filter((p) => p.peer_id !== peer); - return new Response(null, { status: 204 }); - } - } - if (rest === "skills") return this.skills === null ? json({ error: "computer_stopped", message: "the computer is STOPPED" }, 409) : json({ skills: this.skills }); - if (rest === "tools") return this.tools === null ? json({ error: "computer_stopped", message: "the computer is STOPPED" }, 409) : json({ tools: this.tools, mcp_servers: this.mcpServers }); - const secretsOf = () => ({ - dot_id: record.id, - secrets: mcpServerNames(record.config).flatMap((server) => - record.config.mcp_servers[server]!.secrets.map((name) => ({ server, name, set: this.mcpSecretsSet.has(`${server}/${name}`) })), - ), - }); - if (rest === "mcp-secrets" && method === "GET") return json(secretsOf()); - const secret = /^mcp-secrets\/([^/]+)\/([^/]+)$/.exec(rest); - if (secret && (method === "PUT" || method === "DELETE")) { - const [server, name] = [decodeURIComponent(secret[1]!), decodeURIComponent(secret[2]!)]; - if (!record.config.mcp_servers[server]?.secrets.includes(name)) return json({ error: "not_found", message: `no secret ${server}/${name}` }, 404); - const value = method === "PUT" ? (JSON.parse(String(init?.body)) as { value: string }).value : null; - this.mcpSecretWrites.push({ server, name, value }); - if (value === null) this.mcpSecretsSet.delete(`${server}/${name}`); - else this.mcpSecretsSet.add(`${server}/${name}`); - return json(secretsOf()); - } - if (rest === "usage") return json({ dot_id: record.id, since: searchParams.get("since"), spent_usd: searchParams.get("since") ? this.spentUsd : this.spentTotalUsd }); - if (rest === "computer") { - const state = record.computer_state ?? "STOPPED"; - const answer: Partial = { - dot_id: record.id, - state, - last_error: this.computerLastError, - stop_reason: this.computerStopReason, - next_automation_at: this.nextAutomationAt, - ready: this.ready, - system: computerIsUp(state) ? this.system : null, - last_active_at: "2026-03-10T12:00:00Z", - ...this.computerImages, - }; - return json(answer); - } - if (rest === "files/list" || rest === "files") { - if (!computerIsUp(record.computer_state)) return json({ error: COMPUTER_STOPPED, message: `the computer is ${record.computer_state}` }, 409); - if (this.failFiles) return json({ error: this.failFiles.error, message: this.failFiles.message }, this.failFiles.status); - const asked = searchParams.get("path") ?? ""; - const path = asked === "" || asked === "~" ? "/home/dot" : asked.startsWith("/") ? asked.replace(/\/+$/, "") : `/home/dot/${asked.replace(/\/+$/, "")}`; - if (!path.startsWith("/home/dot")) return json({ error: "invalid_path", message: "path must be inside /home/dot" }, 400); - const isDir = path === "/home/dot" || [...this.files.keys()].some((file) => file.startsWith(`${path}/`)); - if (rest === "files") { - const file = this.files.get(path); - if (isDir) return json({ error: "is_a_directory", message: `${path} is a directory` }, 400); - if (!file) return json({ error: "not_found", message: `no such file ${path}` }, 404); - return new Response(file.content as Uint8Array, { headers: { "content-type": "application/octet-stream" } }); - } - if (this.files.has(path)) return json({ error: "not_a_directory", message: `${path} is not a directory` }, 400); - if (!isDir) return json({ error: "not_found", message: `no such folder ${path}` }, 404); - const entries = new Map(); - for (const [file, { content, mtime }] of this.files) { - if (!file.startsWith(`${path}/`)) continue; - const [name, ...more] = file.slice(path.length + 1).split("/"); - entries.set(name!, more.length > 0 ? { name: name!, type: "dir", size: 0, mtime } : { name: name!, type: "file", size: content.length, mtime }); - } - return json({ path, entries: [...entries.values()] }); - } - if (/^computer\/(start|stop|reboot)$/.test(rest) && method === "POST") { - return this.failComputerAction ? json({ error: "refused", message: `the computer refused (${this.failComputerAction})` }, this.failComputerAction) : json({ accepted: true }, 202); - } - } - return json({ error: "not_found", message: `${method} ${pathname}` }, 404); - } -} diff --git a/apps/web/test/support/tokens.ts b/apps/web/test/support/tokens.ts deleted file mode 100644 index 6e7129ce..00000000 --- a/apps/web/test/support/tokens.ts +++ /dev/null @@ -1,23 +0,0 @@ -/** The design tokens as the tests read them: the custom properties of src/app/tokens.css, per theme. */ -import { readFileSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; - -export const SRC = join(dirname(fileURLToPath(import.meta.url)), "../../src"); - -const tokensCss = readFileSync(join(SRC, "app/tokens.css"), "utf8"); - -/** The custom properties declared in one block of tokens.css. */ -function block(selector: string): Record { - const start = tokensCss.indexOf(`${selector} {`); - if (start < 0) throw new Error(`no ${selector} block in tokens.css`); - const body = tokensCss.slice(start, tokensCss.indexOf("}", start)); - return Object.fromEntries([...body.matchAll(/^\s*--([\w-]+):\s*([^;]+);/gm)].map((m) => [m[1]!, m[2]!.trim()])); -} - -/** Not colors, so not themed: they are declared once, in the light block. */ -const SHARED = ["radius", "font-ui", "font-code"]; -const colorsOf = (tokens: Record) => Object.fromEntries(Object.entries(tokens).filter(([key]) => !SHARED.includes(key))); - -export type Theme = "light" | "dark"; -export const themes: Record> = { light: colorsOf(block(":root")), dark: colorsOf(block(':root[data-theme="dark"]')) }; diff --git a/apps/web/test/task-events.test.ts b/apps/web/test/task-events.test.ts deleted file mode 100644 index c6347d5b..00000000 --- a/apps/web/test/task-events.test.ts +++ /dev/null @@ -1,172 +0,0 @@ -import { MAX_EVENT_PAGE, type StoredEvent } from "@invisible-dots/shared/browser"; -import { describe, expect, it } from "vitest"; -import { isTaskEvent, readTaskProgress, readTaskStory, TASK_EVENT_TYPES, mergeTaskEvents, storyOf, taskIdOf } from "../src/lib/task-events"; - -let n = 0; -function event(type: string, data: Record = {}, at = `2026-03-10T12:00:${String(n).padStart(2, "0")}Z`): StoredEvent { - n += 1; - return { id: n, dot_id: "d1", type, data, source: "guest", guest_seq: n, created_at: at } as StoredEvent; -} - -describe("which events belong to a task", () => { - it("are the ones that name it, an approval's answer among them", () => { - expect(isTaskEvent(event("task.progress", { task_id: "t1", text: "x" }))).toBe(true); - expect(isTaskEvent(event("tool.called", { task_id: "t1", tool: "exec" }))).toBe(true); - expect(isTaskEvent(event("approval.resolved", { approval_id: "a1", decision: "approve", task_id: "t1" }))).toBe(true); - expect(isTaskEvent(event("approval.resolved", { approval_id: "a2", decision: "approve" }))).toBe(false); - expect(isTaskEvent(event("tool.called", { tool: "exec" }))).toBe(false); - expect(isTaskEvent(event("message.assistant", { text: "hi" }))).toBe(false); - expect(taskIdOf(event("task.started", { task_id: "t9" }))).toBe("t9"); - expect(taskIdOf(event("agent.state", { state: "IDLE" }))).toBe(""); - }); - - it("merge in id order without repeating one", () => { - const a = event("task.started", { task_id: "t1" }); - const b = event("task.progress", { task_id: "t1", text: "one" }); - const chat = event("message.assistant", { text: "not a task" }); - const merged = mergeTaskEvents([b], [a, b, chat]); - expect(merged.map((e) => e.id)).toEqual([a.id, b.id]); - }); -}); - -describe("readTaskStory", () => { - /** The control plane's `types` and `task_id` filters: the limit counts what is kept. */ - function serving(log: StoredEvent[]) { - const asked: Array<{ after?: number; before?: number; limit?: number; types?: readonly string[]; taskId?: string; order?: string }> = []; - return { - asked, - async events(_dot: string, options: { after?: number; before?: number; limit?: number; types?: readonly string[]; taskId?: string; order?: "asc" | "desc" } = {}) { - asked.push(options); - const kept = log.filter( - (e) => e.id > (options.after ?? 0) && (options.before === undefined || e.id < options.before) && (!options.types || options.types.includes(e.type)) && (options.taskId === undefined || e.data.task_id === options.taskId), - ); - return (options.order === "desc" ? [...kept].reverse() : kept).slice(0, options.limit); - }, - }; - } - - it("asks for the events of that task and of the types of a story, page by page to the end, and nothing else crosses the wire", async () => { - const log: StoredEvent[] = []; - for (let i = 0; i < 3 * MAX_EVENT_PAGE + 300; i++) { - log.push(i % 3 === 0 ? event("task.progress", { task_id: "t1", text: `p${i}` }) : i % 3 === 1 ? event("task.progress", { task_id: "t2", text: `q${i}` }) : event("agent.state", { state: "IDLE" })); - } - const client = serving(log); - const read = await readTaskStory(client, "d1", "t1"); - expect(read).toHaveLength(MAX_EVENT_PAGE + 100); - expect(read.every((e) => e.type === "task.progress" && e.data.task_id === "t1")).toBe(true); - expect(client.asked.map((a) => a.types)).toEqual([TASK_EVENT_TYPES, TASK_EVENT_TYPES]); - expect(client.asked.map((a) => a.taskId)).toEqual(["t1", "t1"]); - expect(client.asked.map((a) => a.limit)).toEqual([MAX_EVENT_PAGE, MAX_EVENT_PAGE]); - expect(client.asked[1]!.after).toBe(read[MAX_EVENT_PAGE - 1]!.id); - }); - - it("holds the approval and its answer, and no tool call or approval of the chat", async () => { - const log = [ - event("tool.called", { tool: "exec", task_id: "t1" }), - event("tool.called", { tool: "exec" }), - event("approval.requested", { approval_id: "a1", tool: "exec" }), - event("approval.requested", { approval_id: "a2", tool: "exec", task_id: "t1" }), - event("approval.resolved", { approval_id: "a1", decision: "approve" }), - event("approval.resolved", { approval_id: "a2", decision: "approve", task_id: "t1" }), - event("message.assistant", { text: "hi" }), - ]; - const read = await readTaskStory(serving(log), "d1", "t1"); - expect(read.map((e) => e.id)).toEqual([log[0]!.id, log[3]!.id, log[5]!.id]); - }); - - it("ends on a page that is exactly full only after asking once more", async () => { - const log = Array.from({ length: MAX_EVENT_PAGE }, (_, i) => event("task.progress", { task_id: "t1", text: String(i) })); - const client = serving(log); - expect(await readTaskStory(client, "d1", "t1")).toHaveLength(MAX_EVENT_PAGE); - expect(client.asked).toHaveLength(2); - }); - - it("lets a failure through", async () => { - await expect( - readTaskStory( - { - async events() { - throw new Error("the log is down"); - }, - }, - "d1", - "t1", - ), - ).rejects.toThrow("the log is down"); - }); -}); - -describe("readTaskProgress", () => { - it("asks once for the newest progress line of that task, and is null for a task that has said nothing", async () => { - const log = [ - event("task.progress", { task_id: "t1", text: "first" }), - event("task.progress", { task_id: "t2", text: "other task" }), - event("task.progress", { task_id: "t1", text: "second" }), - event("tool.called", { task_id: "t1", tool: "exec" }), - ]; - const asked: unknown[] = []; - const client = { - async events(_dot: string, options: { limit?: number; types?: readonly string[]; taskId?: string; order?: "asc" | "desc" } = {}) { - asked.push(options); - const kept = log.filter((e) => options.types?.includes(e.type) && e.data.task_id === options.taskId); - return (options.order === "desc" ? [...kept].reverse() : kept).slice(0, options.limit); - }, - }; - expect(await readTaskProgress(client, "d1", "t1")).toMatchObject({ id: log[2]!.id, text: "second" }); - expect(asked).toEqual([{ limit: 1, types: ["task.progress"], taskId: "t1", order: "desc" }]); - expect((await readTaskProgress(client, "d1", "t2"))?.text).toBe("other task"); - expect(await readTaskProgress(client, "d1", "t3")).toBeNull(); - }); -}); - -describe("storyOf", () => { - const events = [ - event("task.created", { task_id: "t1", description: "Write it", priority: 10 }), - event("task.started", { task_id: "t1" }), - event("task.progress", { task_id: "t1", text: "Reading the sources" }), - event("tool.called", { task_id: "t1", tool: "exec", permission: "computer.exec", decision: "allow", ok: true, duration_ms: 1200, target: "ls -la" }), - event("tool.called", { task_id: "t2", tool: "exec", decision: "allow", ok: true, duration_ms: 5 }), - event("approval.requested", { task_id: "t1", approval_id: "a1", tool: "write_file", permission: "files.write", reason: "needs to save the report", arguments: {} }), - event("approval.resolved", { task_id: "t1", approval_id: "a1", decision: "approve", note: "go on" }), - event("approval.resolved", { task_id: "t2", approval_id: "other", decision: "reject" }), - event("tool.called", { task_id: "t1", tool: "browser_navigate", decision: "deny", ok: false, duration_ms: 0 }), - event("tool.called", { task_id: "t1", tool: "exec", decision: "allow", ok: false, duration_ms: 0, interrupted: true }), - event("task.completed", { task_id: "t1", summary: "Done: **report.md**" }), - ]; - - it("lists one task's steps in order, each typed, and none of another task's", () => { - const story = storyOf(events, "t1"); - expect(story.map((s) => s.kind)).toEqual(["created", "started", "progress", "tool", "approval", "tool", "tool", "completed"]); - expect(story[0]).toMatchObject({ kind: "created", description: "Write it", priority: 10 }); - expect(story[2]).toMatchObject({ kind: "progress", text: "Reading the sources" }); - expect(story[3]).toMatchObject({ kind: "tool", tool: "exec", label: "Ran a command", target: "ls -la", state: "ok", durationMs: 1200 }); - expect(story[7]).toMatchObject({ kind: "completed", summary: "Done: **report.md**" }); - }); - - it("says a call that started a terminal session as that", () => { - const [step] = storyOf([event("tool.called", { task_id: "t9", tool: "exec", decision: "allow", ok: true, duration_ms: 5, target: "python3", tty: true })], "t9"); - expect(step).toMatchObject({ kind: "tool", label: "Started a terminal session" }); - expect(storyOf(events, "t1")[3]).toMatchObject({ kind: "tool", label: "Ran a command" }); - }); - - it("folds an approval and its answer into one step, found by the approval's id", () => { - const approval = storyOf(events, "t1").find((s) => s.kind === "approval"); - expect(approval).toMatchObject({ approvalId: "a1", tool: "write_file", reason: "needs to save the report", outcome: "approved", note: "go on" }); - const waiting = storyOf(events.slice(0, 6), "t1").find((s) => s.kind === "approval"); - expect(waiting).toMatchObject({ outcome: "waiting", note: "" }); - }); - - it("keeps a refused call and an interrupted one for what they are", () => { - const tools = storyOf(events, "t1").filter((s) => s.kind === "tool"); - expect(tools[1]).toMatchObject({ tool: "browser_navigate", state: "denied" }); - expect(tools[2]).toMatchObject({ tool: "exec", state: "interrupted" }); - }); - - it("ends a failed task with its error and a cancelled one with that", () => { - const failed = storyOf([event("task.failed", { task_id: "t5", error: "cost cap reached" })], "t5"); - expect(failed).toEqual([expect.objectContaining({ kind: "failed", error: "cost cap reached" })]); - const cancelled = storyOf([event("task.cancelled", { task_id: "t6" })], "t6"); - expect(cancelled.map((s) => s.kind)).toEqual(["cancelled"]); - expect(storyOf(events, "nope")).toEqual([]); - }); -}); diff --git a/apps/web/test/task-form.test.ts b/apps/web/test/task-form.test.ts deleted file mode 100644 index 508d32f4..00000000 --- a/apps/web/test/task-form.test.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { EMPTY_TASK_FORM, localInputToIso, parsePriority, taskFormProblem, taskRequest } from "../src/lib/task-form"; - -describe("localInputToIso", () => { - it("turns local wall time into an instant, and nothing into nothing", () => { - expect(localInputToIso("")).toBeUndefined(); - expect(localInputToIso("not a date")).toBeUndefined(); - expect(localInputToIso("2026-03-10T09:30")).toBe(new Date(2026, 2, 10, 9, 30).toISOString()); - }); -}); - -describe("parsePriority", () => { - it("takes whole numbers, signed, and refuses everything else", () => { - expect(parsePriority("10")).toBe(10); - expect(parsePriority(" -10 ")).toBe(-10); - expect(parsePriority("0")).toBe(0); - for (const bad of ["", "1.5", "high", "1e3", "--1", "12345678901"]) expect(parsePriority(bad)).toBeNull(); - }); -}); - -describe("the New task form", () => { - it("cannot be sent without a description, or with a priority that is no number, or a date that is none", () => { - expect(taskFormProblem(EMPTY_TASK_FORM)).toMatch(/what the Dot should do/); - expect(taskFormProblem({ ...EMPTY_TASK_FORM, description: " " })).toMatch(/what the Dot should do/); - expect(taskFormProblem({ ...EMPTY_TASK_FORM, description: "x", priority: "soon" })).toMatch(/whole number/); - expect(taskFormProblem({ ...EMPTY_TASK_FORM, description: "x", notBefore: "tomorrow" })).toMatch(/date and time/); - expect(taskFormProblem({ ...EMPTY_TASK_FORM, description: "x" })).toBeNull(); - }); - - it("asks for the description alone when the rest is as the API has it by default", () => { - expect(taskRequest({ description: " Write the report ", priority: "0", notBefore: "" })).toEqual({ description: "Write the report" }); - }); - - it("sends a priority that is not normal and a time that is set", () => { - expect(taskRequest({ description: "x", priority: "10", notBefore: "2026-03-10T09:30" })).toEqual({ - description: "x", - priority: 10, - scheduled_at: new Date(2026, 2, 10, 9, 30).toISOString(), - }); - expect(taskRequest({ description: "x", priority: "-10", notBefore: "" })).toEqual({ description: "x", priority: -10 }); - }); -}); diff --git a/apps/web/test/task-view.test.ts b/apps/web/test/task-view.test.ts deleted file mode 100644 index 9db6f4f4..00000000 --- a/apps/web/test/task-view.test.ts +++ /dev/null @@ -1,117 +0,0 @@ -import { readFileSync } from "node:fs"; -import { dirname, join } from "node:path"; -import { fileURLToPath } from "node:url"; -import { describe, expect, it } from "vitest"; -import { filterHistory, groupTasks, HISTORY_PAGE, isFinished, isRunning, priorityLabel, PRIORITIES, sectionOf, statusLabel, statusTone, workedSeconds } from "../src/lib/task-view"; -import type { Task } from "../src/lib/types"; -import { taskRecord as task } from "./support/control-plane"; - -const NOW = Date.parse("2026-03-10T12:00:00Z"); -const iso = (offsetMin: number) => new Date(NOW + offsetMin * 60_000).toISOString(); - -describe("which section a task is in", () => { - it("puts working and waiting tasks in Running, finished ones in History", () => { - expect(sectionOf(task("a", { status: "RUNNING" }), NOW)).toBe("running"); - expect(sectionOf(task("a", { status: "WAITING_APPROVAL" }), NOW)).toBe("running"); - for (const status of ["COMPLETED", "FAILED", "CANCELLED"] as const) expect(sectionOf(task("a", { status }), NOW)).toBe("history"); - }); - - it("separates a pending task that has a time ahead of it from the queue", () => { - expect(sectionOf(task("a", { scheduled_at: iso(30) }), NOW)).toBe("scheduled"); - expect(sectionOf(task("a", { scheduled_at: iso(-30) }), NOW)).toBe("queue"); - expect(sectionOf(task("a"), NOW)).toBe("queue"); - }); - - it("agrees with the helpers the rest of the page asks", () => { - expect(isRunning("WAITING_APPROVAL")).toBe(true); - expect(isRunning("PENDING")).toBe(false); - expect(isFinished("CANCELLED")).toBe(true); - expect(isFinished("RUNNING")).toBe(false); - }); -}); - -describe("groupTasks", () => { - const tasks = [ - task("old-low", { priority: -10, created_at: iso(-300) }), - task("new-urgent", { priority: 100, created_at: iso(-10) }), - task("old-normal", { created_at: iso(-200) }), - task("new-normal", { created_at: iso(-20) }), - task("later", { scheduled_at: iso(600) }), - task("sooner", { scheduled_at: iso(60) }), - task("run-late", { status: "RUNNING", started_at: iso(-5) }), - task("run-early", { status: "WAITING_APPROVAL", started_at: iso(-50) }), - task("done-old", { status: "COMPLETED", finished_at: iso(-500) }), - task("done-new", { status: "FAILED", finished_at: iso(-15) }), - ]; - const sections = groupTasks(tasks, NOW); - const ids = (list: Task[]) => list.map((t) => t.id); - - it("queues in the order the dispatcher takes them: priority, then age", () => { - expect(ids(sections.queue)).toEqual(["new-urgent", "old-normal", "new-normal", "old-low"]); - }); - - it("breaks a tie of priority and creation time by id, as the dispatcher does, whatever order the host listed them in", () => { - const at = iso(-40); - const tied = (order: string[]) => ids(groupTasks(order.map((id) => task(id, { created_at: at })), NOW).queue); - expect(tied(["c", "a", "b"])).toEqual(["a", "b", "c"]); - expect(tied(["a", "b", "c"])).toEqual(["a", "b", "c"]); - }); - - it("is pinned to the dispatcher's own ORDER BY: it fails when CLAIM_SQL changes and this order does not", () => { - const source = readFileSync(join(dirname(fileURLToPath(import.meta.url)), "../../../packages/database/src/tasks.ts"), "utf8"); - const claim = source.slice(source.indexOf("export const CLAIM_SQL")); - expect(/ORDER BY ([^\n]+)/.exec(claim)?.[1]).toBe("t.priority DESC, t.created_at, t.id"); - }); - - it("orders the rest the way a person looks for them", () => { - expect(ids(sections.running)).toEqual(["run-early", "run-late"]); - expect(ids(sections.scheduled)).toEqual(["sooner", "later"]); - expect(ids(sections.history)).toEqual(["done-new", "done-old"]); - }); - - it("loses and invents nothing", () => { - expect(Object.values(sections).flat()).toHaveLength(tasks.length); - }); -}); - -describe("the history filter", () => { - const history = [task("a", { status: "COMPLETED" }), task("b", { status: "FAILED" }), task("c", { status: "CANCELLED" }), task("d", { status: "COMPLETED" })]; - - it("keeps the tasks that ended that way, or all of them", () => { - expect(filterHistory(history, "all")).toHaveLength(4); - expect(filterHistory(history, "COMPLETED").map((t) => t.id)).toEqual(["a", "d"]); - expect(filterHistory(history, "FAILED").map((t) => t.id)).toEqual(["b"]); - expect(filterHistory(history, "CANCELLED").map((t) => t.id)).toEqual(["c"]); - }); - - it("is shown a page at a time", () => { - expect(HISTORY_PAGE).toBeGreaterThan(5); - }); -}); - -describe("how a task reads", () => { - it("has a word and a tone for every state, and passes an unknown one through", () => { - expect(statusLabel("PENDING")).toBe("Queued"); - expect(statusLabel("WAITING_APPROVAL")).toBe("Waiting for you"); - expect(statusTone("FAILED")).toBe("error"); - expect(statusTone("COMPLETED")).toBe("ok"); - expect(statusTone("WAITING_APPROVAL")).toBe("warn"); - expect(statusLabel("SOMETHING_NEW")).toBe("SOMETHING_NEW"); - expect(statusTone("SOMETHING_NEW")).toBe("neutral"); - }); - - it("names the four priorities, lowest first, and shows any other number as it is", () => { - expect(PRIORITIES.map((p) => p.label)).toEqual(["Low", "Normal", "High", "Urgent"]); - expect(PRIORITIES.map((p) => p.value)).toEqual([...PRIORITIES.map((p) => p.value)].sort((a, b) => a - b)); - expect(priorityLabel(0)).toBe("Normal"); - expect(priorityLabel(100)).toBe("Urgent"); - expect(priorityLabel(7)).toBe("Priority 7"); - }); - - it("counts the time worked from the start to the end, or to now while it runs", () => { - expect(workedSeconds({ started_at: null, finished_at: null }, NOW)).toBeNull(); - expect(workedSeconds({ started_at: iso(-2), finished_at: null }, NOW)).toBe(120); - expect(workedSeconds({ started_at: iso(-10), finished_at: iso(-7) }, NOW)).toBe(180); - expect(workedSeconds({ started_at: iso(5), finished_at: null }, NOW)).toBe(0); - }); -}); diff --git a/apps/web/test/tasks.test.tsx b/apps/web/test/tasks.test.tsx deleted file mode 100644 index 0ed25603..00000000 --- a/apps/web/test/tasks.test.tsx +++ /dev/null @@ -1,537 +0,0 @@ -// @vitest-environment jsdom -import { act, cleanup, render, screen, waitFor, within } from "@testing-library/react"; -import { TASK_LIST_LIMIT } from "@invisible-dots/shared/browser"; -import userEvent from "@testing-library/user-event"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { DotShell } from "../src/components/DotShell"; -import { DotEventScope, EventStreamProvider } from "../src/components/events"; -import { AttentionProvider } from "../src/components/shell/attention"; -import { TaskDrawer } from "../src/components/tasks/task-drawer"; -import { TasksView } from "../src/components/tasks/TasksView"; -import { Toaster } from "../src/components/ui/sonner"; -import { stubMatchMedia } from "./support/browser"; -import { approvalRecord, dotRecord, FakeControlPlane, taskRecord } from "./support/control-plane"; - -const router = { push: vi.fn(), replace: vi.fn() }; -let segment: string | null = null; -vi.mock("next/navigation", () => ({ - usePathname: () => "/dots/d1/tasks", - useRouter: () => router, - useSelectedLayoutSegment: () => segment, -})); - -let plane: FakeControlPlane; - -beforeEach(() => { - segment = null; - router.push.mockClear(); - plane = new FakeControlPlane(); - plane.dots = [dotRecord("d1", { name: "fares" })]; - plane.install(); - stubMatchMedia(); -}); - -afterEach(() => { - cleanup(); - vi.unstubAllGlobals(); - vi.restoreAllMocks(); -}); - -const minutesAgo = (minutes: number) => new Date(Date.now() - minutes * 60_000).toISOString(); -const minutesAhead = (minutes: number) => new Date(Date.now() + minutes * 60_000).toISOString(); -const requested = (pattern: RegExp) => plane.requests.filter((r) => pattern.test(r)); - -/** The Tasks page as the Dot's layout puts it: inside the Dot's shell, with the drawer's page as its child. */ -async function renderTasks(drawerOf: string | null = null) { - segment = drawerOf; - render( - - - - - {drawerOf ? : null} - - - - - , - ); - await screen.findByRole("heading", { level: 1, hidden: true }); - await waitFor(() => expect(plane.streamOpen).toBe(true)); -} - -function runningTask(id: string, change: Partial> = {}) { - return taskRecord(id, { status: "RUNNING", started_at: minutesAgo(3), spent_usd: 0.12, description: `Running ${id}`, ...change }); -} - -describe("the Tasks page", () => { - it("invites the first task when there are none, and does not read the event log for nothing", async () => { - await renderTasks(); - expect(await screen.findByText("No tasks yet")).toBeTruthy(); - expect(screen.getByRole("button", { name: "New task" })).toBeTruthy(); - expect(plane.eventQueries).toEqual([]); - }); - - it("says so when the tasks cannot be loaded", async () => { - plane.tasks = []; - const real = plane.fetch; - vi.stubGlobal( - "fetch", - vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => - String(input).endsWith("/tasks") && (init?.method ?? "GET") === "GET" ? Response.json({ error: "down", message: "the control plane said no" }, { status: 500 }) : real(input, init), - ), - ); - await renderTasks(); - expect((await screen.findByText("Could not load the tasks")).closest("[role=alert]")?.textContent).toContain("the control plane said no"); - }); - - it("puts each task in its section: running, scheduled, queued in dispatch order, and finished", async () => { - plane.tasks = [ - runningTask("r1"), - taskRecord("q-low", { description: "Queued low", priority: -10, created_at: minutesAgo(50) }), - taskRecord("q-urgent", { description: "Queued urgent", priority: 100, created_at: minutesAgo(10) }), - taskRecord("q-normal", { description: "Queued normal", created_at: minutesAgo(30) }), - taskRecord("s1", { description: "Scheduled later", scheduled_at: minutesAhead(185) }), - taskRecord("h1", { description: "Finished well", status: "COMPLETED", finished_at: minutesAgo(5), started_at: minutesAgo(9), summary: "All done", spent_usd: 0.4 }), - ]; - await renderTasks(); - const running = await screen.findByRole("region", { name: /^Running/ }); - expect(within(running).getByRole("article", { name: "Running r1" })).toBeTruthy(); - - const queue = screen.getByRole("region", { name: /^Queue/ }); - const queued = within(queue).getAllByRole("article"); - expect(queued.map((q) => q.getAttribute("aria-label"))).toEqual(["Queued urgent", "Queued normal", "Queued low"]); - expect(within(queued[0]!).getByText("Next")).toBeTruthy(); - expect(within(queued[1]!).getByText("#2")).toBeTruthy(); - expect(within(queued[0]!).getByText("Urgent")).toBeTruthy(); - expect(within(queued[2]!).getByText("Low")).toBeTruthy(); - - const scheduled = screen.getByRole("region", { name: /^Scheduled/ }); - expect(within(scheduled).getByText(/Starts/).textContent).toContain("in 3h"); - - const history = screen.getByRole("region", { name: /^History/ }); - const row = within(history).getByRole("row", { name: /Finished well/ }); - expect(within(row).getByText("Completed")).toBeTruthy(); - expect(within(row).getByText("$0.40")).toBeTruthy(); - expect(within(row).getByText("All done")).toBeTruthy(); - expect(within(row).getByText("4m 0s")).toBeTruthy(); - }); - - it("lists the newest page, and reads the older ones when asked, joining them to what is shown", async () => { - // The tasks of the Queue region's ordered list, read from the DOM: a role query over two hundred cards costs a - // few hundred milliseconds in jsdom, and waitFor repeats it every 50, which alone used up the test's time. The - // region is found again at each call (by its heading, a text query), since it is drawn anew when the list grows. - const queueItems = () => screen.getByText("Queue", { selector: "h2" }).closest("section")!.querySelectorAll(":scope ol > li"); - // 205 queued tasks, one created a minute after the other: the newest 200 are the first page, five are past it. - const base = Date.now() - 3_600_000; - plane.tasks = Array.from({ length: TASK_LIST_LIMIT + 5 }, (_, i) => taskRecord(`q${String(i).padStart(3, "0")}`, { created_at: new Date(base + i * 1000).toISOString() })); - await renderTasks(); - // Found by its words, then checked to be a status: a page-wide role query is what costs the time here. - const notice = (await screen.findByText(new RegExp(`The newest ${TASK_LIST_LIMIT} tasks are listed`))).closest("[role=status]")!; - expect(notice).not.toBeNull(); - expect(queueItems()).toHaveLength(TASK_LIST_LIMIT); - expect(screen.queryByText("task q000")).toBeNull(); - const pagedQueries = () => plane.taskQueries.filter((query) => query.before !== null); - expect(pagedQueries()).toEqual([]); - - await userEvent.click(within(notice).getByRole("button", { name: "Show older tasks" })); - await waitFor(() => expect(queueItems()).toHaveLength(TASK_LIST_LIMIT + 5)); - expect(screen.getByText("task q000")).toBeTruthy(); - // The page went on after the last task of the newest one, and the list ended there: nothing more to ask for. - expect(pagedQueries()).toEqual([{ limit: null, before: "q005" }]); - expect(screen.queryByRole("button", { name: "Show older tasks" })).toBeNull(); - - // A task that arrives is read with the newest page again, and the older ones stay. - plane.tasks.push(taskRecord("q999", { created_at: new Date().toISOString() })); - act(() => plane.push("d1", "task.created", { task_id: "q999" })); - // Waited for by the list's length, as above: a page-wide text query over two hundred cards, repeated every 50 ms, - // used up waitFor's second on a loaded machine before the refresh (300 ms after the event) had drawn the list. - await waitFor(() => expect(queueItems()).toHaveLength(TASK_LIST_LIMIT + 6)); - expect(screen.getByText("task q999")).toBeTruthy(); - expect(screen.getByText("task q000")).toBeTruthy(); - }); - - it("does not offer older tasks to a Dot whose tasks fit a page", async () => { - plane.tasks = Array.from({ length: TASK_LIST_LIMIT - 1 }, (_, i) => taskRecord(`f${i}`)); - await renderTasks(); - await screen.findByRole("region", { name: /^Queue/ }); - expect(screen.queryByRole("button", { name: "Show older tasks" })).toBeNull(); - expect(screen.queryByText(/tasks are listed/)).toBeNull(); - }); - - it("links every task to its own address", async () => { - plane.tasks = [runningTask("r1"), taskRecord("q1"), taskRecord("h1", { status: "FAILED", finished_at: minutesAgo(1), error: "boom" })]; - await renderTasks(); - for (const description of ["Running r1", "task q1", "task h1"]) { - const id = description.split(" ")[1]; - expect(screen.getByRole("link", { name: description }).getAttribute("href")).toBe(`/dots/d1/tasks/${id}`); - } - }); -}); - -describe("the live progress line", () => { - it("shows what the task last reported, from the log, and follows each new report without a reload", async () => { - plane.tasks = [runningTask("r1")]; - plane.store("d1", "task.progress", { task_id: "r1", text: "Reading the first source" }, minutesAgo(2)); - plane.store("d1", "task.progress", { task_id: "other", text: "Not this task" }, minutesAgo(1)); - await renderTasks(); - const card = await screen.findByRole("article", { name: "Running r1" }); - await waitFor(() => expect(within(card).getByText("Reading the first source")).toBeTruthy()); - expect(within(card).queryByText("Not this task")).toBeNull(); - expect(within(card).getByText("$0.12")).toBeTruthy(); - - plane.tasks[0]!.spent_usd = 0.31; - act(() => plane.push("d1", "task.progress", { task_id: "r1", text: "Writing the summary", spent_usd: 0.31 })); - await waitFor(() => expect(within(card).getByText("Writing the summary")).toBeTruthy()); - expect(within(card).queryByText("Reading the first source")).toBeNull(); - await waitFor(() => expect(within(card).getByText("$0.31")).toBeTruthy()); - // One request for the newest report of that task; the second line came from the stream. - expect(plane.eventQueries).toEqual([{ after: 0, limit: 1, types: ["task.progress"], tools: null, taskId: "r1", order: "desc" }]); - }); - - it("says plainly that a running task has not reported yet", async () => { - plane.tasks = [runningTask("r1")]; - await renderTasks(); - const card = await screen.findByRole("article", { name: "Running r1" }); - await waitFor(() => expect(within(card).getByText(/has not reported anything yet/)).toBeTruthy()); - }); - - it("says so when the log cannot be read, and does not pretend the task is quiet", async () => { - plane.tasks = [runningTask("r1")]; - plane.failEvents = 500; - await renderTasks(); - const card = await screen.findByRole("article", { name: "Running r1" }); - await waitFor(() => expect(within(card).getByText(/could not be read/)).toBeTruthy()); - expect(within(card).queryByText(/has not reported anything yet/)).toBeNull(); - }); - - it("moves a task to the history when it ends, with its summary", async () => { - plane.tasks = [runningTask("r1")]; - await renderTasks(); - await screen.findByRole("article", { name: "Running r1" }); - Object.assign(plane.tasks[0]!, { status: "COMPLETED", finished_at: new Date().toISOString(), summary: "Report written" }); - act(() => plane.push("d1", "task.completed", { task_id: "r1", summary: "Report written" })); - const history = await screen.findByRole("region", { name: /^History/ }); - await waitFor(() => expect(within(history).getByText("Report written")).toBeTruthy()); - expect(screen.queryByRole("article", { name: "Running r1" })).toBeNull(); - expect(screen.getByText("Nothing is running.")).toBeTruthy(); - }); - - it("shows the approval a task waits for as a card on the task, answers it there and keeps the receipt in place", async () => { - plane.tasks = [runningTask("r1", { status: "WAITING_APPROVAL" }), runningTask("r2")]; - plane.approvals = [ - approvalRecord("a1", "d1", { task_id: "r1", tool: "write_file", permission: "files.write", arguments: { path: "report.md", content: "hello" }, reason: "to save the report" }), - approvalRecord("a2", "d1", { task_id: "r2", tool: "exec", permission: "computer.exec", arguments: { command: "ls" } }), - ]; - await renderTasks(); - const card = await screen.findByRole("article", { name: "Running r1" }); - expect(within(card).getAllByText("Waiting for you").length).toBeGreaterThan(0); - const approval = await within(card).findByRole("article", { name: "Wants to write a file" }); - expect(within(approval).getByText("to save the report")).toBeTruthy(); - // The other task's approval is on the other task's card. - expect(within(card).queryByRole("article", { name: "Wants to run a command" })).toBeNull(); - expect(within(await screen.findByRole("article", { name: "Running r2" })).getByRole("article", { name: "Wants to run a command" })).toBeTruthy(); - - await userEvent.click(within(approval).getByRole("button", { name: "Allow once" })); - expect(plane.answers).toEqual([{ id: "a1", decision: "approve", body: {} }]); - // The host no longer lists it as waiting, and the card still says what became of it. - await waitFor(() => expect(within(card).getByRole("status").textContent).toBe("Allowed")); - await waitFor(() => expect(plane.requests.filter((r) => r === "GET /api/approvals").length).toBeGreaterThan(1)); - expect(within(card).getByRole("status").textContent).toBe("Allowed"); - }); - - it("shows the approval in the task's drawer too, above what happened", async () => { - plane.tasks = [runningTask("r1", { status: "WAITING_APPROVAL" })]; - plane.approvals = [approvalRecord("a1", "d1", { task_id: "r1", tool: "exec", permission: "computer.exec", arguments: { command: "make test" } })]; - await renderTasks("r1"); - const drawer = await screen.findByRole("dialog", { name: "Running r1" }); - const approval = await within(drawer).findByRole("article", { name: "Wants to run a command" }); - expect(within(approval).getByText("make test")).toBeTruthy(); - }); -}); - -describe("cancelling", () => { - it("asks first, and does nothing when the person keeps the task", async () => { - plane.tasks = [runningTask("r1")]; - await renderTasks(); - const card = await screen.findByRole("article", { name: "Running r1" }); - await userEvent.click(within(card).getByRole("button", { name: /^Cancel/ })); - const dialog = await screen.findByRole("dialog", { name: "Cancel this task?" }); - expect(dialog.textContent).toContain("The Dot stops working on it now"); - await userEvent.click(within(dialog).getByRole("button", { name: "Keep it" })); - await waitFor(() => expect(screen.queryByRole("dialog")).toBeNull()); - expect(requested(/POST \/api\/tasks\//)).toEqual([]); - }); - - it("cancels once confirmed: the task leaves Running and shows in the history as cancelled", async () => { - plane.tasks = [runningTask("r1")]; - await renderTasks(); - const card = await screen.findByRole("article", { name: "Running r1" }); - await userEvent.click(within(card).getByRole("button", { name: /^Cancel/ })); - await userEvent.click(within(await screen.findByRole("dialog")).getByRole("button", { name: "Cancel task" })); - await waitFor(() => expect(requested(/POST \/api\/tasks\/r1\/cancel/)).toHaveLength(1)); - const history = await screen.findByRole("region", { name: /^History/ }); - await waitFor(() => expect(within(history).getByText("Cancelled", { selector: "span" })).toBeTruthy()); - expect(screen.queryByRole("article", { name: "Running r1" })).toBeNull(); - }); - - it("tells a queued task's consequence differently, and shows a refusal inside the dialog", async () => { - plane.tasks = [taskRecord("q1", { description: "Wait in line" })]; - plane.failCancel = { status: 409, error: "task_finished", message: "task q1 is already COMPLETED" }; - await renderTasks(); - const card = await screen.findByRole("article", { name: "Wait in line" }); - await userEvent.click(within(card).getByRole("button", { name: /^Cancel/ })); - const dialog = await screen.findByRole("dialog"); - expect(dialog.textContent).toContain("It will not run."); - await userEvent.click(within(dialog).getByRole("button", { name: "Cancel task" })); - expect((await within(dialog).findByRole("alert")).textContent).toContain("task q1 is already COMPLETED"); - expect(screen.getByRole("dialog")).toBeTruthy(); - }); -}); - -describe("the history", () => { - function finished(count: number) { - return Array.from({ length: count }, (_, i) => - taskRecord(`h${i}`, { - description: `Finished ${i}`, - status: i % 5 === 0 ? "FAILED" : i % 7 === 0 ? "CANCELLED" : "COMPLETED", - finished_at: minutesAgo(i + 1), - started_at: minutesAgo(i + 2), - error: i % 5 === 0 ? "ran out of budget" : null, - }), - ); - } - - it("shows a page of twenty and the next twenty when asked", async () => { - plane.tasks = finished(45); - await renderTasks(); - const history = await screen.findByRole("region", { name: /^History/ }); - expect(within(history).getAllByRole("row")).toHaveLength(1 + 20); - await userEvent.click(within(history).getByRole("button", { name: "Show 20 more" })); - expect(within(history).getAllByRole("row")).toHaveLength(1 + 40); - await userEvent.click(within(history).getByRole("button", { name: "Show 5 more" })); - expect(within(history).getAllByRole("row")).toHaveLength(1 + 45); - expect(within(history).queryByRole("button", { name: /Show/ })).toBeNull(); - }); - - it("filters by how the tasks ended, and shows an error as the Dot gave it", async () => { - plane.tasks = finished(12); - await renderTasks(); - const history = await screen.findByRole("region", { name: /^History/ }); - await userEvent.click(within(history).getByRole("button", { name: "Failed" })); - expect(within(history).getByRole("button", { name: "Failed" }).getAttribute("aria-pressed")).toBe("true"); - const rows = within(history).getAllByRole("row").slice(1); - expect(rows).toHaveLength(3); - for (const row of rows) { - expect(within(row).getByText("Failed")).toBeTruthy(); - expect(within(row).getByText("ran out of budget")).toBeTruthy(); - } - await userEvent.click(within(history).getByRole("button", { name: "Cancelled" })); - expect(within(history).getAllByRole("row")).toHaveLength(1 + 1); - plane.tasks = []; - }); - - it("says when no finished task matches", async () => { - plane.tasks = [taskRecord("h1", { status: "COMPLETED", finished_at: minutesAgo(1) })]; - await renderTasks(); - const history = await screen.findByRole("region", { name: /^History/ }); - await userEvent.click(within(history).getByRole("button", { name: "Failed" })); - expect(within(history).getByText("No failed tasks.")).toBeTruthy(); - }); -}); - -describe("New task", () => { - async function openDialog() { - await renderTasks(); - await screen.findByText("No tasks yet"); - await userEvent.click(screen.getByRole("button", { name: "New task" })); - return screen.findByRole("dialog", { name: "New task" }); - } - - it("will not send an empty description, and says why next to the field", async () => { - const dialog = await openDialog(); - await userEvent.click(within(dialog).getByRole("button", { name: "Create task" })); - expect(within(dialog).getByText("Say what the Dot should do.")).toBeTruthy(); - expect(within(dialog).getByLabelText("What should it do?").getAttribute("aria-invalid")).toBe("true"); - expect(plane.createdTasks).toEqual([]); - }); - - it("sends the description alone for a normal task, closes, and lists the new task", async () => { - const dialog = await openDialog(); - await userEvent.type(within(dialog).getByLabelText("What should it do?"), " Collect the fares "); - expect(within(dialog).getByRole("radio", { name: "Normal" })).toHaveProperty("checked", true); - await userEvent.click(within(dialog).getByRole("button", { name: "Create task" })); - await waitFor(() => expect(plane.createdTasks).toEqual([{ description: "Collect the fares" }])); - await waitFor(() => expect(screen.queryByRole("dialog")).toBeNull()); - expect(await screen.findByRole("article", { name: "Collect the fares" })).toBeTruthy(); - }); - - it("sends a named priority as its number, and a not-before time as an instant", async () => { - const dialog = await openDialog(); - await userEvent.type(within(dialog).getByLabelText("What should it do?"), "Check in the morning"); - await userEvent.click(within(dialog).getByRole("radio", { name: "High" })); - const when = within(dialog).getByLabelText(/Not before/); - await userEvent.type(when, "2030-05-01T09:30"); - await userEvent.click(within(dialog).getByRole("button", { name: "Create task" })); - await waitFor(() => expect(plane.createdTasks).toHaveLength(1)); - expect(plane.createdTasks[0]).toEqual({ description: "Check in the morning", priority: 10, scheduled_at: new Date(2030, 4, 1, 9, 30).toISOString() }); - }); - - it("keeps the raw number under Advanced, in step with the named choice, and refuses what is no number", async () => { - const dialog = await openDialog(); - await userEvent.type(within(dialog).getByLabelText("What should it do?"), "Urgent thing"); - expect(within(dialog).queryByLabelText("Priority as a number")).toBeNull(); - await userEvent.click(within(dialog).getByRole("button", { name: "Advanced" })); - const raw = within(dialog).getByLabelText("Priority as a number"); - await userEvent.click(within(dialog).getByRole("radio", { name: "Urgent" })); - expect((raw as HTMLInputElement).value).toBe("100"); - await userEvent.clear(raw); - await userEvent.type(raw, "7"); - for (const radio of within(dialog).getAllByRole("radio")) expect(radio).toHaveProperty("checked", false); - await userEvent.clear(raw); - await userEvent.type(raw, "soon"); - expect(within(dialog).getByText("The priority is a whole number.")).toBeTruthy(); - await userEvent.click(within(dialog).getByRole("button", { name: "Create task" })); - expect(plane.createdTasks).toEqual([]); - await userEvent.clear(raw); - await userEvent.type(raw, "7"); - await userEvent.click(within(dialog).getByRole("button", { name: "Create task" })); - await waitFor(() => expect(plane.createdTasks).toEqual([{ description: "Urgent thing", priority: 7 }])); - }); - - it("shows what the control plane refused, in the dialog, and keeps what was typed", async () => { - plane.failCreateTask = { status: 400, error: "invalid", message: "the description is too long" }; - const dialog = await openDialog(); - await userEvent.type(within(dialog).getByLabelText("What should it do?"), "A very long thing"); - await userEvent.click(within(dialog).getByRole("button", { name: "Create task" })); - expect((await within(dialog).findByRole("alert")).textContent).toContain("the description is too long"); - expect((within(dialog).getByLabelText("What should it do?") as HTMLTextAreaElement).value).toBe("A very long thing"); - }); - - it("starts empty again after it was closed without sending", async () => { - const dialog = await openDialog(); - await userEvent.type(within(dialog).getByLabelText("What should it do?"), "never mind"); - await userEvent.click(within(dialog).getByRole("button", { name: "Cancel" })); - await waitFor(() => expect(screen.queryByRole("dialog")).toBeNull()); - await userEvent.click(screen.getByRole("button", { name: "New task" })); - expect((within(await screen.findByRole("dialog")).getByLabelText("What should it do?") as HTMLTextAreaElement).value).toBe(""); - }); -}); - -describe("the task drawer", () => { - it("tells the whole story of a finished task: its steps, its result as markdown, its numbers", async () => { - plane.tasks = [ - taskRecord("t1", { - description: "Write the weekly report", - status: "COMPLETED", - priority: 10, - started_at: minutesAgo(12), - finished_at: minutesAgo(2), - spent_usd: 0.73, - summary: "Wrote **report.md** with [the numbers](https://example.com/n).", - }), - ]; - plane.store("d1", "task.created", { task_id: "t1", description: "Write the weekly report", priority: 10 }, minutesAgo(13)); - plane.store("d1", "task.started", { task_id: "t1" }, minutesAgo(12)); - plane.store("d1", "task.progress", { task_id: "t1", text: "Collecting the figures" }, minutesAgo(10)); - plane.store("d1", "tool.called", { task_id: "t1", tool: "exec", permission: "computer.exec", decision: "allow", ok: true, duration_ms: 2500, target: "python3 collect.py" }, minutesAgo(9)); - plane.store("d1", "approval.requested", { task_id: "t1", approval_id: "a1", tool: "write_file", permission: "files.write", reason: "to save report.md", arguments: {} }, minutesAgo(5)); - plane.store("d1", "approval.resolved", { task_id: "t1", approval_id: "a1", decision: "approve" }, minutesAgo(4)); - plane.store("d1", "task.completed", { task_id: "t1", summary: "x" }, minutesAgo(2)); - plane.store("d1", "task.progress", { task_id: "elsewhere", text: "Another task's line" }, minutesAgo(1)); - - await renderTasks("t1"); - const drawer = await screen.findByRole("dialog", { name: "Write the weekly report" }); - await waitFor(() => expect(within(drawer).getByText("Collecting the figures")).toBeTruthy()); - expect(within(drawer).queryByText("Another task's line")).toBeNull(); - expect(within(drawer).getByText("python3 collect.py")).toBeTruthy(); - expect(within(drawer).getByText(/done, 2.5 s/)).toBeTruthy(); - expect(within(drawer).getByText(/Allowed:/)).toBeTruthy(); - // The engine's tool names are in the words of the chat (TOOL_LABELS of packages/shared), the name being the line's title. - expect(within(drawer).getByText("Ran a command").getAttribute("title")).toBe("exec"); - expect(within(drawer).getByText("wrote a file").getAttribute("title")).toBe("write_file"); - expect(within(drawer).getByText("$0.73")).toBeTruthy(); - expect(within(drawer).getByText("High")).toBeTruthy(); - expect(within(drawer).getByText("10m 0s")).toBeTruthy(); - // The result is rendered, not shown as source. - const result = within(drawer).getByRole("region", { name: "Result" }); - expect(result.querySelector("strong")?.textContent).toBe("report.md"); - expect(result.querySelector("a")?.getAttribute("href")).toBe("https://example.com/n"); - // A finished task cannot be cancelled. - expect(within(drawer).queryByRole("button", { name: /^Cancel/ })).toBeNull(); - expect(requested(/GET \/api\/tasks\/t1$/)).toHaveLength(1); - // Its story is read by the task: the rest of the Dot's log is not. - expect(plane.eventQueries.length).toBeGreaterThan(0); - expect(plane.eventQueries.every((q) => q.taskId === "t1")).toBe(true); - }); - - it("shows a failed task's reason as it was given, and a running task's story growing live", async () => { - plane.tasks = [runningTask("t2", { description: "Long job" })]; - plane.store("d1", "task.started", { task_id: "t2" }, minutesAgo(3)); - await renderTasks("t2"); - const drawer = await screen.findByRole("dialog", { name: "Long job" }); - await waitFor(() => expect(within(drawer).getByText("The Dot started on it")).toBeTruthy()); - expect(within(drawer).getByRole("button", { name: /^Cancel/ })).toBeTruthy(); - - act(() => plane.push("d1", "task.progress", { task_id: "t2", text: "Halfway through" })); - await waitFor(() => expect(within(drawer).getByText("Halfway through")).toBeTruthy()); - - Object.assign(plane.tasks[0]!, { status: "FAILED", finished_at: new Date().toISOString(), error: "the per-task cost cap was reached" }); - act(() => plane.push("d1", "task.failed", { task_id: "t2", error: "the per-task cost cap was reached" })); - await waitFor(() => expect(within(drawer).getAllByText("the per-task cost cap was reached").length).toBeGreaterThan(0)); - await waitFor(() => expect(within(drawer).queryByRole("button", { name: /^Cancel/ })).toBeNull()); - }); - - it("says when the task does not exist, and offers the way back", async () => { - plane.tasks = []; - await renderTasks("ghost"); - const drawer = await screen.findByRole("dialog"); - expect((await within(drawer).findByText("This task does not exist")).closest("[role=alert]")).toBeTruthy(); - expect(within(drawer).getByRole("link", { name: "Back to the tasks" }).getAttribute("href")).toBe("/dots/d1/tasks"); - }); - - it("treats a task of another Dot as missing under this Dot's address, and shows nothing of it", async () => { - plane.dots = [dotRecord("d1", { name: "fares" }), dotRecord("d2", { name: "mailer" })]; - plane.tasks = [taskRecord("theirs", { dot_id: "d2", description: "Their private errand", status: "COMPLETED", finished_at: minutesAgo(1), summary: "Their secret result" })]; - await renderTasks("theirs"); - const drawer = await screen.findByRole("dialog"); - expect((await within(drawer).findByText("This task does not exist")).closest("[role=alert]")).toBeTruthy(); - expect(within(drawer).getByRole("link", { name: "Back to the tasks" }).getAttribute("href")).toBe("/dots/d1/tasks"); - expect(screen.queryByText("Their private errand")).toBeNull(); - expect(screen.queryByText("Their secret result")).toBeNull(); - expect(within(drawer).queryByRole("button", { name: /^Cancel/ })).toBeNull(); - }); - - it("says when the history cannot be read, and reads it again on request", async () => { - plane.tasks = [taskRecord("t3", { description: "Quiet", status: "COMPLETED", finished_at: minutesAgo(1) })]; - plane.failEvents = 500; - await renderTasks("t3"); - const drawer = await screen.findByRole("dialog", { name: "Quiet" }); - await within(drawer).findByText("Could not read the task's history"); - plane.failEvents = null; - plane.store("d1", "task.started", { task_id: "t3" }); - await userEvent.click(within(drawer).getByRole("button", { name: "Try again" })); - await waitFor(() => expect(within(drawer).getByText("The Dot started on it")).toBeTruthy()); - }); - - it("goes back to the list when it is closed", async () => { - plane.tasks = [taskRecord("t4", { description: "Close me", status: "COMPLETED", finished_at: minutesAgo(1) })]; - await renderTasks("t4"); - const drawer = await screen.findByRole("dialog", { name: "Close me" }); - await userEvent.click(within(drawer).getByRole("button", { name: "Close" })); - expect(router.push).toHaveBeenCalledWith("/dots/d1/tasks"); - }); -}); - -describe("the Dot header while a task runs", () => { - it("shows the newest thing the task reported, linked to the task", async () => { - plane.tasks = [runningTask("r1")]; - await renderTasks(); - await screen.findByRole("article", { name: "Running r1" }); - act(() => plane.push("d1", "task.progress", { task_id: "r1", text: "Comparing the fares" })); - const line = await screen.findByRole("link", { name: /Comparing the fares/ }); - expect(line.getAttribute("href")).toBe("/dots/d1/tasks/r1"); - act(() => plane.push("d1", "task.completed", { task_id: "r1", summary: "done" })); - await waitFor(() => expect(screen.queryByRole("link", { name: /Comparing the fares/ })).toBeNull()); - }); -}); diff --git a/apps/web/test/theme.test.tsx b/apps/web/test/theme.test.tsx deleted file mode 100644 index c4a5060b..00000000 --- a/apps/web/test/theme.test.tsx +++ /dev/null @@ -1,139 +0,0 @@ -// @vitest-environment jsdom -import { act, cleanup, render, screen, waitFor } from "@testing-library/react"; -import userEvent from "@testing-library/user-event"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { ThemeToggle } from "../src/components/shell/ThemeToggle"; -import { applyTheme, storedPreference, THEME_BOOTSTRAP, THEME_KEY } from "../src/lib/theme"; -import { useResolvedTheme } from "../src/lib/use-theme"; -import { stubResizeObserver } from "./support/browser"; - -let systemDark: boolean; -let mediaListeners: Set<() => void>; - -beforeEach(() => { - systemDark = false; - mediaListeners = new Set(); - localStorage.clear(); - delete document.documentElement.dataset.theme; - // The toggle's tooltip is positioned, which jsdom cannot measure. - stubResizeObserver(); - vi.stubGlobal( - "matchMedia", - vi.fn(() => ({ - get matches() { - return systemDark; - }, - addEventListener: (_type: string, listener: () => void) => mediaListeners.add(listener), - removeEventListener: (_type: string, listener: () => void) => mediaListeners.delete(listener), - })), - ); -}); - -afterEach(() => { - cleanup(); - vi.unstubAllGlobals(); -}); - -const theme = () => document.documentElement.dataset.theme; - -describe("applying the theme", () => { - it("follows the system when nothing is stored, and obeys what is stored", () => { - applyTheme(THEME_KEY); - expect(theme()).toBe("light"); - systemDark = true; - applyTheme(THEME_KEY); - expect(theme()).toBe("dark"); - localStorage.setItem(THEME_KEY, "light"); - applyTheme(THEME_KEY); - expect(theme()).toBe("light"); - systemDark = false; - localStorage.setItem(THEME_KEY, "dark"); - applyTheme(THEME_KEY); - expect(theme()).toBe("dark"); - }); - - it("follows the system, and does not throw, when the storage is blocked", () => { - vi.spyOn(Storage.prototype, "getItem").mockImplementation(() => { - throw new DOMException("blocked", "SecurityError"); - }); - systemDark = true; - expect(() => applyTheme(THEME_KEY)).not.toThrow(); - expect(theme()).toBe("dark"); - expect(storedPreference()).toBe("system"); - vi.restoreAllMocks(); - }); - - it("ignores a stored value that is neither light nor dark", () => { - localStorage.setItem(THEME_KEY, "purple"); - expect(storedPreference()).toBe("system"); - }); - - it("runs on its own as the inline script of the document head, before any module is loaded", () => { - systemDark = true; - // The script is a string with the function written out in it: nothing it uses may come from this module. - new Function(THEME_BOOTSTRAP)(); - expect(theme()).toBe("dark"); - localStorage.setItem(THEME_KEY, "light"); - new Function(THEME_BOOTSTRAP)(); - expect(theme()).toBe("light"); - }); -}); - -describe("the theme toggle", () => { - function Probe() { - return

{useResolvedTheme()}

; - } - - it("stores the choice, applies it at once and says which one is in force", async () => { - applyTheme(THEME_KEY); - render( - <> - - - , - ); - expect(screen.getByRole("button", { name: "Theme: System" })).toBeTruthy(); - await userEvent.click(screen.getByRole("button", { name: "Theme: System" })); - await userEvent.click(screen.getByRole("menuitemradio", { name: "Dark" })); - expect(localStorage.getItem(THEME_KEY)).toBe("dark"); - expect(theme()).toBe("dark"); - expect(screen.getByTestId("resolved").textContent).toBe("dark"); - expect(screen.getByRole("button", { name: "Theme: Dark" })).toBeTruthy(); - - await userEvent.click(screen.getByRole("button", { name: "Theme: Dark" })); - await userEvent.click(screen.getByRole("menuitemradio", { name: "System" })); - expect(localStorage.getItem(THEME_KEY)).toBeNull(); - expect(theme()).toBe("light"); - }); - - it("follows the system's change while the choice is System, and not otherwise", async () => { - applyTheme(THEME_KEY); - render( - <> - - - , - ); - systemDark = true; - act(() => mediaListeners.forEach((listener) => listener())); - await waitFor(() => expect(screen.getByTestId("resolved").textContent).toBe("dark")); - - // With a choice stored the system's change is not followed. - localStorage.setItem(THEME_KEY, "light"); - applyTheme(THEME_KEY); - await waitFor(() => expect(screen.getByTestId("resolved").textContent).toBe("light")); - act(() => mediaListeners.forEach((listener) => listener())); - expect(theme()).toBe("light"); - }); - - it("still switches this page, though it cannot remember the choice, when the storage is blocked", async () => { - render(); - vi.spyOn(Storage.prototype, "setItem").mockImplementation(() => { - throw new DOMException("blocked", "QuotaExceededError"); - }); - await userEvent.click(screen.getByRole("button", { name: /^Theme:/ })); - await userEvent.click(screen.getByRole("menuitemradio", { name: "Dark" })); - expect(theme()).toBe("dark"); - vi.restoreAllMocks(); - }); -}); diff --git a/apps/web/test/time.test.ts b/apps/web/test/time.test.ts deleted file mode 100644 index 8f986551..00000000 --- a/apps/web/test/time.test.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { relativeTime } from "../src/lib/time"; - -const NOW = Date.parse("2026-03-10T12:00:00Z"); -const at = (ms: number) => new Date(NOW + ms).toISOString(); -const MIN = 60_000; -const HOUR = 60 * MIN; - -describe("relativeTime", () => { - it("says how long ago, in the unit that reads best", () => { - expect(relativeTime(at(-20_000), NOW)).toBe("Just now"); - expect(relativeTime(at(-5 * MIN), NOW)).toBe("5m ago"); - expect(relativeTime(at(-59 * MIN), NOW)).toBe("59m ago"); - expect(relativeTime(at(-3 * HOUR), NOW)).toBe("3h ago"); - }); - - it("gives the date once it is a day old or more", () => { - const old = at(-3 * 24 * HOUR); - expect(relativeTime(old, NOW)).toBe(new Date(old).toLocaleDateString()); - }); - - it("says how long until a time that has not come", () => { - expect(relativeTime(at(10_000), NOW)).toBe("Just now"); - expect(relativeTime(at(5 * MIN), NOW)).toBe("in 5m"); - expect(relativeTime(at(3 * HOUR + 10), NOW)).toBe("in 3h"); - expect(relativeTime(at(2 * 24 * HOUR + HOUR), NOW)).toBe("in 2d"); - }); - - it("takes a Date or a number as well, and gives back what is not a time", () => { - expect(relativeTime(new Date(NOW - 2 * MIN), NOW)).toBe("2m ago"); - expect(relativeTime(NOW - 2 * HOUR, NOW)).toBe("2h ago"); - expect(relativeTime("yesterday-ish", NOW)).toBe("yesterday-ish"); - }); -}); diff --git a/apps/web/test/tokens.test.ts b/apps/web/test/tokens.test.ts deleted file mode 100644 index f666b2d0..00000000 --- a/apps/web/test/tokens.test.ts +++ /dev/null @@ -1,146 +0,0 @@ -import { readdirSync, readFileSync } from "node:fs"; -import { join } from "node:path"; -import { describe, expect, it } from "vitest"; -import { FAVICON_ATTENTION, FAVICON_PRIMARY } from "../src/lib/favicon"; -import { contrast as contrastOf, fromHex, over } from "./support/contrast"; -import { SRC, themes } from "./support/tokens"; - -const globalsCss = readFileSync(join(SRC, "app/globals.css"), "utf8"); - -/** WCAG 2.x contrast ratio of two #rrggbb colors. */ -const contrast = (a: string, b: string) => contrastOf(fromHex(a), fromHex(b)); - -/** Text on its surface: WCAG AA for normal text. */ -const TEXT_PAIRS: [string, string][] = [ - ["foreground", "background"], - ["card-foreground", "card"], - ["popover-foreground", "popover"], - ["muted-foreground", "background"], - ["muted-foreground", "card"], - ["muted-foreground", "muted"], - ["secondary-foreground", "secondary"], - ["accent-foreground", "accent"], - ["primary-foreground", "primary"], - ["primary-foreground", "primary-hover"], - ["destructive-foreground", "destructive"], - // Status text on its soft surface, and on the plain surfaces it also appears on. - ["ok", "ok-soft"], - ["warn", "warn-soft"], - ["danger", "danger-soft"], - ["info", "info-soft"], - ["ok", "card"], - ["warn", "card"], - ["danger", "card"], - ["danger", "popover"], - ["info", "card"], - ["warn", "background"], - ["danger", "background"], - // The accent as the color of a link or a label on the page. - ["primary", "background"], - ["primary", "card"], -]; - -/** Things that are drawn, not read (rings, focus outline, badges): WCAG 1.4.11, three to one. */ -const GRAPHIC_PAIRS: [string, string][] = [ - ["ring", "background"], - ["ring", "card"], - // The border of a field is what shows where it is (WCAG 1.4.11): it is drawn at its own color, not decoration like --border. - ["input", "background"], - ["input", "card"], - ["primary", "card"], - ["ok", "card"], - ["danger", "card"], - ["attention", "card"], - ["attention", "background"], -]; - -describe("the contrast measure", () => { - it("is the WCAG ratio: 21 for black on white, 1 for a color on itself, and below AA for a pale grey on white", () => { - expect(contrast("#000000", "#ffffff")).toBeCloseTo(21, 5); - expect(contrast("#5b5bd6", "#5b5bd6")).toBe(1); - expect(contrast("#aaaaaa", "#ffffff")).toBeLessThan(4.5); - }); -}); - -describe("the design tokens", () => { - for (const [name, tokens] of Object.entries(themes)) { - describe(`${name} theme`, () => { - it("defines every token the other theme defines", () => { - expect(Object.keys(tokens).sort()).toEqual(Object.keys(name === "light" ? themes.dark : themes.light).sort()); - }); - - it.each(TEXT_PAIRS)("%s on %s reads at AA (4.5:1)", (foreground, background) => { - expect(contrast(tokens[foreground]!, tokens[background]!)).toBeGreaterThanOrEqual(4.5); - }); - - it.each(GRAPHIC_PAIRS)("%s against %s is seen at 3:1", (color, surface) => { - expect(contrast(tokens[color]!, tokens[surface]!)).toBeGreaterThanOrEqual(3); - }); - }); - } - - it("maps every color token into Tailwind, and only those", () => { - const mapped = [...globalsCss.matchAll(/--color-([\w-]+): var\(--([\w-]+)\);/g)].map((m) => [m[1], m[2]]); - for (const [utility, token] of mapped) expect(utility).toBe(token); - expect(mapped.map(([utility]) => utility).sort()).toEqual(Object.keys(themes.light).sort()); - }); - - it("keeps the shimmering label visible under forced colors, where the gradient behind the transparent text fill is dropped", () => { - const shimmer = /@utility shimmer \{([\s\S]*?)\n\}/.exec(globalsCss)?.[1] ?? ""; - expect(shimmer).toContain("-webkit-text-fill-color: transparent"); - const forced = /@media \(forced-colors: active\) \{([^}]*)\}/.exec(shimmer)?.[1] ?? ""; - expect(forced).toContain("-webkit-text-fill-color: currentColor"); - expect(forced).toContain("background-image: none"); - }); - - it("keeps the favicon's two colors equal to the tokens they copy", () => { - expect(FAVICON_PRIMARY).toBe(themes.light.primary); - expect(FAVICON_ATTENTION).toBe(themes.light.attention); - }); -}); - -/** Every .tsx file of the components, as text. */ -function sources(): { file: string; text: string }[] { - const found: { file: string; text: string }[] = []; - const walk = (dir: string) => { - for (const entry of readdirSync(dir, { withFileTypes: true })) { - const path = join(dir, entry.name); - if (entry.isDirectory()) walk(path); - else if (entry.name.endsWith(".tsx")) found.push({ file: path.slice(SRC.length + 1), text: readFileSync(path, "utf8") }); - } - }; - walk(join(SRC, "components")); - return found; -} - -describe("the focus indicator", () => { - const all = sources(); - - it("is a ring whose opacity still reads at 3:1 against every surface it can sit on, in both themes (the solid token passing is not enough: the ring is drawn at its opacity)", () => { - const opacities = new Set(all.flatMap(({ text }) => [...text.matchAll(/(?:focus-visible|focus-within|focus|has-\[:focus-visible\]):ring-ring\/(\d+)/g)].map((m) => Number(m[1])))); - expect(opacities.size).toBeGreaterThan(0); - for (const [name, tokens] of Object.entries(themes)) { - for (const surface of ["background", "card", "popover"]) { - for (const opacity of opacities) { - const drawn = over(fromHex(tokens.ring!), opacity / 100, fromHex(tokens[surface]!)); - expect(contrastOf(drawn, fromHex(tokens[surface]!)), `ring-ring/${opacity} on ${surface}, ${name}`).toBeGreaterThanOrEqual(3); - } - } - } - }); - - it("is also an outline that forced colors keep: no control drops its outline for good, and every file that draws a focus ring says `outline-hidden` for the same trigger", () => { - for (const { file, text } of all) { - expect(text, file).not.toMatch(/(? { - const card = all.find(({ file }) => file.endsWith("ApprovalCard.tsx"))!.text; - expect(card).toContain('selected && "outline-2 outline-offset-2 outline-ring"'); - expect(card).not.toMatch(/selected && "ring-/); - }); -}); diff --git a/apps/web/test/use-frame.test.tsx b/apps/web/test/use-frame.test.tsx deleted file mode 100644 index 8e746b6f..00000000 --- a/apps/web/test/use-frame.test.tsx +++ /dev/null @@ -1,58 +0,0 @@ -// @vitest-environment jsdom -import { act, cleanup, renderHook } from "@testing-library/react"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { useFrame } from "../src/components/computer/use-frame"; -import { stubObjectUrls } from "./support/browser"; - -beforeEach(() => { - stubObjectUrls(); -}); - -afterEach(() => { - cleanup(); - vi.unstubAllGlobals(); -}); - -const bytes = (...values: number[]) => Uint8Array.from(values) as Uint8Array; - -describe("useFrame", () => { - it("asks for one frame at a time: a refresh during a read in flight starts no second one", async () => { - const answers: ((value: Uint8Array) => void)[] = []; - const load = vi.fn(() => new Promise>((resolve) => answers.push(resolve))); - const { result } = renderHook(() => useFrame(load, "image/jpeg", 1000, false)); - - let first!: Promise; - let second!: Promise; - act(() => { - first = result.current.refresh(); - second = result.current.refresh(); - }); - expect(load).toHaveBeenCalledTimes(1); - expect(result.current.pending).toBe(true); - - await act(async () => { - answers[0]!(bytes(1)); - await Promise.all([first, second]); - }); - expect(result.current.url).not.toBeNull(); - expect(result.current.pending).toBe(false); - - // The read has ended: the next refresh reads again. - act(() => { - void result.current.refresh(); - }); - expect(load).toHaveBeenCalledTimes(2); - await act(async () => answers[1]!(bytes(2))); - }); - - it("lets a refresh through again after a failed read", async () => { - const load = vi.fn().mockRejectedValueOnce(new Error("offline")).mockResolvedValue(bytes(1)); - const { result } = renderHook(() => useFrame(load, "image/jpeg", 1000, false)); - await act(() => result.current.refresh()); - expect(result.current.error).toBeInstanceOf(Error); - await act(() => result.current.refresh()); - expect(load).toHaveBeenCalledTimes(2); - expect(result.current.error).toBeNull(); - expect(result.current.url).not.toBeNull(); - }); -}); diff --git a/apps/web/test/use-resource.test.tsx b/apps/web/test/use-resource.test.tsx deleted file mode 100644 index 09ba5f1c..00000000 --- a/apps/web/test/use-resource.test.tsx +++ /dev/null @@ -1,53 +0,0 @@ -// @vitest-environment jsdom -import { act, cleanup, renderHook, waitFor } from "@testing-library/react"; -import { afterEach, describe, expect, it } from "vitest"; -import { useResource } from "../src/components/ui"; - -afterEach(cleanup); - -describe("useResource", () => { - it("does not keep what the last key said, its data or its error, for the key that replaced it while that one loads", async () => { - let finish: (value: string) => void = () => {}; - const loads: Record Promise> = { - a: () => Promise.reject(new Error("a failed")), - b: () => new Promise((resolve) => (finish = resolve)), - }; - const { result, rerender } = renderHook(({ key }) => useResource(loads[key]!, key), { initialProps: { key: "a" } }); - await waitFor(() => expect((result.current.error as Error | null)?.message).toBe("a failed")); - - rerender({ key: "b" }); - await waitFor(() => expect(result.current.error).toBeNull()); - expect(result.current.data).toBeUndefined(); - expect(result.current.loading).toBe(true); - - await act(async () => finish("b loaded")); - await waitFor(() => expect(result.current.data).toBe("b loaded")); - expect(result.current.error).toBeNull(); - }); - - it("drops an answer that arrives after a newer load started", async () => { - const resolvers: Array<(value: string) => void> = []; - const { result } = renderHook(() => useResource(() => new Promise((resolve) => resolvers.push(resolve)), "one")); - await waitFor(() => expect(resolvers).toHaveLength(1)); - act(() => result.current.reload()); - await waitFor(() => expect(resolvers).toHaveLength(2)); - await act(async () => resolvers[1]!("newer")); - await act(async () => resolvers[0]!("older")); - expect(result.current.data).toBe("newer"); - }); - - it("returns the same object until one of its fields changes, so what is built from it is not rebuilt for nothing", async () => { - let finish: (value: string) => void = () => {}; - const { result, rerender } = renderHook(() => useResource(() => new Promise((resolve) => (finish = resolve)), "one")); - const loading = result.current; - rerender(); - expect(result.current).toBe(loading); - await act(async () => finish("loaded")); - await waitFor(() => expect(result.current.data).toBe("loaded")); - const loaded = result.current; - expect(loaded).not.toBe(loading); - rerender(); - rerender(); - expect(result.current).toBe(loaded); - }); -}); diff --git a/apps/web/test/use-visible-interval.test.tsx b/apps/web/test/use-visible-interval.test.tsx deleted file mode 100644 index 16c02a85..00000000 --- a/apps/web/test/use-visible-interval.test.tsx +++ /dev/null @@ -1,97 +0,0 @@ -// @vitest-environment jsdom -import { act, cleanup, render } from "@testing-library/react"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { useVisibleInterval } from "../src/lib/use-visible-interval"; - -let hidden = false; - -function Probe({ tick, ms = 1000, enabled = true }: { tick: () => void | Promise; ms?: number; enabled?: boolean }) { - useVisibleInterval(tick, ms, enabled); - return null; -} - -function setHidden(value: boolean) { - hidden = value; - document.dispatchEvent(new Event("visibilitychange")); -} - -beforeEach(() => { - vi.useFakeTimers(); - hidden = false; - Object.defineProperty(document, "hidden", { configurable: true, get: () => hidden }); -}); - -afterEach(() => { - cleanup(); - vi.useRealTimers(); -}); - -describe("useVisibleInterval", () => { - it("polls at once and then once per period", async () => { - const tick = vi.fn(); - render(); - await act(() => vi.advanceTimersByTimeAsync(0)); - expect(tick).toHaveBeenCalledTimes(1); - await act(() => vi.advanceTimersByTimeAsync(3000)); - expect(tick).toHaveBeenCalledTimes(4); - }); - - it("asks nothing while the page is hidden, and polls at once when it is shown again", async () => { - const tick = vi.fn(); - render(); - await act(() => vi.advanceTimersByTimeAsync(0)); - setHidden(true); - await act(() => vi.advanceTimersByTimeAsync(5000)); - expect(tick).toHaveBeenCalledTimes(1); - await act(async () => setHidden(false)); - await act(() => vi.advanceTimersByTimeAsync(0)); - expect(tick).toHaveBeenCalledTimes(2); - }); - - it("never runs two at a time: the next is scheduled when the last has finished", async () => { - let running = 0; - let overlap = 0; - const tick = vi.fn(async () => { - running++; - overlap = Math.max(overlap, running); - await new Promise((resolve) => setTimeout(resolve, 2500)); - running--; - }); - render(); - await act(() => vi.advanceTimersByTimeAsync(10_000)); - expect(overlap).toBe(1); - // Each takes 2.5 s and the period starts after it: 0, 3.5, 7.0 s. - expect(tick).toHaveBeenCalledTimes(3); - }); - - it("keeps time when a poll throws", async () => { - const tick = vi.fn(async () => { - throw new Error("no answer"); - }); - render(); - await act(() => vi.advanceTimersByTimeAsync(2000)); - expect(tick).toHaveBeenCalledTimes(3); - }); - - it("stops when disabled and when unmounted, and calls the latest function it was given", async () => { - const first = vi.fn(); - const second = vi.fn(); - const view = render(); - await act(() => vi.advanceTimersByTimeAsync(0)); - view.rerender(); - await act(() => vi.advanceTimersByTimeAsync(1000)); - expect(first).toHaveBeenCalledTimes(1); - expect(second).toHaveBeenCalledTimes(1); - - view.rerender(); - await act(() => vi.advanceTimersByTimeAsync(5000)); - expect(second).toHaveBeenCalledTimes(1); - - view.rerender(); - await act(() => vi.advanceTimersByTimeAsync(0)); - expect(second).toHaveBeenCalledTimes(2); - view.unmount(); - await act(() => vi.advanceTimersByTimeAsync(5000)); - expect(second).toHaveBeenCalledTimes(2); - }); -}); diff --git a/apps/web/test/yaml.test.ts b/apps/web/test/yaml.test.ts deleted file mode 100644 index c238b3a0..00000000 --- a/apps/web/test/yaml.test.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { parseDotConfig } from "@invisible-dots/shared"; -import { describe, expect, it } from "vitest"; -import { toYaml } from "../src/lib/yaml"; - -/** The example of the architecture document: every section of a config, written the way a person would. */ -const EXAMPLE_CONFIG = `name: fare-watch -instructions: > - Write findings to ~/workspace/fares.csv. -model: - provider: openrouter - id: z-ai/glm-5.3-flash -computer: - cpu: 2 - memory: 4gb - disk: 40gb - idle_timeout: 15m -permissions: - computer.exec: allow - browser.identity.delete: ask -limits: - max_steps_per_task: 60 - max_cost_per_task_usd: 1.00 -`; - - -describe("toYaml", () => { - it("round-trips the example config through the shared schema", () => { - const config = parseDotConfig(EXAMPLE_CONFIG); - expect(parseDotConfig(toYaml(config))).toEqual(config); - }); - - it("keeps strings that YAML would otherwise read as other types", () => { - const config = parseDotConfig({ - name: "tricky", - instructions: "key: value # not a comment\n indented second line\n", - model: { provider: "openrouter", id: "vendor/model:free" }, - models: { summary: "0123" }, - computer: { cpu: 4, memory: "8gb", disk: "100gb", idle_timeout: "0" }, - permissions: { "computer.exec": "ask" }, - }); - const text = toYaml(config); - expect(parseDotConfig(text)).toEqual(config); - expect(toYaml({ answer: "yes" })).toBe('answer: "yes"\n'); - }); - - it("writes multi-line text as a literal block and round-trips edge cases", () => { - const cases = ["one\ntwo", "one\ntwo\n", " leading\nspace", "a\n\n\nb", "trailing\n\n", "tab\there\nx", "#hash\n- dash"]; - for (const instructions of cases) { - const config = parseDotConfig({ name: "m", instructions, model: { provider: "openrouter", id: "a/b" } }); - expect(parseDotConfig(toYaml(config)).instructions).toBe(instructions); - } - const block = toYaml({ text: "one\ntwo" }); - expect(block).toBe("text: |-\n one\n two\n"); - }); - - it("writes empty objects inline and nests objects by two spaces", () => { - expect(toYaml({ a: {}, b: { c: { d: true } }, e: null, f: 1.5 })).toBe("a: {}\nb:\n c:\n d: true\ne: null\nf: 1.5\n"); - }); - - it("refuses a non-object at the top level", () => { - expect(() => toYaml("x")).toThrow(); - }); -}); diff --git a/apps/web/tsconfig.json b/apps/web/tsconfig.json deleted file mode 100644 index 20c8c96a..00000000 --- a/apps/web/tsconfig.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "lib": [ - "ES2023", - "DOM", - "DOM.Iterable" - ], - "module": "ESNext", - "moduleResolution": "Bundler", - "jsx": "preserve", - "allowJs": false, - "types": [ - "node", - "react", - "react-dom" - ], - "plugins": [ - { - "name": "next" - } - ] - }, - "include": [ - "next-env.d.ts", - "src/**/*.ts", - "src/**/*.tsx", - "test/**/*.ts", - "test/**/*.tsx" - ], - "exclude": [ - "node_modules", - ".next" - ] -} diff --git a/docs/architecture.md b/docs/architecture.md deleted file mode 100644 index 2e1a134a..00000000 --- a/docs/architecture.md +++ /dev/null @@ -1,3037 +0,0 @@ -# invisible_dots architecture - -This document is the contract every part of the repository is written against. -When code and this document disagree, one of them is a bug: fix the code or -change the document in the same commit. - -## 1. What a Dot is - -A Dot is a persistent agent that owns a computer. Every Dot has: - -- a configuration (name, instructions, model, permissions, resources), -- its own QEMU virtual machine (hardware accelerated) with a persistent qcow2 disk, -- inside that VM, its own agent runtime, which calls models through OpenRouter, -- its own memory, conversation and task state, stored inside the VM, -- zero or more browser identities, each one a separate browser profile with - its own cookies, storage, logins and fingerprint. - -The control plane on the host manages infrastructure: the Dot registry, VM -lifecycle, task dispatch, events, approvals. It never decides what a Dot does -step by step. The reasoning happens inside the Dot's VM. - -### 1.1 One mechanism on every host - -invisible_dots runs on Linux and on Windows with the SAME code path: the same -QEMU command line, the same networking, the same host to guest channel, the -same way of starting, watching and stopping QEMU, the same database, the same -data directory layout, the same commands. A -difference between the two is allowed only where the operating system makes -sameness impossible, and every such difference lives in ONE function that a -test covers. The complete list today: - -| what | Linux | Windows | where | -|---|---|---|---| -| QEMU accelerator | `-accel kvm` | `-accel whpx` | `apps/vm-manager/src/host.ts` `accelerator()` | -| how `doctor` (the CLI's and `GET /api/doctor`'s) reads the accelerator before probing it | `/dev/kvm` opens read-write | the `HypervisorPlatform` feature state through `Get-CimInstance` | `apps/vm-manager/src/accelerator-access.ts` `checkAcceleratorAccess()`, given the host's platform by `hostAccessDeps()` in the same file | -| how `invisible-dots setup` installs QEMU and enables the accelerator | `sudo apt-get install` (or the distribution's equivalent, printed) | one UAC prompt: enables the Windows Hypervisor Platform feature and runs the official QEMU installer silently | `apps/cli/src/setup/install.ts` `installHostPrerequisites()`, given the host's platform by `hostAccessDeps()` of `apps/vm-manager/src/accelerator-access.ts` | -| `setup` run as root | refused: it would check `/dev/kvm` as root and add root to the `kvm` group, not the person who runs the server; it calls `sudo` itself | there is no root; setup always runs as the normal user and elevates its one step | `apps/cli/src/setup/install.ts` `setupRefusal()` | -| a file or directory private to the user (`config/`, `master.key`, `api.token`, `db/`, the data directory) | mode `0600` / `0700` | an ACL that no account but the current user can use, inheritance removed (`icacls`), because Windows ignores the mode bits and a directory under a drive root inherits "Authenticated Users: Modify"; SYSTEM and the local Administrators may stay, as root does on Linux, since some machines grant them explicitly on every new directory | `packages/shared/src/files.ts` `permissionBitsEnforced()` and `restrictToOwner()` | - -One line names a platform without being a branch: `agentdBuildCommand()` in -`guest/image-builder/src/runtime.ts` builds dot-agentd with `GOOS=linux -GOARCH=amd64` on both hosts, because that is the guest's platform, not the -host's. - -One more branch exists only to let tests run on a Windows developer host: -`packages/shared/src/sockets.ts` `testSocketPath()` hands out a named pipe -there, because Node cannot serve a unix socket on Windows. The guest -code that serves sockets decides from the path (`socketIsAFile()`), never -from the platform. dot-agentd ships for linux/amd64 only, and the build -constraints of ten of its files let its package compile and its tests run on a -Windows developer host; no shipped binary contains the `!unix` or `!linux` side: - -- the account the model's work runs as (`LookupAccount`, `actAs`, `ForgetAmbientCapabilities`, `RequireCapabilities`; passwd, `setfsuid` and `prctl` are Linux calls, section 4.1): `guest/dot-agentd/internal/agentd/account_linux.go`, `guest/dot-agentd/internal/agentd/account_other.go` -- `setProcessGroup`: `guest/dot-agentd/internal/agentd/exec_unix.go`, `guest/dot-agentd/internal/agentd/exec_other.go` -- `listenUnixPrivate`: `guest/dot-agentd/internal/agentd/listen_unix.go`, `guest/dot-agentd/internal/agentd/listen_other.go` -- `diskUsage`: `guest/dot-agentd/internal/agentd/platform_unix.go`, `guest/dot-agentd/internal/agentd/platform_other.go` -- the process route and its relay (`startProc`, `killGroup`, `TerminalSize`, `MakeRaw`, `WatchTerminalSize`, `ForwardedSignals`; pseudo-terminals are Linux ioctls): `guest/dot-agentd/internal/agentd/proc_linux.go`, `guest/dot-agentd/internal/agentd/proc_other.go` - -`tests/repo/platform-branches.test.ts` reads every product source file -(TypeScript and JavaScript, the build scripts, the guest's shell scripts and -dot-agentd's Go) and fails on a platform check outside this list: Node's -platform and OS probes, `getuid`, the Windows path module, a platform name -as a string, Go's `runtime.GOOS`, build constraints and platform file name -suffixes. The engine under `invisible_engine_dots/` (section 2) is Python and -the test does not read it: it runs only inside the Linux guest, uses unix -sockets and process groups, and has no platform branch (the Windows, macOS and -service-manager code of the nanobot it was forked from is gone). - -QEMU has no monitor (no QMP, no HMP) on either host. The control plane sees a -VM's QEMU only as a process, through `process.kill(pid, 0)` and the guest -port its user networking listens on, and reaches the guest only through -dot-agentd (section 5.1); both behave the same on Linux and Windows. That is -what removed the one host-side branch a monitor needed: Node has no AF_UNIX -client on Windows, QEMU's `pipe` chardev there blocks the start until a client -connects and serves one client for the life of the process, and a monitor on -TCP would be reachable from every guest (section 3.6). - -Never fall back silently: when the accelerator is missing, starting a VM fails -with a message that says which command fixes it. Software emulation (TCG) is -never used. macOS is not supported in this version: it needs an arm64 guest -image and an arm64 browser build, which are not wired. - -## 2. Repository layout - -```text -apps/ - api/ control-plane HTTP API + SSE, and the control plane composed as one process (`invisible-dots server` runs it) - scheduler/ task dispatcher, wake on work, sleep on idle - vm-manager/ QEMU driver: overlays, seed and runtime ISOs, QEMU argv, port forwards, the guest client, the host's one process runner, the doctor report (section 11.1) - web/ Next.js web client - cli/ `invisible-dots`: setup, doctor, image build, server, and the API client commands -packages/ - shared/ types and schemas shared by host and guest: config, protocol, events, states - database/ PostgreSQL schema (PGlite embedded or an external server), migrations, repositories, durable queue - iso/ ISO 9660 + Joliet writer in plain TypeScript (seed and runtime disks) - events/ event types, the host event log and its fan-out to SSE subscribers - channels/ the messaging channel hub and the Telegram and WhatsApp adapters (section 9.8): pairing, who may talk, the messages between a chat and its Dot; runs inside the control plane process - sdk/ typed HTTP client for the API (used by cli and web) -guest/ - dot-agentd/ the computer daemon (Go): the guest endpoint, exec, files, screenshots - image-builder/ golden image and runtime disk builders (TypeScript), guest systemd units -invisible_engine_dots/ - the Dot's engine, run by `python -I -B -m nanobot` in the guest: a hard fork of - nanobot (HKUDS/nanobot, MIT, commit f75470e7), Python 3.11 or newer, provenance - and everything removed since the import in invisible_engine_dots/UPSTREAM.md. - Import package `nanobot`, distribution `invisible-dots-engine`. Only - the engine core is kept: no channel, web UI, TUI, audio, pairing, nanobot - skill machinery, subagent, web tool or CLI (the one entry point answers - `--version` and refuses every other command). What it carries: nanobot's - tool-calling turn runner, chat completions to OpenRouter and no other - provider, the tools of the permission table (section 8.8), nanobot's cron - service and its MCP client (which serves the Dot's browsers, - `invisible-playwright-mcp`, through the `BrowserManager`, and the MCP servers - the person declares, through `nanobot/dots/mcp_servers.py`). The Dot's own - layer, the contract of sections 5.3 to 5.4, its skills (section 8.6, - `nanobot/dots/skills.py`) included, is `nanobot/dots/`. Its own pytest suite runs in CI's `engine` job - (`.github/workflows/tests.yml`, Linux only: unix sockets) -virtualization/ - qemu/ the pinned QEMU version for Windows setup (installer URL + SHA-256) and argv notes - cloud-init/ NoCloud templates - images/ pinned base image metadata -tests/ - repo/ checks over the whole repository (the platform branches of section 1.1, and the - contract of the end-to-end run below with the product) - e2e/ the real-VM acceptance run (`node tests/e2e/run.ts`, on a Linux host with an - accelerator and an OpenRouter key; not in CI), its stand-in Linux host for - machines that cannot run it directly, and README.md - bench/ the agent benchmark on real Dots (Harbor tasks, written by its generators before - every run, and LongMemEval), run by hand in a container; README.md -docs/ -``` - -There is no host installer script, no service unit and no container: the host -needs Node 24 and QEMU, and `invisible-dots setup` gets QEMU (section 11). - -Everything under `apps/`, `packages/` and `guest/image-builder/` is TypeScript in one -npm workspace. `invisible_engine_dots/` is not part of it: it is a Python project -(`pyproject.toml`, pytest) of its own, and this repository's TypeScript project, -vitest run and npm workspaces all leave `invisible_engine_dots/` out -(`tests/repo/vendored-nanobot.test.ts` checks it). It is a hard fork: it is -changed in place and upstream changes are never merged. `dot-agentd` -is a Go module. Node 24 or newer for the TypeScript. Go 1.25 or newer. Python 3.11 or newer for -the engine; the guest runs it on Ubuntu 24.04's CPython 3.12. - -The control plane runs as ONE process (`invisible-dots server`) that composes -`api`, `scheduler` and `vm-manager`. They are separate packages so that each -can be tested alone, not separate daemons. It runs in the foreground the same -way on every host; running it as a service is left to the person. - -## 3. Host - -### 3.1 Requirements - -Linux or Windows on x86-64, Node 24, and QEMU 8.2 or newer (Ubuntu 24.04's -own package; the number lives in `MIN_QEMU_VERSION`, `apps/vm-manager/src/host.ts`) -(`qemu-system-x86_64` and `qemu-img`) with its hardware accelerator usable: -`/dev/kvm` readable and writable by the user on Linux, the Windows Hypervisor -Platform feature enabled on Windows. No administrator rights are needed at run -time. `invisible-dots doctor` checks each item and names the command that fixes -it; `invisible-dots setup` performs those commands (section 11). Building the -golden image boots a builder VM with 2 vCPUs and 4 GiB of memory by default -(`GOLDEN_DEFAULTS` in `guest/image-builder/src/golden.ts`). - -QEMU is found in `INVISIBLE_DOTS_QEMU_DIR` alone when that is set (a QEMU from -there mixed with a `qemu-img` from somewhere else would run two versions on the -same disks); otherwise first in the default install location of the official -Windows installer, `%ProgramW6432%\qemu` (read from the variable Windows sets, -so a Program Files on another drive is found; Linux has no such variable and -nothing to check), then on `PATH`. The installer's directory comes first so -the QEMU `setup` just installed wins over an older one on `PATH`; when the -installer reused an earlier install directory, `setup` prints the -`INVISIBLE_DOTS_QEMU_DIR` line that points at it. QEMU is always invoked by -absolute path once found, with the server's home as its working directory -(a Windows process holds its working directory open, so inheriting the -directory the server was started from would keep that directory from being -moved or deleted while any Dot runs). - -### 3.2 Host filesystem - -One directory, `INVISIBLE_DOTS_HOME`, default `~/.invisible-dots` on every -host (`%USERPROFILE%\.invisible-dots` on Windows): - -```text -~/.invisible-dots/ - config/ - master.key 32 random bytes: encrypts secrets in the database - api.token bearer token for the API (its first line) - db/ the embedded PostgreSQL (PGlite) data directory - server.lock { pid, host_uptime_s } of the one server running on this home - images/ - noble-minimal-cloudimg-amd64.img pinned by SHA-256 (virtualization/images/base.json) - golden-.qcow2 immutable, read-only - golden-.json its manifest: inputs, versions, SHA-256 - runtime-.iso our code: agent bundle + dot-agentd + units - runtime-.json its manifest - .cache/ verified downloads (re-hashed on every build) - .golden-.work/ a golden build in progress, kept when it fails - .golden-build.lock .runtime-build.lock - vms// - disk.qcow2 overlay, backing file = a golden image - seed.iso NoCloud seed - qemu.json pid file of the running VM: { pid, guest_port, host_uptime_s }, written at spawn - serial.log the guest serial console, truncated at each start - logs/ - qemu-.log QEMU's own output -``` - -Both hosts use the same layout and the same names; only the root differs, and -only because home directories differ. `` is `-`, so the newest build sorts last and rebuilding unchanged inputs -is a no-op; `packages/shared/src/paths.ts` is the one place that names these -files. A path under `INVISIBLE_DOTS_HOME` that contains a comma or any -character outside plain ASCII is refused (`qemuPathProblem()`, -`apps/vm-manager/src/qemu-args.ts`), and `doctor` reports such a home before a -build or a Dot fails on it: QEMU's option syntax cannot carry a comma in every -flag, and the pinned Windows QEMU 11.1, measured, cannot open a `-drive` file -whose path holds a non-ASCII character (an accented Latin letter as much as a -Cyrillic one). Linux would accept UTF-8, but the default home sits under the -account name, and a home that works on one host and not on the other is the -divergence section 1.1 rules out. Nothing under it is a unix socket, so its -length is not limited. - -The data directory, `config/` and `db/` are private to the user who runs the -server, and so are `master.key` and `api.token` (section 1.1 says how on each -host); every start brings an existing directory back to that. - -`server.lock`, the two build locks and every `qemu.json` record the host's -uptime when they were written (`host_uptime_s`). It only grows within one boot -and starts from zero at the next, so a record from before a host restart is -recognized as stale without trusting a pid, which the operating system may -have handed to another process since. (Windows Fast Startup keeps counting -across a shutdown; such a record falls back to the pid checks of section 3.4.) -The locks are one helper, `acquirePidLock()` in `packages/shared/src/process.ts`: -a lock is taken over only when it is stale (the host restarted, or its pid and -the child process it records are gone, or it names this process's own pid, -which can only be an earlier holder's), only by the one process that creates -`.takeover` first, and only after reading the lock again unchanged; it is -released only by its holder. The golden build records its builder VM as that -child, so a build killed outright never has its work directory deleted under -a QEMU that still runs. A refusal names the holder and the file to remove. - -`qemu.json` is written by the vm-manager (to a temporary name, then renamed) -right after it spawns QEMU, from the spawned process's pid and the guest port -it passed to QEMU. It is the one record of which process and which port belong -to the Dot: QEMU's own `-pidfile` is not used, and the `computers` row (section -9.1) is corrected from it on reconciliation, never the other way round. A file -that does not parse makes the VM's state ERROR, never STOPPED, because a second -QEMU on the same disk would corrupt it. - -### 3.3 Two images, two lifetimes - -- The **golden image** carries the operating system and third-party software: - Ubuntu 24.04, Xvfb and a minimal XFCE session, the libraries - the browser needs, `uv`, `invisible-playwright-mcp` in its own - Python environment, the browser engine already downloaded (the library - keeps its own GeoIP database, fetched from its release at a launch), and the Python - environment of the Dot's engine (`/opt/invisible-dots-engine`). It changes - rarely. It is never modified once a VM uses it: a new one gets a new version - in its name. -- The engine's environment holds its third-party packages and not its code. - `builder/build-engine-env.sh` makes a venv from Ubuntu's own - `/usr/bin/python3` (CPython 3.12) and installs into it, with - `uv pip install --require-hashes --only-binary :all:`, every package of - `guest/image-builder/builder/engine-requirements.lock`: exact versions with - the SHA-256 of their files, transitive packages included, wheels only, so no - build script of a third-party package runs as root in the builder VM. The - lock is the one place the engine's dependency versions are fixed (a test - checks that every dependency `invisible_engine_dots/pyproject.toml` declares - is in it), and it is part of the inputs digest, so a changed package is a new - image. The build also prefetches tiktoken's `cl100k_base` table into - `share/tiktoken` (the engine never fetches it) and leaves the whole venv - owned by root and writable by nobody else. -- The engine's own code is on the runtime disk, at `/opt/invisible-dots/engine` - (its `nanobot` package, `LICENSE` and `UPSTREAM.md`), and a `.pth` - file in the venv's site-packages puts that directory on the venv's path. Our - code is ours to change often, so a change to it is a new ISO and not an hour - of golden build (the engine's source is not an input of the golden digest). - A runtime disk that needs another dependency needs a new golden image. -- Every input of the golden image but the browser is pinned by content: the - cloud image, `uv` and the tunnel by SHA-256 (`virtualization/images/base.json`, - `guest/image-builder/pins.json`). The browser is never pinned: each build - takes the newest release of `invisible-playwright-mcp` on PyPI, with the - newest `invisible-playwright` it accepts and the `invisible-core` that one - names (`guest/image-builder/src/browser-release.ts`). The three versions are - part of the inputs digest, so a new release is a new image, and the image's - manifest records the ones installed; the scheduled run of - `.github/workflows/golden-image.yml` publishes that image without any change - to this repository. The builder installs them with - `uv pip install --only-binary :all:`, so no package's build script runs, and - refuses an environment that holds another browser (the names in - `guest/image-builder/builder/other-browsers.txt`). The apt packages come from the Ubuntu archive as it - was at the moment `apt_snapshot` of `pins.json` names (snapshot.ubuntu.com, - `apt-get --snapshot`), so they and their dependencies are the same on any day - of the build; apt checks their signatures. -- The **runtime disk** (`runtime-.iso`, attached read-only to every VM - and mounted at `/opt/invisible-dots`) carries our code: the engine's source, - the `dot-agentd` binary and the systemd units. A new - version of our code is a new ISO and a VM restart, not a new golden image and - not a rebuilt overlay. -- `install.sh` (cloud-init's per-boot script) is what starts the Dot's units, - on every boot, after the VM proxy: they are copied into - `/etc/systemd/system` and never enabled. Enabled, systemd would start them at - `multi-user.target`, and cloud-init's final stage, which runs `install.sh` - and writes the seed's `config.json` first, comes after that target - (`cloud-final.service` is `After=multi-user.target`): on every boot after the - first, a Dot would have gone out before its proxy was set. - -The host builds the runtime disk from this checkout, and the golden image from -public sources (`invisible-dots image build`, code in `guest/image-builder/`) -unless CI has published the golden image of exactly the same inputs: the -release `golden-` of `.github/workflows/golden-image.yml`, -which `image build` downloads instead (`guest/image-builder/src/prebuilt.ts`; -section 11.3). Either way the golden image is provisioned with the same QEMU a -host runs Dots with and the same machine: the builder VM's command line is built from the vm-manager's `machineArgs()`, drive and device -functions, the ones `qemuArgs()` uses, plus only what a builder needs, so a -golden image is provisioned on the machine type, accelerator and CPU model a -Dot boots it on. The seed and runtime ISO labels are each defined once, in -`apps/vm-manager/src/seed.ts`. Both ISOs are written by -`packages/iso`, so no ISO tool is needed on any host. - -### 3.4 VM definition - -One QEMU process per running Dot, started by the vm-manager with an argument -array (never a shell) built by ONE function, `qemuArgs()`, the same on every -host apart from the accelerator: - -```text -qemu-system-x86_64 - -name invisible-dot- - -machine q35 -accel -cpu host,-vmx,-svm - -smp -m - -drive if=virtio,file=,format=qcow2,discard=unmap - -drive if=virtio,file=,format=raw,readonly=on - -drive if=virtio,file=.iso>,format=raw,readonly=on - -netdev user,id=net0,hostfwd=tcp:127.0.0.1:-:1024 - -device virtio-net-pci,netdev=net0 - -device virtio-rng-pci - -serial file: - -display none -``` - -- No monitor and nothing that pauses: no `-qmp`, no `-monitor`, no `-S`, no - `-no-shutdown`. QEMU starts running the guest at once and exits when the - guest powers off. A test fails on any of these flags. -- No fallback: if `-accel` fails, the start fails and the error names the - fix. If the CPU model is rejected by an accelerator, the error says so; it - is not replaced by a guessed model without an explicit decision recorded here. -- Start: spawn QEMU detached (so the control plane can restart without - stopping Dots), with an allowlist of the server's environment - (`allowlistedEnvironment()`, `packages/shared/src/environment.ts`: what a - program needs to start and find its files, never `INVISIBLE_DOTS_TOKEN` or a - `DATABASE_URL`, which a QEMU that outlives the server would otherwise keep), - write `qemu.json`, then wait until QEMU listens on the guest - port and is still running 1 s later. The listening port means QEMU parsed - its command line, opened the accelerator and set up networking; the second - look catches what fails just after (QEMU checks the CPU model when it builds - the machine). A QEMU that exits is reported with what it wrote to - `logs/qemu-.log` during this start; one that never listens within - 30 s is killed and reported the same way. -- A QEMU that starts but never brings the guest up (a vCPU the accelerator - stopped, a guest stuck in its firmware or kernel) is caught by the READY - procedure (section 9.3): waiting for `GET /v1/health` fails after its timeout - with the end of `logs/qemu-.log` and of `vms//serial.log` in - the error, and at once when QEMU exits meanwhile. -- State: STOPPED when there is no `qemu.json`, its process is gone, or it was - written before the host last restarted; RUNNING when its process is this - Dot's QEMU (the rule below); ERROR when a live pid cannot be proven to be (it - is reported and never killed). Whether the guest inside is up is guest - health, which the READY procedure checks; the state of a Dot is both - (section 9.3). -- Which process may be killed, the one rule: a process this control plane - spawned and whose exit Node has not reported (Node holds that process, so - its pid cannot be recycled meanwhile), or a pid from a `qemu.json` written - in this boot that is alive and this user's (`process.kill(pid, 0)` - succeeds; EPERM means another user's process, and a QEMU this control plane - spawned always runs as its own user) AND whose guest port from the same file - accepts a TCP connection on 127.0.0.1. QEMU's user networking listens on that - port from the moment it is set up until the process exits, whether or not the - guest is up; a process that inherited a recycled pid after a crash or a host - restart does not listen on that one port. The rule is checked again right - before every kill. Anything else is never killed. A listener that passes it - is still not trusted with the Dot's token: the guest must prove it holds the - token first (section 5.1). -- Stop: `POST /v1/system/poweroff` through the guest channel with the Dot's - token (section 5.2), then wait up to 60 s for QEMU to exit, then kill it - (SIGKILL on Linux, TerminateProcess on Windows, both through - `process.kill`) and wait up to 10 s for it to go. A guest that does not - accept the poweroff (it is not up, or the request fails) cannot power itself - off, so its QEMU is killed at once. Destroy kills without a poweroff: the - disk goes too. The VM's files are rewritten or removed right after its QEMU - stopped (the seed at the next start, everything at a destroy) through one - retry while a file is in use (`retryWhileInUse()`, - `packages/shared/src/replace-file.ts`, which also replaces every file this - repository writes atomically, through `replaceFile()`): Windows can report a process gone before - its handles are closed, and an antivirus may hold a file for a moment; on - Linux the first try succeeds. -- Reboot is a stop and a start, not a reset: a reset skips the guest's - shutdown and would not apply a new runtime ISO, CPU count or memory size. - The guest port changes on reboot. -- Reconciliation, the one path for it: reading a VM's state (the vm-manager's - `state()`, which the control plane's recovery after a restart calls for - every Dot) removes a `qemu.json` whose process is gone the moment it reads - it, under the Dot's lock, so a pid recycled later cannot make it look alive. - A VM whose process is its QEMU is adopted with the pid and port from the - file, and the READY procedure then checks its guest. -- The seed is rewritten at every start. Its cloud-init instance-id is - `iid--`, so cloud-init re-runs its per-instance - steps only when the seed's content changes. -- Measured with QEMU 8.2.2 (Ubuntu 24.04) and KVM: this argv starts, the - forward listens about 0.2 s after the spawn, and a taken port exits with - `Could not set up host forwarding rule`, the message the start retries on. -- The CPU model is the host's own without the virtualization extensions, - `host,-vmx,-svm` (`CPU_MODEL` in apps/vm-manager/src/qemu-args.ts), on every - accelerator: a Dot never runs a hypervisor. Measured with QEMU 11.1 on - Windows 11 (Intel Core Ultra 7 255H, hypervisor running): `-cpu host` and - `-cpu max` pause the VM at its first firmware instructions with - `WHPX: Unexpected VP exit code 4`, while `-cpu host,-vmx` boots the Ubuntu - cloud image to its login prompt; no other host CPU feature mattered - (bisected). `-svm` is the AMD name of the same extension. - -### 3.5 Port forwards - -`` is a free TCP port on 127.0.0.1, chosen at each start (bind to -port 0, read it, release it, pass it to QEMU; retried if QEMU reports the port -taken) and recorded in the `computers` table. A port is never a credential: -every request to a guest carries the Dot's token (section 5.1). - -### 3.6 Networking and its limits - -QEMU user-mode networking gives every Dot its own NAT with no bridge, no -administrator rights and the same behaviour on every host. Dots cannot reach -each other's guest addresses. A guest CAN reach services on the host's -loopback through `10.0.2.2`, which includes the control plane API, the web -client and the other Dots' forwarded ports. All of them require a credential -the guest does not have: the API's bearer token, the web client's session -(section 9.7; its Host and Origin checks are written by the client and so -cannot be what lets a request through), and each Dot's own token, which a -forwarded port answers only after proving it holds it (section 5.1). QEMU has -no monitor anywhere. Nothing else should listen unauthenticated on the host's -loopback while Dots run; `invisible-dots doctor` cannot check that, and this -document says so. - -## 4. Guest - -### 4.1 Processes (systemd, each unit as the user it names) - -| unit | user | what | -|---|---|---| -| `dot-desktop.service` | `dot` | `Xvfb :0 -nolisten tcp` plus a minimal XFCE session on it | -| `dot-agentd.service` | `dotagentd` | the computer daemon; TCP port 1024 (reached only through the host's port forward) and a local unix socket | -| `invisible-dots-agent.service` | `dotengine` | the Dot itself: the engine (`/opt/invisible-dots-engine/bin/python -I -B -m nanobot`) | - -Three users, none of them shared (all made by the image builder's seed): - -- `dot` runs the desktop, the browser and **every command of the model**. It is - the one user the model controls. It is in the groups `audio`, `video` and - `systemd-journal` (it reads its computer's system journal; cloud-init adds no - group to a user that exists already, so the groups are given where the user is - created, by the image builder's seed). It has no sudo rule. -- `dotagentd` runs dot-agentd. It holds the Dot's token (`/etc/invisible-dots/config.json`, - 0600) and owns `agentd.sock`. It has no home, no login and no group but its own. - It is the one user that may run one command as root, `/usr/bin/systemctl - poweroff` without a password, which is what dot-agentd starts when the host - stops the VM (section 5.2). The Dot's seed writes that rule; the golden image's - builder seed gives nobody one and the provisioner removes any rule the image - had, because a Dot's seed only adds its rule to that file. -- `dotengine` runs the engine. It is in group `dot` so it can read and seed the - Dot's workspace, and its unit runs it with `UMask=0002`. It holds the - OpenRouter key in memory and owns its state (`/home/dotengine/state`, 0700). - It needs no privilege: there is no sudo rule for it and no root-owned - configuration file (the Dot's config is stored in the engine's own database, - section 8.8), and its unit sets `NoNewPrivileges=yes`, so nothing it starts can - gain one. - -dot-agentd does not run the model's work as itself. Every command (`POST /v1/exec` -and the engine's `dot-agentd relay`), every file operation of the file routes of -both listeners, the browser's MCP servers (which the engine starts through the -relay) and the screenshot run as `dot`: - -- a process is started with `SysProcAttr.Credential`: dot's uid, gid and - supplementary groups (the daemon's own are not inherited); -- a file route acts as `dot` for the length of the request: the file-system user - and groups of the thread that serves it are dot's, as an NFS server's are for a - client's. The files it opens, creates and renames are dot's and are checked as - dot's, and the home confinement of the TCP listener (section 5.2) is decided on - top of that, unchanged; -- a pseudo-terminal for a command is opened as `dot`, so the terminal belongs to - it. - -The privileges that takes are the unit's `AmbientCapabilities=CAP_SETUID -CAP_SETGID CAP_KILL` (change to dot; end a process group of dot's) and nothing -else. The daemon empties its ambient set before it starts anything, whatever -`--run-as` says, so no program it starts holds one: a command of the model has -`CapPrm`, `CapEff` and `CapAmb` zero, and the smoke asserts it. It refuses to -start without the three when it is to act as another user. `--run-as` (default -`dot`, `INVISIBLE_DOTS_RUN_AS`) names the user; an empty value runs the work as -the daemon's own user, which is for development and which the daemon says in its -log (the ambient set is emptied then too). - -What the daemon runs as itself is found through no directory the model writes. -The unit gives it a `PATH` of system directories only, and every program it -starts for itself is named by path (`/usr/bin/sudo -n /usr/bin/systemctl -poweroff`, `/bin/bash`, `/usr/bin/import`): a `sudo` planted in dot's -`~/.local/bin` (dot owns its home and can open it) would otherwise be run as -`dotagentd` by the next poweroff. The `PATH` of the model's commands, dot's -`~/.local/bin` in front of the system's, is built by the daemon with the rest of -their environment (`execEnv`), not inherited from its own. The engine, which holds -the OpenRouter key, has the same rule: its unit's `PATH` is system directories -only, and the one program of dot's it needs, the browser's MCP server, is named by -its path (`INVISIBLE_DOTS_MCP_COMMAND=/home/dot/.local/bin/invisible-playwright-mcp`) -and started as dot through the relay. - -What this closes: no process of the model's shares a uid with a daemon, so the -model can neither read the Dot's token nor read the memory of dot-agentd or of -the engine, nor signal or stop either, nor reach either socket (their -directories admit only the two daemons, section 4.2). The engine's socket does -not rely on its directory alone: the engine asks the kernel who connected -(`SO_PEERCRED`) and refuses a process of `dot` (`INVISIBLE_DOTS_MODEL_USER`, -default `dot`; the engine does not start when that user does not exist). The -engine's API has no authentication of its own, so what would let the model -`PUT /config` with every permission `allow`, or approve its own parked calls -with `POST /events`, is exactly what these two walls keep out, and the smoke -tries both as the model does. The model can connect to TCP port 1024 inside its own -computer too, and gets `401` without the token. A process of `dot` could listen -on port 1024 only while dot-agentd is not running, and the host does not send -the token or the key to a listener that cannot answer the proof of section 5.1. -What this does not protect, stated as it is: - -- The split is the guest operating system's. A flaw in the guest's kernel, or a - way to become root that is not known here, would let a process of `dot` read - what the other two users hold. The VM is the outer wall then, and what it lets - the guest reach is section 3.6's. -- The daemon's three capabilities make a flaw in dot-agentd worth more than a - flaw in an ordinary program: `CAP_SETUID` lets it become any user, root - included, and it is the one user with a sudo rule (the power-off). It is a - small program and its one door, the TCP port, needs the token, which `dot` - cannot read; the model can reach the port and gets `401`. -- `dot` owns the desktop, the browser and `/home/dot`, and reads the proxy of an - identity whose browser is open (section 6). That is the model's computer, not - a leak out of it. -- Nothing limits how much disk, memory or processor time `dot`'s processes use - of the VM's (no cgroup limit, no quota), so a runaway command can starve the - daemons of the same VM until the host stops or restarts it. The control plane - records a VM that stops without being asked as STOPPED and starts it again - when the Dot has work (section 9.5). - -There is no long-running browser service. The engine's `BrowserManager` -starts one `invisible-playwright-mcp` process per launched browser identity -(section 6), through `dot-agentd relay`, so the process runs as `dot` and not -as `dotengine`: its environment is the one the relay builds for `dot` (none of -the engine's variables, never the OpenRouter key) plus the variables of -section 6. The engine closes them with the rest of its work (section 8.8). - -Everything the model runs (the engine's `exec`, `POST /v1/exec`) therefore runs -as `dot`, which has no way to become root: the model cannot power its computer -off through sudo. A Dot that stops without being asked is still recorded as -STOPPED and started again when it has work (section 9.5). - -### 4.2 Guest filesystem - -```text -/etc/invisible-dots/config.json written by cloud-init: dotId, token (0600, owner dotagentd: dot-agentd's alone) -/opt/invisible-dots/ the runtime ISO, read-only: the engine's source (engine/), dot-agentd and the units -/opt/invisible-dots-engine/ the engine's Python environment, built into the golden image (section 3.3) -/home/dotengine/state/ the engine's state (0700): engine.sqlite, which holds the Dot's tables and the - transcripts (section 8.8), and cron/jobs.json, the automations -/home/dot/ - workspace/ the engine's agent workspace too: group dot, setgid, 2775 - downloads/ documents/ - memory/ long-term memory notes (files; MEMORY.md is in every prompt; section 8.6) - conversations/ chat/.md and tasks/-.md, what was said (section 8.6) - browsers// - profile/ the browser profile -/var/lib/invisible-dots/ root, 0755 - mcp// INVISIBLE_MCP_HOME for that identity's server (dot, 0700; outside /home/dot on purpose) -/run/invisible-dots/ dotagentd:dotengine 2750 - agentd.sock dot-agentd's local API for the engine (dotagentd:dotengine 0660) -/run/invisible-dots-agent/ dotengine:dotagentd 2750 - agent.sock the engine's API, reached by dot-agentd's proxy (dotengine:dotagentd 0660) -``` - -The home of an identity's MCP server is not under `/home/dot` because the -server saves who its browser is, and for an identity that has a proxy of its -own that proxy with its password included, in -`/sessions/.json` on every `browser_open`: a file under -`/home/dot` would be served by the host's file routes (section 9.6), which read -`/home/dot` and nothing else, and dot-agentd's TCP listener refuses a path -outside it (`403 outside_home`). The directory is made by the image's -provisioner and by `install.sh` (dot owns it, the server runs as dot) and the -`BrowserManager` makes and removes one subdirectory per identity. - -Each socket sits in a directory its server owns and only the other side may -enter, setgid so the socket takes that side's group (`install.sh` writes both -to tmpfiles): dot-agentd's directory is `dotagentd:dotengine`, so the engine -(group `dotengine`) reaches `agentd.sock`, and the engine's is -`dotengine:dotagentd`, so dot-agentd (group `dotagentd`) reaches `agent.sock`. -`dot` is in neither group and owns neither directory: it cannot enter either -one, so it can neither talk to the engine or to dot-agentd's local socket nor -put another socket where the host pushes the key. The engine adds a check of -its own on the connection (section 4.1). - -One engine process owns `engine.sqlite`: it opens it in SQLite's exclusive -locking mode and takes the write lock at once, so a second process on the same -file fails at open ("another engine owns ...") and refuses to start, and the -kernel releases the lock the moment the owner dies, so a restart opens it -again without waiting. Nothing else opens the file; the host reads the guest -only through the engine's API. - -dot-agentd reads the Dot's home from `DOT_HOME` (default `/home/dot`; the -units do not set it). `INVISIBLE_DOTS_HOME` is the host's data directory -(section 3.2) and is never read in the guest: one name, one place. - -### 4.3 Secrets - -The OpenRouter key is never written into the golden image, the runtime ISO or -the seed. After the guest reports healthy, the control plane pushes it over -the guest channel (`POST /v1/agent/secrets`) and the engine keeps it in memory only. A VM -that restarts asks for nothing: the control plane pushes it again on every -READY transition, and an engine process that restarts inside a running VM -(systemd restarts it after a crash) announces itself with an `agent.started` -outbound event, on which the control plane pushes the key and the config -again. The Dot's own token is the one secret in the seed; it only -authorizes requests to this one VM. - -"Memory only" keeps the key off every disk; it is not on its own what keeps -it from the commands the model runs. What does: they run as `dot`, a user that -is neither the engine's nor dot-agentd's, and `dot` cannot become root (section -4.1), so it cannot read another process's memory through root; Ubuntu's Yama -`ptrace_scope=1` lets a process trace only its own descendants, and the -model's commands descend from dot-agentd, not from the engine; and the engine -is a process of another user, so ptrace of it is refused in any case. CPython opens no debugger or inspector -on a signal, so nothing can be asked of the process from outside; the unit's -`LimitCORE=0` keeps the key out of core files and `NoNewPrivileges=yes` keeps -anything the engine starts from gaining a privilege. A change to any of these -reopens the question. - -In the engine the key lives in one object, `KeyHolder` -(`nanobot/dots/secrets.py`): `POST /secrets` sets it, and the one place that -builds the model provider (`nanobot/dots/provider.py`) reads it. It is never -logged, written or put in an environment, and the server logs a request's -method, path and status and never a body. Nor does it leave in an error: -the holder refuses a key that cannot travel in a header (anything but printable -ASCII without spaces: a newline inside a key makes h11 raise `Illegal header -value b'Bearer '`, and the openai client chains that under its own -exception). That rule has one owner, `packages/shared` (`OPENROUTER_KEY_PATTERN` -and `OPENROUTER_KEY_RULE`): the host applies it where the key enters, so -`PUT /api/secrets/openrouter` answers `400 invalid_request` for a key that breaks -it, and the holder applies the engine's copy of the same two constants again on -`POST /secrets` (a repository test keeps the copy equal). The same key again changes nothing: -no new provider is built and no running turn is disturbed, so the host's -pushes at every READY and `agent.started` cost nothing. A new key builds the -provider of the next turn; a running turn keeps the one it began with. The -engine reads no credential from its environment: no provider spec names an -environment variable. - -The secrets of the MCP servers a Dot declares (section 7) travel the same -way: stored encrypted under the Dot's scope as `mcp//`, set and -cleared write-only (section 9.6), pushed in the same `POST /secrets` as the key -(`mcp_secrets`, by server and name, only those the config names), held in -memory by `McpSecrets` (`nanobot/dots/secrets.py`) and masked in the -conversation files. Unlike the key, a secret leaves the engine for the server -that names it, because that is what it is for, as in every MCP host: a -command's secret is an environment variable of its process, which the relay -receives in its own environment (only its name is on the relay's command line, -`--env-from`), and a URL's is a header of its requests. Its value is printable -ASCII, a space allowed (`Bearer `): one rule, `MCP_SECRET_PATTERN` and -`MCP_SECRET_RULE` in `packages/shared`, applied by the host where it enters and -by the engine on `POST /secrets`. A config that no longer names a secret (its -server removed, its name dropped) deletes its value in the same transaction, so -none is kept that nothing uses. - -A secret also never travels in an error: a failed push of the key is reported -by route, status and code only (`the guest did not take the OpenRouter key -(status 502, ...)`), never with the text the guest or a proxy answered, which -could echo the key into a Dot's error, the log or the event log. - -## 5. Host to guest protocol - -### 5.1 Transport - -HTTP/1.1 over TCP. `dot-agentd` listens on port 1024 inside the guest, and -QEMU forwards `127.0.0.1:` on the host to it (section 3.5). Node -calls `http.request({ host: "127.0.0.1", port })`, the same on every host. -Every request carries -`Authorization: Bearer `; `dot-agentd` answers 401 to anything else. -Connections go host to guest only: the guest never connects to the host. Events -flow back over a stream the host opens (5.3). - -A port is not a credential (section 3.5), so the host does not send the token -to whatever listens on one: after a host restart or a QEMU that died, another -local process can listen on a port the host still has on record. Before the -first request that carries the token, the guest client asks -`GET /v1/proof?nonce=<16 random bytes in hex>`, the one route without a token, -and requires `{ proof: HMAC-SHA256(token, "invisible-dots guest proof v1\n" + -nonce) }`, compared in constant time. A listener that cannot answer it never -sees the token, the key or the config; the call fails with -`guest_unproven`, which is not retried. - -### 5.2 dot-agentd routes (TCP port 1024, token required) - -| method and path | body / query | answer | -|---|---|---| -| `GET /v1/proof` | `?nonce=` (32 to 128 lowercase hex characters); no token | `{ proof }`, section 5.1 | -| `GET /v1/health` | | `{ agentd: "ok", agent: , uptime_s }` | -| `GET /v1/system` | | `{ hostname, uptime_s, cpus, mem_total_bytes, mem_available_bytes, disk_total_bytes, disk_free_bytes }` | -| `POST /v1/exec` | `{ command, cwd?, timeout_ms? }` | `{ exit_code, stdout, stderr, timed_out }` (bash -lc, output capped at 1 MiB each) | -| `GET /v1/files` | `?path=` | file bytes | -| `PUT /v1/files` | `?path=`, body = bytes | `204` | -| `GET /v1/files/list` | `?path=` | `{ entries: [{ name, type: "file"\|"dir"\|"other", size, mtime }] }` | -| `GET /v1/screenshot` | | `image/png` of display `:0` | -| `POST /v1/system/poweroff` | | `202 { status: "powering_off" }` after starting `/usr/bin/sudo -n /usr/bin/systemctl poweroff` detached (the seed lets `dotagentd`, the user the daemon runs as, run exactly that without a password, section 4.1); `500 poweroff_failed` when it cannot be started. How the control plane stops a VM (section 3.4) | -| `* /v1/agent/` | | reverse proxy to `unix:/run/invisible-dots-agent/agent.sock` at `/` | - -The three file routes are limited to home on the TCP port, the host's door: -the path is resolved with every symbolic link followed and a real location not -under home is a `403 outside_home`. On `agentd.sock` they take any path the -Dot's user may open. On both, the routes act as `dot` (section 4.1): the daemon -runs as `dotagentd`, which cannot open the Dot's files, and a file one of them -creates belongs to `dot`. So a path `dot` may not open, the Dot's token file -and the engine's state among them, is a `403 permission_denied` on both. - -The same routes, without `/v1/agent`, `/v1/proof` and `/v1/system/poweroff`, -are served on `agentd.sock` for the engine (no token: section 4.2 says who can -reach the socket), plus one route of that socket only: - -| method and path | body | answer | -|---|---|---| -| `POST /v1/proc` | `{ argv, cwd?, env?, tty?: { cols, rows } }`, with `Connection: Upgrade`, `Upgrade: dots-proc/1` | `101 Switching Protocols`, then frames both ways (one type byte, a big-endian uint32 length, the payload): from the caller `i` input, `e` end of input (^D on a terminal), `r` size (uint16 cols, uint16 rows), `s` a signal number for the process group; from dot-agentd `o` output, `E` error output (none on a terminal), and last `x`, the JSON `{ exit_code, signal? }`. `426` without the upgrade, `400` for an empty program or a bad cwd | - -It runs the program as `dot` (the daemon changes to that user for the program; -section 4.1), without a shell, in its own process group, on a pseudo-terminal -when asked (a new session whose controlling terminal it is, and which `dot` -owns). The program's environment is the daemon's with `HOME`, `USER`, `LOGNAME` -and `SHELL` of `dot`'s account, plus `DISPLAY` and the request's `env`. -The process lives exactly as long as the connection: a caller that goes away -takes the whole group with it. Its client is `dot-agentd relay [--socket P] -[--cwd DIR] [--tty] [--env NAME=VALUE]... [--env-from NAME]... -- PROGRAM -[ARGS...]`, which copies its own standard input and output through and exits with the program's code -(128 plus the signal number when a signal ended it); with `--tty` and a -terminal on its input it puts that terminal in raw mode and forwards its size -changes. The engine runs the model's every command through it, and reads and -writes the model's files through the `/v1/files` routes of the same socket -(section 8.8). Sleep, stop and reboot are the control plane's decisions, -and the agent's socket offers no poweroff. Nor can the model power its computer -off through sudo: `dot` has no sudo rule (section 4.1), and cannot read the -token in `/etc/invisible-dots/config.json` either. A Dot owns its computer, not -the daemon that serves it. What the control plane guarantees is the outcome: a -VM that stops without being asked is recorded as STOPPED, and started again -when its Dot has work (section 9.5). -Paths in file routes are resolved against `/home/dot` when relative. - -A command of `POST /v1/exec` runs in its own process group, and dot-agentd -kills the whole group when the command reaches its timeout and when the -request that started it goes away: a cancelled call, a tool cut at the stop -grace and an agent that died all take their command with them. A command that -runs through a relay (the engine's) is held the same way: the relay is a child -of the engine that lives as long as the remote command, and ending it ends the -remote process group. Only what the -command detached into another session outlives it. - -### 5.3 Agent routes (the Dot's engine, reached as `/v1/agent/...`) - -| method and path | body | answer | -|---|---|---| -| `GET /health` | | `{ status: "ok"\|"starting", state: AgentState, openrouter_configured: bool, browser: { identities: n, open: n }, checks }`; `identities` is the number of rows and `open` the number of identities with a live browser | -| `POST /secrets` | `{ openrouter_api_key, mcp_secrets }`; `mcp_secrets` is `{ : { : value } }`, the secrets of the config's MCP servers that are set | `204`; `400 invalid_secret` holds neither, and never says a value | -| `PUT /config` | `DotRuntimeConfig` (section 7) | `204`, validated, persisted in the engine's database (`dots_kv`) and projected onto the engine's settings in process (section 8.8); a config that does not validate is `400 invalid_config` | -| `POST /events` | `InboundEvent` | `202 { accepted: true }` | -| `GET /events/stream` | `?after=` | `text/event-stream`, one SSE message per outbound event, `id: ` | -| `GET /state` | | `{ state, current_task_id, pending_approval }`; `pending_approval` is the id of the oldest approval the engine waits on, or `null` | -| `GET /browser-identities` | | `{ identities: BrowserIdentity[] }`, oldest first; an identity says `hasProxy` and nothing of its proxy, which is a secret | -| `POST /browser-identities` | `{ name, proxy? }`; `proxy` is an explicit option: absent, `null` or blank means none, the normal case | `201 BrowserIdentity`; `400 invalid` (a name that is empty or too long, or a proxy that is not a string; a proxy that is a string is kept as written and judged by invisible-playwright-mcp when the identity launches), `409 limit` (the 20 identities the engine keeps at most) | -| `GET /browser-identities/:id` | | `BrowserIdentity`; `404 not_found` | -| `DELETE /browser-identities/:id` | | `204` after the identity's browser is closed and its directory removed; `404 not_found` | -| `GET /browser-identities/:id/frame` | | `200 image/jpeg` (`Cache-Control: no-store`), one frame of the identity's window, taken with the server's `browser_watch`; `404 not_found`; `409 not_open` when the identity is closed; `503 busy` when a call of the Dot held the identity for longer than 5 seconds; `502 frame_failed` (the server has no page to show, or sent a frame that is not a JPEG: the engine is the one owner of that rule, the host passes the bytes on as `image/jpeg`) or `crashed` | -| `POST /browser-identities/:id/close` | | `204` after the identity's browser is closed through `browser_close` and its server has ended; the profile stays. Closing a closed identity is a `204` too; `404 not_found` | -| `GET /tools` | | `{ tools: [{ name, permission, offered, description }], mcp_servers: [{ name, state, error, tools }] }`: the engine's tool table (section 8.8) in its order, then the tools of the declared MCP servers that are connected (permission `mcp.`); `offered` is whether the model is offered the tool now (its permission is not `deny`, and a tool that creates or deletes a browser identity needs the Dot to manage its identities; before the first config, none); `description` is the one in the tool's schema. `mcp_servers` is every declared server by name, `state` `connecting`, `connected` or `failed`, `error` why it failed (section 8.3) | -| `GET /skills` | | `{ skills: [{ name, description, source, path, content }] }`: the Dot's skills (section 8.6) by name, the built-in ones and its own (`source` is `builtin` or `dot`), each with the whole of its SKILL.md | -| `POST /prepare-sleep` | | `204` after the state is flushed and browser sessions are closed; the agent then starts no new work. A model request in flight is abandoned; a tool in flight gets up to 20 seconds to finish and record its result, then is aborted (section 8.7). A `POST /secrets` (the READY procedure of a VM whose stop failed, so no shutdown followed) lifts that, and so does a new inbound event | - -The identity routes and the model's identity tools are one code path, the -`BrowserManager` (section 6), so its limits hold for both. A route answers an -error as `{ error: , message }` with the status of its code: `invalid` -400, `not_found` 404, `limit` and `not_open` 409, `busy` 503, `crashed` and -`frame_failed` 502. The engine has no route that launches an -identity: only the model's tools open a browser, so `browser.identity.launch` -alone decides whether one starts, and `launch_failed` is an answer of those tools, -never of a route. The host may look at an open identity (the -frame) and close it (or delete it); these are the owner's actions, not the Dot's, so -no permission of the Dot applies to them. A frame is no use of the identity: it does -not move it in the least-recently-used order, does not touch `last_used_at`, -and never reopens a browser the server lost (that is `not_open`, and the identity is closed, as for a call that finds it out); it waits for -the call in flight on the identity at most 5 seconds, so a page that asks for a -frame every two seconds cannot hold a browser open or starve the model. - -Outbound events are written to an outbox table in the Dot's database before -they are streamed, with a monotonically increasing `seq`. The host stores the -last `seq` it saved per Dot and reconnects with `?after=`. Nothing is lost when -the control plane restarts or the VM sleeps. - -The engine serves these routes from `nanobot/dots/server.py` (aiohttp), on the -unix socket `INVISIBLE_DOTS_AGENT_SOCKET` names (mode 0660; a socket file left -by a crash is removed first), and binds no TCP listener. Its outbox is the -`dots_outbox` table of `engine.sqlite` (section 8.8). Every outbox row is -written by this one process through `DotStore.write`, which wakes the open -streams after the transaction commits, so the stream replays from the table -after `?after=` (or `Last-Event-ID`) and then sends each row once as it -commits, with a keep-alive comment every 15 seconds; there is no poll. - -### 5.4 Event shapes - -```ts -// guest to host, persisted in the guest outbox -interface OutboundEvent { seq: number; id: string; type: OutboundEventType; ts: string; data: object } -// host to guest -interface InboundEvent { id: string; type: InboundEventType; ts: string; data: object } -``` - -Inbound types: `user.message {text}`, `task.created {task_id, description, -priority}`, `approval.received {approval_id, decision: "approve"|"reject", -note?}`, `system.event {name, data}`. A task cancelled after its -`task.created` may have reached the guest is cancelled there with -`system.event { name: "task.cancelled", data: { task_id } }`. The guest keeps -the id of every inbound event it accepted and ignores one it already has, so -the host may send an event again whenever the outcome of a send is unknown -(section 9.2). An inbound event is recorded and applied in one transaction -(a `user.message` is applied when its chat turn is answered): an event the -guest has accepted is never lost to a restart, and one whose transaction -failed was not accepted, so its redelivery is. - -Outbound types: `agent.started {}` (the first event of every start of the -agent process, section 4.3), `agent.state {state}`, `message.assistant {text, -in_reply_to?, spent_usd?}`, `task.started {task_id}`, `task.progress {task_id, -text, spent_usd?}`, `task.completed {task_id, summary, spent_usd?}`, -`task.failed {task_id, error, spent_usd?}`, -`approval.requested {approval_id, task_id?, tool, permission, arguments, -reason}`, `tool.called {task_id?, tool, permission, decision, ok, -duration_ms, target?, tty?, interrupted?}`, `browser.identity.created|deleted|launched|closed -{identity_id, name}`, `automation.next_run {next_run_at_ms}`, -`memory.updated {conversations, changed, spent_usd?}`, `mcp.server.added {arguments}`. -`interrupted: true` marks a call -the engine stopped during: its outcome is unknown and it was not run again, so -`ok` is false and `duration_ms` is 0. `tty: true` marks a call that started a -terminal session (`exec` with `tty`, section 8.3), which is not what its target -says (the command): a client shows "Started a terminal session: python3". It is -absent for every other call, and the permission table -(`TOOL_PERMISSIONS[...].starts_terminal`) is where a tool says it can. - -`task.progress {task_id, text}` is the model saying what it is about to do: the -text an assistant message of a running task carries beside its tool calls (the -message is not the final answer). It is sent once per such message, in the -transaction that stores the message, so a restart neither loses nor repeats it -and it comes before the `tool.called` of those calls. The text is trimmed and -cut at 2000 characters, the last one an ellipsis. A message with no text beside -its calls sends nothing, and neither does the chat: the answer of a chat turn -is its `message.assistant`, and the final answer of a task is its -`task.completed`. - -`tool.called.target` is what the call acted on, in one line of at most 160 -characters (`TOOL_TARGET_MAX`, in code points as the host's schema counts them; -it refuses a longer, empty or multi-line one), so a client can say "ran `make test`" and not only "exec ok". -It is a name or a place, never content. The permission table -(`nanobot/dots/permissions.py`) gives each tool a function that states what of -its arguments may be shown: `exec` the first line of the command, as it is; -`read_file`, `list_dir`, `write_file` and `edit_file` the path; `find_files` the -query, else the glob, else the path; `grep` the pattern; `apply_patch` the path, -or `N files, first `; `cron` the action and the name or job id (`add daily-standup`); -`exec_session` `input to `, `terminate ` or `output of `, and never -the input; `list_exec_sessions` nothing. The engine computes it when the call -starts and keeps it with the call's intent, so a call cut by a stop is reported -with what it was doing. A call that never started has no intent and no target: -a denied call, a call to a tool that is not offered, one whose arguments did not -fit. The key is then absent, as it is for a tool with nothing to name. - -`spent_usd` on `message.assistant`, `task.progress`, `task.completed`, -`task.failed` and `memory.updated` is the model spend of the session the event belongs to, in USD, -read from the same ledger the cost cap uses (section 8.2) in the transaction -that stores the event, to the hundred-millionth of a USD. On a task's events it -is the spend of the task so far (it only grows, and survives a restart, an -approval and a resume); on the chat's `message.assistant` it is what the chat -spent since its last answer, because the answer takes the chat's spend with it -(a chat that parked a call for approval reports the spend before and after the -approval in the one answer it gives); on `memory.updated` it is what the memory -passes spent since the last one, a pass that failed included (section 8.6). The -engine always sends it (0 when nothing was spent); the schema makes it optional -so events logged before it existed stay valid. The host reads it as the -guest's report: it is never used to enforce anything (the cap is the guest's). - -`mcp.server.added {arguments}` is a call of `add_mcp_server` that ran (section 8.3): the Dot adds an MCP server to its -own config, and `arguments` are the call's, as the model made it. The config is the host's, so the host decides -again: it writes the server only when the entry passes the config's own schema under a name not yet declared, and -the Dot's config allows `mcp.servers.add` or the person approved a call of exactly these arguments (an approval of the -Dot with that tool, its arguments equal as JSON values, keys in any order). That approval is used up in the same -transaction (`approvals.used_at`), so one approval adds one server once: a server the person removes is not brought -back by the call that added it. It writes the server in the transaction that stores the event, with `dot.updated -{name, mcp_server_added, approval_id?}`, and pushes the config; otherwise it logs `mcp.server.refused {name, reason}` -(a call refused for its entry or its name keeps its approval). The server's own permission is left to its default, ask. - -`memory.updated {conversations, changed}` ends a memory pass (section 8.6): the -Dot had been quiet for a while, and the summary model rewrote its `MEMORY.md` -from what was said since the last pass. `conversations` is how many -conversation files it took messages from, `changed` whether `MEMORY.md` is different. A pass -that found nothing new sends nothing, and one that failed sends nothing either: -its spend goes with the next `memory.updated`. - -`automation.next_run {next_run_at_ms}` is when the earliest enabled automation of -the Dot is next due, in milliseconds since the epoch, or `null` when none is (no -job, every job paused, or only one-time jobs that already ran). The engine sends -it each time that value changes, so the last one the host has is the true one: when -the cron service arms its timer, which it does after every change of the jobs (the -`cron` tool) and after every tick, the engine compares -the value with the one it last reported, kept in its database in the transaction -that writes the event, and writes the event only when they differ. A restart -therefore does not say it again, a computer that never had a job has said nothing -(the host takes that as `null`), and a time already past is reported as it is: it -is a run the engine has not made yet, which it makes when it starts (section 8.8). -The host keeps the value in `computers.next_automation_at` and uses it to wake a -stopped computer shortly before the run and to not put one to sleep that is about -to need it (section 9.5). - -The `arguments` of `approval.requested` are what the person decides on, and -they leave the guest: a tool argument that carries a secret is redacted there -by the engine, at the point where the event is built, and nowhere after it: the -host, the web UI and a chat prompt (section 9.8) show the arguments as the event -has them. The engine masks the proxy of `browser_identity_create` (shown as `***`: a -proxy is a secret, and neither its user, its password nor its host is shown), and -removes the user and password of the URL of -`browser_navigate` (the query and the fragment stay: see section 6). The pending -call in the Dot's database keeps the full arguments, so the approved call is made -as asked. - -An outbound event is handed to the event stream only after the transaction -that wrote it to the outbox committed: one written inside a transaction that -rolls back is never streamed, so its `seq` cannot reach the host and then be -reused for another event. - -The control plane adds its own: `dot.created`, `dot.updated` (a saved config, -`{name}`, or a Dot that went to ERROR, `{name, status: "ERROR", error}`, which -the Activity page tells apart), `dot.deleted`, -`computer.state {state}`, `computer.started`, `computer.stopped`, -`task.created`, `task.cancelled`, `task.failed {task_id, error}` (the guest's -type: the host writes it when it gives a task up because its `task.created` could not -be delivered, section 9.2, so a client follows one type whoever ended the task), `approval.resolved {approval_id, decision, -task_id?, note?, always?}` (`task_id` is the task the approval was asked in, so -the events of a task hold the answer beside the request), and the two of a messaging -channel: `channel.status {kind, status, detail?}` (`kind` is `telegram` or -`whatsapp`; `status` is `connecting`, `connected`, `needs_relink` or `error`, -and `detail` never holds a credential), `channel.peer.paired {kind, peer_id, -label}` and `channel.changed {kind, change}` (`change` is `paused`, `resumed` -or `removed`: what the person did, which no status says, so that every view of -the channel follows it). A channel lives in the control plane only: the Dot never sees one, so -no inbound or outbound type names it. - -The last of them is `guest.event.refused {seq, type, problem}`. The host parses -every message of the guest's event stream with the schemas of `packages/shared` -and drops one it refuses (it is not an event the host knows, so it is not -stored as one), because one malformed message must not stall the stream. The -drop is not silent: the host writes this event, in order with the guest's, with -the `seq` and `type` the message carried when it was JSON that carried them -(null otherwise) and the problem in words (at most 300 characters, never a -credential: a schema message names a path and a rule), and moves the cursor -past the `seq` in the same transaction, so a reconnect does not record it -again. The Activity page shows it as "The computer sent an event that was not -read". The two sides cannot drift unseen: `tests/dots/test_wire_shapes.py` -writes one event of every type the engine writes, with each set of optional -keys, into `wire_shapes.json`, and `apps/scheduler/test/engine-shapes.test.ts` -requires that each parses back unchanged (the schemas strip an unknown key); -`apps/scheduler/test/host-shapes.test.ts` writes the events and the config the -host sends into `host_wire_shapes.json`, which the engine's own parsers must -accept with every key. - -The message a person sends is logged as a `user.message` host event -`{message_id, text, origin?}`. `origin` is `{channel, binding_id, chat_id, -external_id}` for a message that came through a channel and is absent for the -web, the CLI and the SDK. The event log is the one place that says where a -message came from, and a reply is routed back by it; the guest receives the -event with `{text}` only. Only code inside the control plane can set an origin -(`Scheduler.sendMessage`): `POST /api/dots/:id/messages` takes `{text}` and -ignores anything else. - -## 6. Browser identities - -- Where a browser exits to the internet. An identity has no proxy of its own - unless the person gave it one: that is the default everywhere (the API, the - SDK, the CLI and the engine accept a create with a name only, and nothing - asks for or fills in a proxy). The browser then inherits the egress of the - Dot's VM: through the VM proxy when the Dot has one, directly otherwise. The - VM proxy is a per-Dot secret (`vm_proxy` in `secrets`, set by - `PUT /api/dots/:id/proxy` or `invisible-dots secret proxy`), a - `socks5://[user:password@]host:port` URL that goes into the seed's root-only - `config.json` at every start; the runtime's `install.sh` then routes the whole - VM through `hev-socks5-tunnel` (TCP, UDP and DNS over `tun0`) and an - nftables table drops every other way out, so nothing leaves when the proxy is - down. Nothing tracks or compares the exit. Its time zone, locale and - geography follow the exit the browser actually uses, which the browser layer - learns from the address-echo services at each launch. A per-identity proxy - stays an explicit option for one identity. It is one optional string, stored as - a secret and handed to `invisible-playwright-mcp` unchanged, in the setting the - library reads (`STEALTHFOX_PROXY`): the library owns what a proxy is (its - schemes, its host and port, the errors it raises for one it cannot use, at the - launch), and invisible_dots has no rule, parser or display form of its own for - it. What is invisible_dots's own is that the secret is shown nowhere (below); - when both a VM proxy and an identity proxy are set, the - identity's proxy is reached through the VM's tunnel, so the exit is the - identity proxy's. An identity proxy is not the way to give a browser a - location: a Dot that should appear somewhere gets that from its VM's egress. -- An identity is a row of the engine's `dots_browser_identities` table (id, - name, proxy, created, last used) and a directory under - `/home/dot/browsers//`. Its id is a slug of its name plus a short - random suffix. The guest's database is the only record of which identities - exist; the host never mirrors the list, it asks the guest. Whether an - identity is open is never stored: it is derived from the live browser - sessions of the running engine, so a file never claims an open browser for a - process that is gone. An identity's proxy, when it has one, is stored as given, password included, in - the engine's database (`dotengine`'s state directory, 0700, which the model - cannot read); nothing of it is shown to a model, a person, an event or a log: - an identity says only that it has a proxy (`hasProxy`). The browser's own process is the one exception, by - decision: it runs as `dot` with the proxy in its environment, so the `dot` - user, and the model through `exec`, can read the proxy of an identity whose - browser is open from that process's `/proc//environ`. The server's - session file is the second copy on disk, again by the server's own design - (it saves who its browser is, the proxy included, in - `/var/lib/invisible-dots/mcp//sessions/.json`, as - `dot`, after every `browser_open`): the model's `exec` can read it, but it is - outside `/home/dot`, so the host's file routes (the API, the SDK, the Files - tab) can never serve it; deleting the identity removes it. That the server - keeps the whole proxy at rest is a defect of the library (its `Work.remember()` saves it; its own - identity file keeps only a digest), which this repository does not patch: the fix belongs upstream, - and the browser smoke checks that no file of `/home/dot` (the profile, a cache, a log) holds the - password and that the port refuses the session file. The proxy is never - on a command line, which every user of the VM can read: the engine tells the - relay the variable's name (`--env-from`) and the relay reads the value from - its own environment, which only `dotengine` can read. -- The fingerprint seed of an identity is stored by the browser layer in the - profile itself (`profile/.stealth-identity.json`). invisible_dots never stores - or passes a seed: the first launch of a profile picks one and every later - launch of the same profile gets the same one back. -- Launching an identity starts one `invisible-playwright-mcp` process over - stdio. Its environment is the allowlist of section 4.1 plus, through the MCP - server's documented settings: its home (`INVISIBLE_MCP_HOME=/var/lib/invisible-dots/mcp/`) - and session id (`INVISIBLE_MCP_SESSION_ID=`), the identity's - profile directory (`/profile`), headed mode, `DISPLAY=:0`, and the - identity's proxy only when it has one (otherwise no proxy variable is set at - all), and one setting of the libraries it - uses: `INVISIBLE_CORE_AUTOFIX=off`, which stops `invisible_core` from - reinstalling itself from the package index at a launch when its version - drifts, so a drift fails loudly instead of installing files outside the - hashed lock. The names of the browser layer's own - settings are written in one place, `packages/shared/src/protocol.ts` - (`ENV.PROFILE_DIR`, `ENV.HEADLESS`, `ENV.PROXY`, - `ENV.CORE_AUTOFIX`). The browser therefore runs - on the Dot's desktop and shows up in its screenshots. -- A launch also connects out, which no setting here turns off: to the - address-echo services of the library (`api.ipify.org`, `icanhazip.com`, - `checkip.amazonaws.com`, from the VM's egress, or through the identity's proxy - when it has one) to learn the exit address for the timezone and locale; to the library's launch counter - (a download of a file of `feder-cr/firefox_antidetect_patch`'s releases, - switched off only by a preference the MCP server gives no way to pass); and - it probes the exit's capabilities and caches the answer in - `/tmp/exit_capability.json`; and it keeps its GeoIP database current from - its GitHub release. -- The model never calls `browser_open`: the browser manager opens and closes - each identity's browser, and starts its server with - `INVISIBLE_MCP_HOST_MANAGED=1`, under which the server serves its `main` - browser alone (no tool takes `browser`) and its instructions are the page - rules, without `browser_open` and the `support` browser. The model's page - tools are that server's own (section 8.3), each with the `identity_id` that - picks the open identity's server. The manager opens the browser with - no `profile`, `proxy` or `seed` argument, so for a first launch the - environment above is the only source of those values. A later `browser_open` - with no argument is a reopen, and the library then takes who the browser is - from its session file (`/var/lib/invisible-dots/mcp//sessions/.json`: seed, proxy and - profile directory, written at the first open) and does not read - `STEALTHFOX_PROXY` or `STEALTHFOX_PROFILE_DIR` again: the file wins. An - identity's proxy never changes after it is created (there is no route that - edits it), so the two agree, but a proxy edited in the engine's database would - not reach a browser whose session file already exists. -- At most 3 identities are open at once (roughly 0.8 GB of memory each). - Launching one more closes the least recently used. At most 20 exist. Both are - the engine's own (`DEFAULT_MAX_OPEN` and `DEFAULT_MAX_IDENTITIES` in - `nanobot/dots/main.py`), not settings: the Dot manages its identities itself. -- Launching is explicit. A browser action on an identity that is not open does - not launch it: it fails with "identity is not open; call - browser_identity_launch first". Each permission therefore decides only its own - action, and `browser.identity.launch: deny` cannot be got round by navigating. - The first launch of a machine can take minutes while the engine downloads the - browser; the launch asks again (2 s doubling to 30 s) for up to 15 minutes. - A launch waits only for its own browser: a config change, a close, a delete or - a create of another identity does not wait for it, and neither does the - answer of `PUT /config`, which closes the browsers beyond a lower `max_open` - (least recently used first) after it has answered. A close, a delete or a - launch of an identity that is still opening waits for that launch to finish. -- Closing a session asks the browser to close first (up to 30 s, so Firefox - flushes its profile), then stops the process; the profile stays on disk. - Deleting an identity closes it and removes its directory. The engine closes - every open browser on prepare-sleep and on SIGTERM. -- The only browser of a Dot is `invisible-playwright-mcp`: no tool of the - engine browses any other way, and the engine and the image carry no other - browser, browser library, or web fetch or search tool. -- Screenshots. `browser_take_screenshot` (and `browser_click_at`, which answers - with the page after the click) and `computer_screenshot` return an image - the model looks at. The transcript, the outbox and the events never hold its - bytes: the tool's stored result is its text and `[screenshot, 1280x720, not - stored]`, and the engine adds the newest three images of a turn to each model - request in one user message after the last tool message. A later turn - replays the placeholder; the model takes another screenshot when it wants one. -- Secrets in the arguments. The proxy has no place in anything shown: - `approval.requested` carries `browser_identity_create`'s proxy as `***` and `browser_navigate`'s URL without its user and password - (the parked call keeps the real ones, so the approved call runs as asked), and - no `target` of `tool.called` holds typed text. The query of a URL is not - masked (owner decision): a - query string is how a model that was talked into it sends data out, so it is - what the approver has to see, as with the command of `exec`; a person who does - not want it in a chat turns `show_arguments` off. The `text` of - `browser_type` and the `value` of `browser_select_option` stay in the arguments - of an approval when `browser.act` is set to `ask`: whether a field is a - password cannot be known from the arguments, so a person approving typing sees - what is typed. That includes a chat that carries approvals (section 9.8, not - end-to-end encrypted): a person who does not want typed text there turns - `show_arguments` off for the channel. - -## 7. Dot configuration - -YAML in, validated by one schema in `packages/shared` (zod): - -```yaml -name: fare-watch # [a-z0-9-], 1..40 -instructions: > # optional: how it should work, in its system prompt - Write findings to ~/workspace/fares.csv. -model: - provider: openrouter # the only accepted value - id: z-ai/glm-5.3-flash # any OpenRouter model id -models: # optional per-role models, OpenRouter ids; the roles: summary - summary: openai/gpt-5-mini # writes the summary when the thread outgrows the model's window (section 8.6) -computer: - cpu: 2 # 1..16 - memory: 4gb # 2gb..64gb - disk: 40gb # 20gb..1024gb - idle_timeout: 15m # sleep after this long with nothing to do; 0 = never -permissions: # allow | ask | deny, keyed by permission name - computer.exec: allow - browser.identity.delete: ask -limits: - max_steps_per_task: 60 # model turns before a task is failed - max_cost_per_task_usd: 1.00 # USD of model spend of a task or a chat turn, 0.01..100; the last request may exceed it (section 8.2) -mcp_servers: # optional: MCP servers whose tools the Dot may use, by name - time: # [a-z0-9-], 1..32, starting with a letter or a digit - command: uvx # run on the Dot's computer, as dot, in its home - args: [mcp-server-time] - env: {LOG_LEVEL: warning} # optional: environment written here - secrets: [TIME_API_KEY] # optional: environment variables whose values are secrets (section 9.6) - timeout_s: 120 # optional: the longest one call may take, 1..600 - startup_timeout_s: 60 # optional: the longest it may take to start, 1..600 - search: - url: https://search.example/mcp # streamable HTTP, or SSE for a URL ending in /sse - headers: {X-Client: dots} # optional - secrets: [Authorization] # optional: headers whose values are secrets -``` - -A Dot has no goal: what it is for is what its person asks of it, in the chat, in a task or in its -instructions. Nor does it have browser settings: it manages its identities itself, within the -engine's limits (section 6). Nor does it set any token limit: every request uses its model's own -context window and longest answer, as OpenRouter publishes them (section 8.5). A config saved with -any of these before (migrations `0010_dot_has_no_goal.sql`, `0011_dot_has_no_browser_settings.sql` -and `0012_dot_has_no_context_tokens.sql` take them out) is refused as an unknown key. - -The ranges and defaults of the numbers in that file (`computer.cpu`, `memory`, -`disk`, the default `idle_timeout`, `limits.max_cost_per_task_usd` and -`limits.max_steps_per_task`) are `CONFIG_BOUNDS` in -`packages/shared`. The schema takes its numbers from it and -so does the web client's form, so a slider can never offer what the API refuses. - -`models` has one role, `summary`, and no other: the roles are what the engine -asks a model for, and a role it never asks for would be a setting that does -nothing, so a config that names another is refused as `unknown model role -"fast" (the roles are: summary)` (`MODEL_ROLES` in `packages/shared`, copied -into the guest's `protocol.py`). The role's value is an OpenRouter model id -like `model.id`; without it the Dot's own model writes the summary. A config -push that changes it applies from the next turn. - -The permission names are the ones of `PERMISSIONS` in -`packages/shared/src/tools.ts`: those the tools of section 8.3 exercise, and no -others (a Dot has no tool for web reading or search, sub-agents or messaging, -and its notes are files written with `files.write`). A config that names any -other permission is refused as unknown. `PERMISSION_INFO` in the same file gives -each one the label, description and risk (`low`, `medium`, `high`) a person is -shown when they decide on it. - -Defaults for permissions not listed: everything under `computer.*`, -`files.*`, `browser.*` and `memory.*` is `allow`, except -`browser.identity.delete`, which is `ask`; `automations` is `ask`. Any -permission name the registry does not know is `deny`. - -`DotRuntimeConfig` (what `PUT /config` sends to the guest) is the same object -minus `computer`, with `permissions` resolved by the host: one decision for -every permission the registry knows, defaults applied. The guest applies the -map as it is and denies a permission missing from it, so the defaults live in -one place (`resolvePermission` in `packages/shared`). - -`mcp_servers` are the MCP servers the Dot may use, declared the way -Claude Code (`.mcp.json`), Codex (`[mcp_servers]`) and nanobot declare them: a -`command` with its `args` and `env`, or a `url` with its `headers`. The person -declares one in the settings or the YAML, and the Dot can add one itself, as -NanoClaw's agents can (`add_mcp_server`, section 8.3), once it has installed the -program the server needs (section 4.2): its permission, `mcp.servers.add`, asks -by default, the person reads the whole entry on the approval, and the host checks -that approval again before it writes the server (section 5.4). A stdio server -runs as `dot`, with what `computer.exec` already gives the Dot, so adding one -gives the Dot no new power over its computer: what it adds are tools, and each -asks under the server's own permission until the person allows it. Only the -person changes or removes a server. A server's name is the prefix of its tools -as the model sees them, `mcp__` (nanobot's naming), so it holds no -`_` and the engine reads the server back from a tool's name. `secrets` names the -environment variables (of a command) or headers (of a URL) whose values are -secrets: they are set apart from the config, write-only (section 9.6), never in -it, so the config can be shown, exported and edited as YAML. A secret named twice, -or named as a value the entry also writes, is refused. - -Each declared server has a permission, `mcp.`, that covers all of its -tools: what a server's tool does is that server's to say, which the host cannot -know, so a server is one decision, as it is in Claude Code's permission rules -(`mcp__`), and it asks by default. Its risk is `high`, as running -commands is. A permission of a server the config does not declare is refused, -and `resolvePermission` denies it. `MCP_SERVER_NAME_PATTERN` and -`MCP_TIMEOUT_BOUNDS` and `MCP_STARTUP_TIMEOUT_BOUNDS` (Codex's `tool_timeout_sec` and -`startup_timeout_sec`) are in `packages/shared`; the guest's `protocol.py` keeps a -copy of the name rule. The servers are listed by name everywhere (the status, -the prompt, the settings): the database keeps a config as `jsonb`, which does -not keep the order of an object's keys. - -## 8. Agent runtime - -Sections 8.1 to 8.7 are the contract of a Dot's runtime. Section 8.8 is the -engine that keeps it (`invisible_engine_dots/`), how it keeps it, and what it -does not do yet. - -### 8.1 States - -`IDLE -> THINKING -> EXECUTING -> (THINKING | WAITING_APPROVAL) -> DONE -> IDLE` - -`DONE` follows a failed unit as well as a completed one. `THINKING` is a model -request in flight; `EXECUTING` is a tool running. Every transition is an -`agent.state` event. The protocol also names `PLANNING` (the model's answer -being turned into tool calls); the engine has no such step and never reports it. - -### 8.2 Work - -The runtime is event driven. Work arrives as `user.message` (a chat turn in -the Dot's single persistent conversation), as `task.created` (queued locally, -run one at a time in priority order, then creation order) or as the firing of -one of the Dot's own automations (section 8.8, answered in the chat). A task -ends when the model answers without tool calls (`task.completed`, the answer is -the summary), or when an approval is rejected and the model gives up. It fails -(`task.failed`, with a reason the owner can read) when it exceeds -`max_steps_per_task`; when a model request fails for good after the retries of -section 8.5, or the run ends without an answer; or after being interrupted -three times (section 8.7). A chat turn has the same limits; a failed chat turn -answers "I could not answer: ...". - -`limits.max_cost_per_task_usd` caps the model spend of a task, or of the chat -between one answer and the next. The events that end work carry the spend (section 5.4). The cost of a request is what OpenRouter reports for it in the usage -of the last chunk of its response (`usage.cost`, USD; for a BYOK request the -upstream cost it reports beside it is added, which may count more than was -charged and never less). The engine adds the cost of every response of a turn, -the requests of the turn, their retries and the summary requests alike, to the -spend of its session in the Dot's database as each arrives, and checks the -spend before every model request: when it has reached the cap the turn stops -without asking again, and fails with `stopped: the task reached -limits.max_cost_per_task_usd (spent 1.0423 USD of 1.00)` (`task.failed`'s -`error`; "the turn" instead of "the task" for the chat, which then answers "I -could not answer: ..."). A request cannot be priced before it is answered, so -the cap may be exceeded by the last request, and an answer that crosses it is -delivered and the task completes: the cap only stops the work from going on. A -request that failed has no cost and counts for nothing: OpenRouter reports the cost in the last chunk of a -stream, so a stream that stalled or was cut after tokens were streamed (and billed) has no figure to count, and -neither has an attempt the retries of section 8.5 went past (4 attempts a request). The cap can therefore be -exceeded by those partial streams, each at most the cost of one full response. A task's spend is kept -across a restart, an approval and a resume (a task that was cut by a crash goes -on from what it had spent); the chat's spend starts again with each answer it gives, so an approval or a -restart does not reset it. -A lowered cap applies from the next turn, like the step limit. A response that -reports no cost fails the turn, `stopped: OpenRouter reported -no cost for a request, so limits.max_cost_per_task_usd cannot be enforced`, -before anything is done on that response: its tool calls are not written and do -not run, or its answer is not delivered and the task does not complete (a -summary request is not acted on this way, so the failure comes at the next -check). The note is a column -of the same ledger row as the money (`dots_spend.unpriced`), so a restart does -not forget it: the cap never runs blind. A request abandoned by a sleep may have cost something -that was never reported; that gap is at most one request a sleep. - -### 8.3 Tools - -Function names use `_` because OpenAI-style function names cannot contain -dots. Each tool declares the permission it needs, in one table -(`nanobot/dots/permissions.py`) that offers the model its tools, decides every -call and reports the permission of each `tool.called`. A tool that is not in -the table is neither offered nor allowed. Every permission named below is one -of `PERMISSIONS` (section 7); a tool cannot take a permission the host's config -does not know. - -| tool | permission | what it does | -|---|---|---| -| `exec` | `computer.exec` | runs a command through `bash -lc` in a working directory with a timeout (60 s by default, 600 s at most); with `yield_time_ms` it returns while the command still runs, which makes it a background job; with `tty: true` it runs on a pseudo-terminal, as a background job that `exec_session` drives (no new tool, the same permission) | -| `exec_session` | `computer.exec` | sends input to, waits for, reads or terminates a background job (a terminal's output is the text of its screen, section 8.8) | -| `list_exec_sessions` | `computer.exec` | lists the background jobs | -| `read_file` | `files.read` | reads a text file, by line window; reports a binary file as binary | -| `list_dir` | `files.read` | lists a directory, optionally recursively | -| `find_files` | `files.read` | finds files by path terms, glob or type | -| `grep` | `files.read` | searches file contents by regular expression | -| `write_file` | `files.write` | writes a whole file | -| `edit_file` | `files.write` | replaces text in a file | -| `apply_patch` | `files.write` | applies a list of structured edits (replace or add) to files, with a dry run | -| `cron` | `automations` | adds, lists and removes the Dot's own scheduled automations | -| `computer_screenshot` | `computer.screenshot` | takes a screenshot of the Dot's whole desktop and shows it to the model (no arguments) | -| `browser_identity_list` | `browser.identity.list` | lists the browser identities with their status (open or available), whether each has a proxy of its own (and nothing of the proxy), and the two limits | -| `browser_identity_create` | `browser.identity.create` | makes an identity, closed: `name`, `proxy?` (an explicit option that the model leaves out unless the person gave one) | -| `browser_identity_delete` | `browser.identity.delete` | closes an identity and deletes it with its profile: `identity_id` | -| `browser_identity_launch` | `browser.identity.launch` | opens the browser of an identity, closing the least recently used one at `max_open`: `identity_id` | -| `browser_identity_close` | `browser.identity.close` | closes the browser of an identity, keeping its profile: `identity_id` | -| `browser_navigate` | `browser.navigate` | the server's; the engine refuses a URL that is not `http://` or `https://` before it reaches the server (`file:`, `about:`, `view-source:`, `data:` and `javascript:`), so the permission to navigate is not a permission to read files | -| `browser_snapshot` | `browser.read` | the server's: the page's interactive elements with their selectors and coordinates | -| `browser_read_text` | `browser.read` | the server's: the text of the page or of the element a selector names | -| `browser_read_html` | `browser.read` | the server's: the page's HTML, cleaned | -| `browser_take_screenshot` | `browser.read` | the server's: a screenshot of the page, shown to the model | -| `browser_evaluate` | `browser.read` | the server's: reads with a script, and refuses one that acts on the page | -| `browser_click` | `browser.act` | the server's: clicks the element a selector names | -| `browser_click_at` | `browser.act` | the server's: clicks a point and answers with the page after it | -| `browser_type` | `browser.act` | the server's: fills a field key by key, a long text going on in the background | -| `browser_press_key` | `browser.act` | the server's: presses a key or a shortcut | -| `browser_select_option` | `browser.act` | the server's: chooses an option by its label or its value | -| `browser_upload_files` | `browser.act` | the server's: attaches files of the dot user to a file field | -| `add_mcp_server` | `mcp.servers.add` | adds an MCP server to the Dot's own config: `name`, a `command` with `args` and `env` or a `url` with `headers`, the names of its `secrets`, and a `reason` for the person; the host checks the permission again and writes it (section 7) | - -The page tools are `invisible-playwright-mcp`'s, the only browser of a Dot, -offered as an MCP host offers a server's tools: the server's name, description -and input schema, plus the `identity_id` of the open identity whose server the -call goes to. The engine reads them, with the server's instructions, from the -server its Dot has, when it starts: `nanobot/dots/server_capture.py`, run as dot -by the python of the installed `invisible-playwright-mcp`, starts the server as -the engine starts it and prints what it answers (`nanobot/dots/server_interface.py`). -A Dot keeps the golden image it was made from and takes every new runtime disk, -so the engine never assumes a version of the server: it offers the tools the -server it finds serves. A page tool that server does not serve is not offered -(`GET /tools` says so), and a Dot whose server cannot be read offers no page -tool, and a launch says why. Which of the server's tools the model is -offered is the permission table's: `browser_open`, `browser_close`, -`browser_list` and `browser_status` are the manager's (the identity tools), and -`browser_watch` is the UI's frame. The server's instructions go into the -system prompt whenever a page tool is offered, as an MCP host carries a -server's instructions. A browser action on an identity that is not open does -not launch it, it says so (section 6). An image the server answers with is -shown to the model and not stored (section 6). - -The tools of the MCP servers the person declares (`mcp_servers`, section 7) -are not rows of the table: which they are is each server's to say, at run time. -`nanobot/dots/mcp_servers.py` keeps one nanobot `MCPProvider` per declared -server, on a registry of its own, as the browser has one per identity, so -spawning, `initialize`, `tools/list` and the per-call timeout (`timeout_s`) -are the client's. The client never reconnects a server and never repeats a -call: a server whose process exited or whose connection dropped is reported -(`on_terminated`), and the call that found out fails with its error, since the -server may have run it already. What is the Dot's: - -- where it runs. A `command` is started through `dot-agentd relay`, as the - user `dot`, in its home, like every program of the model: it reads and writes - what the Dot's commands can, and nothing of the engine's. Its `env` is passed - with `--env`, its secrets with `--env-from` (their names only, the values in - the relay's own environment), so no secret is on a command line. A `url` is - reached from the engine, its secrets as headers. -- when. A server is started when it is declared, and again when a config or a - secret changes its entry (that server only). The next turn waits for the - servers being started, each for at most its `startup_timeout_s` (60 s by - default: a first start through `uvx` or `npx` downloads the server). One that - failed (its program is not installed yet, it exited) is started again when the - next turn starts, so a server whose program the Dot installs works from the - next message. One that did not start in time, or whose secret is not set, is - not started again until its entry or its secrets change, so it never holds up - every turn. -- why it is not connected. The client says why a connection failed, and a - command's standard error is drained into a pipe of the engine, of which the - last 2000 characters are kept: the error a person and the model read is what - the server wrote (`npx: not found`, `the API key is missing`), with the - server's secrets masked. -- what the model gets. Each tool of a connected server, under the client's name - `mcp__`, is offered while the permission `mcp.` is - `allow` or `ask`, and the gate decides each call by it (section 8.4). The - system prompt has a section for those servers: each one's instructions (what - it said at `initialize`, cut at 2048 characters as Claude Code cuts them), or - why it is not connected and that it is started again at the next turn. An - image a tool answers with is shown to the model as the browser's are. A call - shows no target in `tool.called`, and an approval shows all its arguments. - -`GET /tools` lists those tools after the table's, each under its server's -permission, and every declared server with its state: `connecting`, -`connected` (with how many tools) or `failed` (with why). - -The tool calls of one response run one at a time, in the order the model -gave them. Only the response's `tool_calls` count: a call written in the -assistant's text runs nothing. The registry validates a call's arguments -before the policy sees it: an unknown tool or invalid arguments never reach -the gate and never run, and the model gets the error. A result comes back -whole: no tool has a fixed cap. A request that outgrows the model's window is -compacted first (old results cleared, then the history summarized, section -8.5); only if it still does not fit are the results of its latest round cut, -to what the window leaves (`ContextGovernor.fit_latest_round`), and the cut says -how many characters came through of how many and how to ask for less. A result -is stored cut only beyond what no compaction could make room for: the window -less the system prompt and the tool definitions. - -### 8.4 Policy - -Every tool call goes through the policy gate before it runs: a function of the -current config, and the only place that denies a tool the config does not -offer. `allow` runs it, `deny` returns an error result to the model, `ask` -emits `approval.requested`, moves to `WAITING_APPROVAL` and persists the -pending call, with its full arguments, in the Dot's database, keyed by the -call's id. A call that needs approval stops the round: later calls of the same -response are not run either, and the turn ends. `approval.received` is -recorded when it is accepted. With `approve` the model is told, in a turn of -its own, that the call was approved and has not run; it makes the same call -again and the gate lets that one call through, once. With `reject` the model -is told the call did not run, with the note if there is one. A decision is -recorded once: a second one for the same approval is ignored. An approved call -runs at most once. Section 8.8 says how the gate does this. - -### 8.5 OpenRouter - -`POST https://openrouter.ai/api/v1/chat/completions` with -`HTTP-Referer: https://github.com/feder-cr/invisible_dots` and `X-Title: invisible_dots` -(sent only when the base URL's host is `openrouter.ai`, so a stand-in used by -tests never receives them), the key from memory (section 4.3). Tool calling in -the OpenAI format, `tool_choice: "auto"`, every request streamed (with -`stream_options.include_usage`, so the final chunk carries the usage and the -cost, section 8.2). - -Every request uses the whole of what its model can do. The engine reads the -model's limits at the start of every turn from the providers OpenRouter lists -for it (`GET /models/{model}/endpoints`), not once: an engine runs for weeks and -OpenRouter changes a model's providers meanwhile. Of the providers serving it, -the widest window and the longest answer at that window are the model's, and -the model's requests go only to the providers of that window (`provider.only`, -by the slug OpenRouter lists for each): OpenRouter routes a request by the -answer it asks for but not by the prompt it carries, so one sized for the -widest window could otherwise land on a narrower provider and fail there. This -gives up the narrower providers as fallbacks; when none of the widest can -serve, OpenRouter refuses the request ("No allowed providers are available") -and the turn fails with those words. A model OpenRouter does not know fails the -turn in words; a router, which lists no provider, gets no limits and no route. Each request -sends the longest answer as `max_tokens`, or what the window leaves after the -prompt when that is less (OpenRouter refuses more, and without a `max_tokens` -each provider applies a default of its own), so no answer is cut shorter than -the model allows. A response cut there all the same (`finish_reason: -"length"`) has none of its tool calls executed: what it said is stored without -them and the model is asked to go on, a bounded number of times; a response cut -before any text (the budget went to a call that never finished) is told that -nothing ran and to do the work in smaller steps. Retries with exponential backoff on 429 and 5xx (at most 4 -attempts, honouring `Retry-After`). A tool result is cut only where the -model's window ends (section 8.3), with a marker that says how much came -through. - -### 8.6 Memory - -- Working memory: the thread is append-only in the Dot's database; what is - sent is bounded only by the model's own context window (section 8.5). Before - every model request the runner measures the request (the system prompt, the - tool definitions and the messages; the provider's reported usage when it - matches) against that window less the room kept for the answer (20000 - tokens, or the model's longest answer when shorter, opencode's rule), and - compacts it when it does not fit, in two steps: - 1. old tool results are cleared: walking back from the newest, the first - 40000 tokens of results stay, and the older ones become a short note - saying the tool can be called again, when that frees 20000 tokens or more. - The calls, the reasoning and every message stay as they are. This needs no - model call; on SWE-bench it costs half as much as a summary for the same - solve rate (JetBrains, "The Complexity Trap", arXiv 2508.21433); - 2. only when the request still does not fit, the thread is replaced by a - summary its model writes: a handoff for the model that resumes the work - (Codex's framing, under the headings of OpenHands' summarizing prompt - plus the approvals and files of the Dot), after which the person's latest - messages are kept as they wrote them, newest first up to 20000 tokens - (Codex's rule). When the thread is too long for the summary model, its - oldest messages go first, each with the results of its calls; when the - model cannot summarize at all, the messages themselves, up to half of its - budget, are the summary. - The summary is stored with the session, at the boundary it covers, when the - turn ends, and the next request is the system prompt, the summary and the - thread after it. `models.summary` (section 7) names another model for the - summary request. That request goes through the same metered provider, so its - real cost counts toward the cap (section 8.2); it works within that model's - own window; and it carries no tool definitions, because a model other than - the turn's may not accept them (the turn's own model keeps sending them so - that its prompt cache is reused). `THIRD_PARTY_NOTICES.md` names the three - projects this takes from. -- Past conversations: the transcripts live in `engine.sqlite`, which the - model's commands cannot read, and a long chat reaches the model only as a - summary of its older part. So after every turn the engine also writes what - the chat and the task said to `/home/dot/conversations` (`conversations.py`): - `chat/.md`, one file a day of the chat, and `tasks/-.md`, - one a task, each message under a heading with its time, each call a line (not - what it returned). A file is written whole from the transcript, and the first - turn after an upgrade writes the chat and every task from before. The files - are under `/home/dot`, which the host's file routes serve, so nothing a - transcript holds that the events mask reaches them: a decision on a call is - written as its public line (the call and its target, never the arguments the - continuation hands the model), and every secret the engine knows is masked - as `***` in whatever text carries it, the person's and the model's included: - the proxies of the browser identities, gathered in `dots_kv` as they appear - and never forgotten (a file is written again whole, after an identity may - be gone), and the OpenRouter key, from the holder's memory only. The memory - pass masks them in `MEMORY.md` the same way. The system - prompt says to search there with grep when something said before matters, and - before advice or a recommendation for the person. On LongMemEval (500 - questions over ~115K tokens of dated chats; `tests/bench/README.md`) this took - a Dot from 7.8% to 88% of the answers; searching plain files is also what - remembers best in the published measures (the agent-memory runs of - LongMemEval, "Is Grep All You Need?"). -- Long-term memory: notes, one file each, in `/home/dot/memory` on the Dot's - computer, which the Dot keeps itself, as Claude Code keeps its own. One of - them, `MEMORY.md`, is given to the Dot in every prompt, whole up to 25000 - characters (cut there, with a word to shorten it, as Claude Code does): what - it should always know about the person, each fact with its day, and a line - for every other note. The system prompt says where the notes are, what goes - in MEMORY.md and in the others, that a note no longer true is changed or - deleted, and names the 20 most recently changed. The Dot reads and searches - them with the file tools (`read_file`, `grep`, `find_files`: `files.read`) - and writes them with `write_file`, `edit_file` and `apply_patch` - (`files.write`); a note is a file, and the host sees it as the `tool.called` - of the call that wrote it, with its path as the target. A listing of the - folder that fails (dot-agentd not answering) leaves the names out of that - prompt and the turn goes on. -- The memory pass (`memory_update.py`): a model often does not think to write - what it learns about the person, so once no turn has run for 5 minutes after - a turn (or after a start), the engine takes the conversation files whose time - is after the newest it took last, and of each only the messages it has not - taken yet (a file is written again whole after every turn but only grows at - its end, so a long day's chat is read once), and has the `summary` model - rewrite MEMORY.md from them and the current MEMORY.md, in one request with no tools - (`templates/agent/memory_update.md`: atomic facts with their day, a changed - fact replacing the old one, nothing copied from what a web page or a command - said). Files that do not fit one request go in several, oldest first, a long - one cut between its messages, each request building on the MEMORY.md the - last wrote. A pass runs beside nothing (a turn that starts does not wait for - it); a sleep or a stop cancels it, and the next quiet spell takes it up. An - answer that is cut, empty or unpriced writes nothing, and neither does one - whose MEMORY.md the Dot changed while the request ran: the next pass takes - the files again. A failed pass is tried again only after the next turn. Its - spend has a ledger of its own, capped like a task's (`max_cost_per_task_usd`), - and `memory.updated` (section 5.4) reports it. Measured on LongMemEval's - questions that need what the person likes: 65% without MEMORY.md, 83% with - it, the same as the best of the memory systems tried (Hindsight), at about a - tenth of its cost ($0.025 a pass over 115K tokens with glm-5.3-flash, as - the product ran it; 90.4% of 115 mixed questions); Mastra's and LangMem's methods made 80%, an agent doing the - pass with file tools 75% at five times the cost (`tests/bench/README.md`). -- Workspace memory: `/home/dot/workspace` and `/home/dot/memory`, reached - through the file tools. -- Skills: how the Dot does a kind of task, one folder each with a SKILL.md, the - Agent Skills layout Claude Code and upstream nanobot read (a frontmatter whose - `name` is the folder's and whose `description` says in one line when it applies, - then the steps). Built-in ones ship with the engine, beside its package - (`invisible_engine_dots/skills/`, on the runtime disk at - `/opt/invisible-dots/engine/skills/`); the first, `invisible-playwright`, says how - to use the browser identities: which one to use, opening and closing one, - and when a plain download is simpler than a browser; how to drive a page - (finding a page, the order a page cannot tell from a person, when to say a - task is impossible) is the browser server's instructions, which the system - prompt carries (section 8.3). The Dot writes its own under `/home/dot/skills//SKILL.md` with - the file tools as it learns, and one of its own replaces a built-in one of the - same name. The system prompt names every skill with its description and the path - of its file, read again each turn (`nanobot/dots/skills.py`), and says to read the - file before a task it covers and how to write one: only the names and the - descriptions are in every request. A file that does not parse is left out and - logged. `GET /skills` (section 5.3) shows the same list to the host. - -### 8.7 Crash recovery - -The engine can die at any point (a crash, a kill, a power cut) and systemd -starts it again. What it guarantees: - -- One process owns the Dot's database (section 4.2). -- Every commit point is one transaction that includes the outbox rows - describing it; section 8.8 lists them. -- An intent left without a result is a call the engine stopped during. When a - unit is entered (a start, and the beginning of every turn), such a call is - never run again: it gets the result "This call was interrupted before its - result was recorded. It may have taken effect, and it may still be running. - Check the current state before calling it again." (plus, for an approved - call, that the approval was used), and `tool.called` with `interrupted: true` - and the permission and decision of its time. A command dies with the engine - that started it (section 5.2), so "may still be running" covers what the - command detached; "may have taken effect" covers everything it did before it - was killed. -- So a call runs at most once, and the model is told whenever its outcome is - unknown. `tool.called` is written exactly once per call that ran or was - refused (a call waiting for approval, or one that never started, reports - nothing). `agent.state` is at-least-once. -- A unit can end between a response and the results of its calls (a failed - write, a failure, a cancel, a stop). The transaction that ends it answers - every call of the newest assistant message that has no result, by what the - database holds for the call (section 8.8). A thread never keeps a call - without its result, which the provider would refuse on every later turn. -- Stopping: `POST /prepare-sleep` and SIGTERM abandon a model request in - flight and give a tool in flight up to 20 seconds to finish and commit its - result, then close the open browsers: at most 4 s on SIGTERM, which leaves about - 6 of systemd's `TimeoutStopSec=30` to checkpoint the database, and up to a - browser's own 30 s close on prepare-sleep: only the host's 60 s bounds it, and - the 20 s of grace, the 5 s of cancel wait and the 30 s close fit inside, with 5 s - to spare for the flush (`PREPARE_SLEEP_TIMEOUT_MS` of packages/shared, with the sum - checked by a test of the engine). A tool cut at - the grace keeps its intent, and the next entry of its unit reports it as - interrupted. Measured in the engine smoke before the browsers existed, with a - task's `sleep 70` still running at SIGTERM and the event stream connected: the - process exited 20.4 s after the signal, 9.6 s inside the limit (the 20 s - grace, then the checkpoint, the exec sessions and aiohttp's cleanup in - 0.4 s); with no browser open the close takes no time. -- A task that was running when the engine stopped is started again with a note - that the previous attempt was interrupted, and fails once it has been started - three times (`stopped: the task was interrupted 3 times`). A start the engine - abandoned on purpose (a sleep, a stop) is given back and does not count. -- An answer the chat owes is given: a `user.message` whose text the transcript - holds and that no answer has covered is answered by a turn of its own at the - next start. - -### 8.8 The runtime on the nanobot fork - -The engine (`invisible_engine_dots/`) carries the contract of sections 5.3 and -5.4 and the runtime of this section on nanobot's tool-calling runner, in -`nanobot/dots/`. `python -I -B -m nanobot` serves the Dot; `--version` -answers and any other argument is refused. The process reads its environment -once, in `main.py`: `INVISIBLE_DOTS_AGENT_SOCKET`, -`INVISIBLE_DOTS_AGENTD_SOCKET`, `INVISIBLE_DOTS_AGENTD_BIN`, -`INVISIBLE_DOTS_WORKSPACE`, `INVISIBLE_DOTS_ENGINE_STATE`, -`INVISIBLE_DOTS_NETWORK_CHECK` (the host the network check reaches), -`INVISIBLE_DOTS_MCP_COMMAND` (the browser server's command; the unit names it by its path in dot's `~/.local/bin`), -`INVISIBLE_DOTS_MODEL_USER` (the user of the model's commands, whom the API -socket refuses), and for tests and the smoke `INVISIBLE_DOTS_OPENROUTER_URL`. -Before it serves it checks that no other engine owns the state. - -- Work. One class, `TurnRunner` (`turns.py`), starts every model turn and - drives nanobot's `AgentRunner` (`nanobot/agent/runner.py`); the `Engine` - (`engine.py`) decides which turn runs and what its end means. The chat is the - session `chat`; a task is the session `task:`. Accepted - `user.message` and automation rows wait in `dots_inbound`; when no chat turn - runs, a chat turn starts with all of them, in the order accepted, as its - opening messages; while it runs, rows accepted since are injected into it - through the runner's injection callback and committed as user messages. The - answer a chat owes (section 8.7) is one rule: when no chat turn runs, no - accepted row waits, a row is in the transcript and no open approval holds - the chat, a chat turn runs with no opening message. A `task.created` is - queued in `dots_tasks` and run one at a time, priority first then arrival; - a chat turn and a task turn may overlap. `system.event task.cancelled` ends - the task, cancels its turn and closes its open calls; no task event follows - `cancelled`. No work starts before the host pushed both the config and the - key, nor while a prepare-sleep holds it (a `POST /secrets` or a new inbound - event lifts it). One asyncio loop runs everything. -- Where the state lives. One SQLite file, `/engine.sqlite` - (`/home/dotengine/state`, 0700), opened in WAL mode with - `synchronous=FULL` and the exclusive locking mode. It holds the Dot's tables - (`dots_outbox`, `dots_inbound`, `dots_tasks`, `dots_tool_intents`, - `dots_tool_decisions`, `dots_approvals`, `dots_spend`, `dots_browser_identities`, `dots_kv`) and the transcripts - (`sessions`, `messages`): SQLite makes a transaction atomic per file only, - and this is what lets an event commit with the transcript row it describes. - `DotStore` (`store.py`) is the one place a write transaction begins and - ends. -- Commit points, each one transaction with the outbox rows describing it: an - inbound event with its effect (a queued task, a recorded decision, a - cancel); a task's start with `task.started`; the opening of a turn (the - calls the previous unit left open are closed, then the opening messages are - appended); every message the runner adds, one at a time and before it goes - on (`AgentRunner._commit` hands it to the commit callback of `TurnRunner`): - the assistant message with its tool calls, each tool result as soon as its - call returns, the final answer, an injected user message; a gate decision - (a park, or a denial); a call's intent with `agent.state EXECUTING` (from - the turn hook, just before the tool runs); the end of a failed turn; the - agent state transitions. `DotStore.append_messages` calls - `record_transcript_append` (`transcript_outbox.py`) in the transaction that - stores a message: a user message that carries an inbound id marks that input - as in the transcript; the final assistant message of the chat emits - `message.assistant` and applies every input the transcript holds (the - `in_reply_to` is the newest `user.message`); the final assistant message of - a running task completes it with `task.completed`, the text as the summary; - any other assistant message of a running task that has tool calls and text - beside them emits `task.progress` (section 5.4); - a tool result emits `tool.called`, its duration measured from the call's - intent and its `target` read from it (section 5.4), which it removes. A turn that fails fails its task in its own - transaction; a chat turn that fails answers "I could not answer: ...". -- Closing open calls (`close_open_calls`, `gate.py`), at every start, at the - beginning of every turn and at the end of a cancelled or failed one. For - each call of the newest assistant message that has no result, one result, - by what the database holds: an intent (the call started) gives the - interrupted text of section 8.7 and `tool.called` with `interrupted: true, - ok: false, duration_ms: 0`, and the decision `ask` when it was the approved - call; a park decision gives the approval message again and no event; a deny - decision gives "The Dot's policy denied this call." and `tool.called` with - `deny`; a skip decision gives the skipped message and no event; nothing - gives "Not executed: the unit ended before this call ran." and no event. -- The model's computer. `AgentdComputer` (`computer.py`) is the one door to - the Dot's computer, and it speaks only to dot-agentd. Every command the model - runs goes through `ExecTool._spawn`, the one spawn of `exec` and of the - background jobs: a local child of the engine in its own session that runs - `dot-agentd relay -- /bin/bash -lc ` with `PATH` as its only - variable (the command's environment is its login shell's, as `dot`); the - relay lives exactly as long as the remote command, and killing it (a - timeout, a cancel, `terminate`, the engine's death) makes dot-agentd kill - the remote process group. A background job is an `exec` that outlived its - `yield_time_ms`; `exec_session` and `list_exec_sessions` act on it. With - `tty: true` (always a background job: a terminal program is interactive) the - relay gets `--tty` and `TERM` (the engine's, else `xterm-256color`) and - dot-agentd runs the command on an 80x24 pseudo-terminal, so what it writes is - one stream and its input is echoed back; `close_stdin` is ^D and the - character with code 3 is ^C. The model reads the text of the screen, not the - byte stream: `terminal_text` (`exec_session.py`) drops escape sequences, - turns `\r\n` into a newline, lets a lone `\r`, a backspace and erase-in-line - overwrite, and holds back an escape sequence cut by the end of one poll until - the next. A program that paints the whole screen (vim, htop) is not - rendered. There is no terminal for the person: a person's keystrokes are not - tool calls and would bypass the approval system (section 10). The file - tools read and write through the `GET /v1/files`, `PUT /v1/files` and - `GET /v1/files/list` routes of `agentd.sock` (relative paths resolve against - `/home/dot/workspace`), keeping nanobot's line windows, read-before-write - checks, fuzzy edit matching and diff summaries. `find_files` and `grep` run - `find`, `stat` and `grep` on the computer, as `dot`, by argv with no shell, - to find the files that can match, and read only those. Nothing of the model's - runs as `dotengine`, and what `dot` may touch is decided by the operating - system: the engine adds no path policy of its own. -- The config. `PUT /config` is validated (`DotRuntimeConfig`, the same checks as - the zod schema), stored in `dots_kv` and projected in process - (`projection.py`): the OpenRouter model id, the tools offered (those of the - table whose permission is `allow` or `ask`), `max_steps_per_task` as the - step limit, and the Dot's section of - the system prompt (its name and instructions, then nanobot's tool - contract, a short note on its computer, the memory notes, MEMORY.md, where the past conversations are, - and the day). - There is no config file, no installer and no sudo rule. The same config - again changes nothing, and the same key again builds no provider - (`provider.py`: a new provider only when the key, the model or the base URL - changes, and a running turn keeps the provider it began with), so the host's - pushes at every READY and `agent.started` disturb no running turn. -- `agent.state` follows the turn hook: a run starting is `THINKING`, a tool - running `EXECUTING`, the last run ending `DONE` then `IDLE` (or - `WAITING_APPROVAL` while an approval waits); every start of the process - records `IDLE` again. -- After a crash (`Engine.start`). The first event is `agent.started`. Every - call left with an intent and no result is closed as interrupted, once. An - approval still `running` is over (its call is one of those interrupted, with - `decision: "ask"`); a turn that was telling a session a decision (`granted`, - `told`) is told again; the approved call still runs at most once. A task left - running is resumed with its description and the note "[The previous attempt - at this task was interrupted by a restart. ...]", unless an approval of its - session is open (then the decision moves it), and fails once it has been - started three times; a run a prepare-sleep abandoned does not count. A chat - turn whose answer is owed runs by the rule above. -- Policy (`gate.py`). Every tool call is decided at one boundary, - `_admit_tool_call` in `nanobot/agent/tools/execution.py`, after the call was - validated and its arguments cast and before anything runs; the gate is a - required argument of the runner (`AgentRunSpec.gate`), the contract it - answers is in `nanobot/agent/tools/gate_types.py`, and the calls of a - response run one at a time, each decided before it runs, so a park stops - the calls after it; `ToolRegistry` has no way to run a tool, so no call - reaches one another way. The decision - is made from the map the host pushed (section 7), read at the moment of the - call, so a push between two calls of one turn applies to the second: a tool - with no permission, or a permission missing from the map, is denied, and - with no config or no database every call is (the gate fails closed). `deny` - returns the reason to the model as the call's result, with no hint to try - another way; `tool.called` reports the call with `decision: "deny"` and - `ok: false` (the decision of a call inside a turn is kept in - `dots_tool_decisions` until its result is written). A provider's tool call id - names a call only inside its own response (models behind OpenRouter number - their calls from zero in every response), so the intent, the decision and the - approval of a call are keyed by the session and the id, and an approval is made - per ask: the same call asked again while its approval is pending (same session, - id, tool and arguments) is the one approval, a later call that only shares the - id is a new one. -- Tools offered. One table (`nanobot/dots/permissions.py`, section 8.3) lists - the tools a Dot may use, the permission each exercises and how to build it; - the registry holds exactly those, and each turn works on a view of it that - holds the offered ones, so a denied tool is not even seen, and the gate - decides every call by the same table. The table also says which arguments - of a call an `approval.requested` may carry (all of them, but for the proxy - of `browser_identity_create`). - The browser's server, `invisible-playwright-mcp`, runs only through the - `BrowserManager`, on registries no turn sees. The servers the person - declares (`mcp_servers`) are `McpServers`' (`nanobot/dots/mcp_servers.py`, - section 8.3): it registers each connected server's tools on the Dot's - registry as `McpServerTool`s, which call the client's wrapper of the - server's own registry at call time, and the permission table maps a tool - named `mcp__` to `mcp.` (`tool_permission`). The - projection keeps the servers whose permission is not `deny` - (`EngineSettings.mcp_servers`); a turn first awaits `McpServers.ready()`, - then offers the table's tools and those servers' tools, and the prompt - carries what each said at `initialize` or why it is not connected. The fork's - MCP client keeps a server's instructions, says why a connection failed, and - takes a stdio server's standard error (`UPSTREAM.md`). -- Browser identities. `BrowserManager` (`nanobot/dots/browser.py`) owns the - identities (their rows in `dots_browser_identities`, their directories under - `/home/dot/browsers` and their servers' homes under `/var/lib/invisible-dots/mcp`, - made and removed as dot through the Computer) and one - `invisible-playwright-mcp` process per open identity, started as dot through - `dot-agentd relay` with the environment of section 6 and nanobot's MCP client - on a registry of its own. Launch, close and delete run one at a time; calls on - one identity run one at a time. At `max_open` a launch closes the least - recently used identity first. A - browser action on an identity that is not open fails with `not_open` and - never launches it. A browser that the server reports gone while its process - lives (Firefox crashed, its window was closed) is not reopened and the call is - not repeated, as the library itself took out on purpose: a repeated click lands - on the blank page of a restarted browser, and the model is never told. It is - a crash like a process that ended (the client reports that instead of - reconnecting, because a restarted process has lost its browser): the identity - is closed and its server stopped, `browser.identity.closed` is emitted once and - the call fails with `crashed`, whose message says the browser is gone and that - `browser_identity_launch` opens it again as the same person, on a blank page - (the model cannot call `browser_open`; the launch is its way, and the one - decision of whether a browser starts). A frame that finds the browser gone - closes the identity the same way. "Gone" is the library's own sentence (its - `NOT_OPEN` and `GONE`, which `server_capture.py` reads from the installed - package's constants) as the - whole error of the call, not a phrase inside one: a failed click or select - echoes text the page controls, and a page that writes the sentence closes nothing. An open is known the same way: - `browser_open`'s answer starts with the library's `OPENED`, recorded beside the two. The manager hears of the end - when it happens, from the client's transport, so a process that dies while idle - is closed at once, frees its slot of `max_open`, and the next action says - `not_open`; a file never claims an open browser for a process that is gone. What a page - tool returns reaches the model as the server wrote it. A close calls - `browser_close` first, so Firefox flushes its profile, then ends the process. - Every `browser.identity.*` event commits with the row change it describes. - The model's identity and page tools (`browser_tools.py`) and the routes of - section 5.3 are the callers; `max_open` and `max_identities` (3 and 20) are - set when the manager is made and nothing changes them. A page tool is the - MCP server's tool of the same name (`SERVER_TOOLS`), called with the - arguments the model gave it. -- Images. A tool's image is shown to the model and kept out of the stored - transcript (`images.py`): the tool message holds its text and - `[screenshot, WxH, not stored]`, the images of the turn go to a per-turn - buffer that keeps the newest three, and the runner asks that buffer for the - messages of each model request, which adds them in one user message after - the last tool message. That message is made for the request and never stored. - nanobot's MCP client returns the image blocks of a server only when it is - made for them (the browser's is), and every other caller still gets text. -- Approvals. `ask` inside a turn stores the call with its full arguments in - `dots_approvals` (`pending`), emits `approval.requested` and records the - decision `park`, in one transaction; the model gets a result saying the call - waits for the user and has not run, the later calls of its response are - recorded as `skipped` (they run nothing, report nothing and say so), and the - runner ends the turn (`stop_reason` `parked`: no further model request). - Nothing waits in memory. A retried turn reuses the approval of the same - tool call id and asks nothing new. A task whose run parked a call is neither - failed nor resumed while an approval of its session is open, and - `agent.state` ends on `WAITING_APPROVAL`; `GET /state` names the oldest - pending approval. `approval.received` is applied in the transaction that - accepts it (`approved` or `rejected`, with the note); a decision for an - unknown or already decided approval is logged and ignored. The engine then - tells the session that made the call, in a turn of its own (the chat's - included): an approved call is to be made again with exactly its arguments - (written as JSON with no spaces), a rejected one did not run, with the note. - The approval is `granted` (or `told`) before that turn starts and `done` - when it ends. The one call an `ask` lets through is the approved one made - again in a turn of the same session: same tool, the same arguments (compared - with keys sorted), once; it moves the approval to `running`, runs like any - call of the turn, and its `tool.called` says `ask` and ends the approval. - Different arguments are asked about anew. -- Automations. nanobot's `CronService` runs inside the engine, its jobs in - `/cron/jobs.json`, and the `cron` tool (permission `automations`, - `ask` by default) adds and removes them. A firing is recorded as a durable - inbound row, `automation.fired`, once per firing, and the chat answers it as - an input: the opening message reads `[Automation "" fired] ` - and the answer is a `message.assistant` without `in_reply_to`. The jobs are - the Dot's own: it lists, adds and removes them with the tool, and the person - asks it to in the chat; the host has no route that lists or changes them, and - hears only when the earliest is next due (`automation.next_run`). `GET /tools` - shows the permission table with what the model is offered now. -- Automations while the computer is off. The jobs live in the guest and the - computer powers off when idle, so a job does not fire while it is off: the - host starts the computer shortly before the earliest run (section 9.5), told - when by `automation.next_run` (section 5.4), which the engine sends whenever - the cron service arms its timer and the earliest next run changed. A run that - came due while no engine ran is made once, when the engine starts. The next - run of a job is stored in `jobs.json` and is what the service starts from: a - time in the past is kept (the service counts a next run from now only for an - enabled job that has none), the first tick finds the job due, runs it once - and counts its next run from that moment. A one-time job (`at`) therefore runs - late and is then over, and a recurring one runs once for the occurrences it - missed, never once for each of them. A firing is recorded under the id - `cron::` before the service saves that the job ran, - and the engine ignores an id it already holds, so a `kill -9` between the two - finds the job due again and the engine recognizes its firing: it runs once. - On a stop the cron timer ends before the engine does, so no firing falls on a - stopped engine, which records nothing while the job moves on as if it had run. -- Prepare-sleep and SIGTERM. The engine stops taking new work; a turn with no - tool running is cancelled at once; a turn with a tool running gets up to 20 - seconds, in which the tool's result commits and the next iteration abandons - the turn, and is cancelled at the deadline (its intent stays, and the next - start reports it interrupted); the attempt of a cut task is given back; the - open browsers are closed (up to 4 seconds in all on SIGTERM and up to 30 on - prepare-sleep, with no turn left to call one; a close that outlasts the wait - keeps running as its own task, and a browser still open when the process - exits is ended with it); the - WAL is checkpointed. The API (`server.py`) answers the host's `prepare-sleep` - to its end even when the host hangs up. -- Removed from nanobot (everything since the import is in - `invisible_engine_dots/UPSTREAM.md`): the app shells, the channels, the bus - and the commands, the agent loop and its hooks, the other approval and guard - mechanisms (the workspace path policy, the command guard, the sandbox, the - SSRF guard), subagents, nanobot's skill machinery (the Dot's skills are its - own, section 8.6), the web tools, image and document reading, the usage - telemetry, the configuration files, every provider but OpenRouter, and the - options nothing of the Dot sets (reasoning effort, concurrent tool batches, - the spill of a long result to a file, the event sink, the MCP reconnect). - -## 9. Control plane - -### 9.1 Database (PostgreSQL) - -PostgreSQL is the database on every host. By default it is PGlite (PostgreSQL -compiled to WebAssembly, `@electric-sql/pglite`) running inside the server -process with its data in `~/.invisible-dots/db`, so nothing has to be -installed. With `DATABASE_URL` set, the same migrations and the same queries -run against an external PostgreSQL 16 or newer through `pg`. The repositories -talk to one small interface (`query` for one statement, `exec` for a script -of several such as a migration file, `transaction`, `close`, and `kind`) with -two adapters; no SQL differs between them, and the test suite runs against -both. int8 comes back as a number (a value above 2^53 is an error, never -rounded) and bytea as `Uint8Array`, on both. - -PGlite has one connection and does not lock its data directory, so one server -runs per `INVISIBLE_DOTS_HOME` (`server.lock`, section 3.2), transactions run -one at a time, and nothing else opens the database: `invisible-dots doctor` -asks the running server instead. Each migration file runs in its own -transaction, which first takes a transaction-scoped advisory lock, so two -servers migrating one external database apply every file once. - -One database serves one control plane. `server.lock` only guards one -`INVISIBLE_DOTS_HOME`, and two servers with different homes on one external -database would each reconcile, dispatch and stop the other's Dots (one finds -no disk for the other's VMs and marks them ERROR). So the server also holds a -session-level advisory lock on its database for its whole life -(`Database.holdInstanceLock()`) and refuses to start without it; the lock goes -with the session, so a server that died leaves nothing to clean up. - -Event ids are visible in id order: every insert into `events` takes a -transaction-scoped advisory lock before its `bigserial` id is drawn, held -until its transaction commits. Without it, on PostgreSQL a transaction that -drew id 10 could commit after one that drew 11, and a client that resumed -`GET /api/stream` after 11 would never see 10. A transaction that inserts an -event and changes other rows inserts the event first, so the lock is never -taken while holding a row lock another event writer waits for. - -A host event is stored in the transaction of the change it tells of, and -published after COMMIT: the Dot or its computer changing state (`dot.created` -with its rows, `computer.state` and `computer.started` with the process and the -images, `computer.stopped` with the stopped state and the Dot's status, -`dot.deleted` with the deletion, `dot.updated` with the saved config, ERROR with -`dot.updated`), a task (`task.created`, `task.cancelled`, and the `task.failed` of a task -whose delivery the host gives up), an answer (`approval.resolved`, and -the `dot.updated` of "always allow"), and what a channel reports or the person -does to it (`channel.status`, `channel.changed`, `channel.peer.paired`). A -control plane killed between two writes therefore leaves the state it was in -before, which recovery knows how to finish (a computer found off that was not -recorded as stopped is recorded now, with its `computer.stopped`), and never a -state nobody was told of: the web's views follow the events, so a state without -its event would stay wrong in them for good. `commit-points.test.ts` of the -scheduler and of the channel hub kill the process at the last write of a stop, a -deletion, a creation, a save, a status, a pause, a removal and a pairing, and -check that neither the state nor the event is there. - -Tables: `dots`, `computers`, `tasks`, `task_runs`, `events`, `approvals`, -`inbound_events`, `secrets`, `channel_bindings`, `channel_peers`, -`channel_pairings`, `channel_prompts`, `schema_migrations`. Migrations are plain -SQL files applied in order at start. - -- `dots(id text pk, name text unique, config jsonb, status text, error text null, config_version int default 1, created_at, updated_at)`: `config_version` grows with every save of `config` (`updateConfig` and `setPermission`) and with nothing else, while `updated_at` also moves with every status change; the PATCH precondition is on the version -- `computers(dot_id pk fk, vm_name, guest_port int null, pid int null, state text, golden_image text, runtime_image text, token_enc bytea, event_cursor bigint default 0, last_active_at, next_automation_at timestamptz null, stop_reason text null, last_error, updated_at)`: `guest_port` and `pid` are null while no QEMU runs, and `guest_port` is not unique (a crashed VM's row may name a port since reused); both are copies of `qemu.json` (section 3.2), which wins on reconciliation; `next_automation_at` is the guest's last `automation.next_run` report (migration `0008_computer_next_automation`), null while none is due or nothing was reported, and it is the guest's report: nothing else of the row moves with it; `stop_reason` (`idle`, `user` or `exited`, same migration) is why the computer is STOPPING or STOPPED and is null in every other state (section 9.5) -- `tasks(id text pk, dot_id fk, description, priority int, status, created_at, scheduled_at, started_at, finished_at, summary, error, spent_usd double precision default 0)`: `spent_usd` is the highest `spent_usd` the guest reported on the task's events (section 5.4), recorded by the host in the transaction that stores each event, so a late or repeated event never lowers it and a cancelled task the guest keeps working on still counts; a task whose guest never reported spend stays 0 -- `task_runs(id pk, task_id fk, started_at, delivered_at, finished_at, outcome)`: `delivered_at` is set when the guest accepted the run's `task.created` -- `events(id bigserial pk, dot_id, type, data jsonb, source 'host'|'guest', guest_seq bigint, created_at)`, unique `(dot_id, guest_seq)`, and unique `(dot_id, data->origin->>binding_id, data->origin->>external_id)` for a `user.message` with an origin: a channel message is stored once, by the channel's own id, in the transaction that stores it, so a redelivery after any failure finds the first one and the Dot gets it once (section 9.8) -- `approvals(id text pk, dot_id, task_id, tool, permission, arguments jsonb, reason, status 'pending'|'approved'|'rejected'|'expired', note, created_at, resolved_at)`: an approval whose task reached a terminal state before anyone decided is `expired`, in the same statement that ends the task, and an `approval.requested` for a task that is already terminal is stored as `expired`, never `pending` -- `inbound_events(seq bigserial pk, id text unique, dot_id fk, type, data jsonb, ts, task_id, run_id, created_at, sent_at, delivered_at, dropped_at, drop_reason, failures int, last_error, retry_at)`: the outbox of host to guest events (section 9.2) -- `secrets(scope text, name text, value_enc bytea, updated_at, pk(scope, name))`: `scope` is `global` or a dot id; no foreign key can cover that, so deleting a Dot deletes the secrets scoped to it in the same statement (`DotsRepository.delete`) - -- `channel_bindings(id text pk, dot_id fk cascade, kind 'telegram'|'whatsapp', enabled bool, settings jsonb, status, status_detail, account, event_cursor bigint, created_at)`, unique `(dot_id, kind)`: one Dot's link to one channel kind (section 9.8), and unique `account` among the `telegram` bindings whose account is known: one bot serves one Dot (a WhatsApp number is learned at the scan and may be linked on several Dots, each a device of the phone); `settings` is `{approvals, notify_tasks, show_arguments}` and never a credential; `account` is the channel's public name for the account (a bot's username); `event_cursor` is the id of the last event of the Dot the hub dealt with -- `channel_peers(binding_id fk cascade, peer_id, chat_id, role 'owner'|'user', label, created_at, pk(binding_id, peer_id))`: the people allowed to talk through the binding, by the channel's stable id, with the chat they paired from -- `channel_pairings(binding_id fk cascade, code_hash, expires_at, consumed_at, pk(binding_id, code_hash))`: one-time pairing codes, stored hashed -- Where a channel message came from is in its `user.message` event (section 5.4), the one owner of that fact; there is no table of handled messages. An index on `events` by `(dot_id, data->>'message_id')` for `user.message` lets an answer find the message it answers -- `channel_prompts(binding_id fk cascade, approval_id fk approvals cascade, chat_id, ref, created_at, pk(binding_id, approval_id, chat_id))`: the approval prompts a channel sent, one message per chat; `ref` is the channel's handle for the message (a Telegram message id), what an edit needs. A row is deleted once its prompt was edited to the outcome - -Secrets are encrypted with AES-256-GCM under `master.key`. The OpenRouter key -is looked up as `(, openrouter_api_key)` first, then -`(global, openrouter_api_key)`. - -### 9.2 Durable queue - -The dispatcher claims work with -`SELECT ... FROM tasks WHERE status = 'PENDING' AND (scheduled_at IS NULL OR scheduled_at <= now()) ... FOR UPDATE SKIP LOCKED`, -one task per Dot at a time (a Dot with a `RUNNING` or `WAITING_APPROVAL` task -is skipped). One dispatcher runs per database (section 9.1); the row locks -make a claim and the rest of its transaction one unit, and are not what would -keep two dispatchers from claiming two tasks of one Dot. - -Everything the control plane tells a guest goes through one durable outbox, -`inbound_events`, written in the same transaction that decided it: a claim -stores its task's `task.created` (and opens a run), a resolved approval its -`approval.received`, a message its `user.message`, a cancel its -`system.event`. Nothing is kept only in memory, so a failed wake or a control -plane restart loses nothing a person saw accepted. One deliverer -(`apps/scheduler/src/inbound.ts`) sends a Dot's undelivered rows in order, -waking the Dot first when it sleeps, and stops at the first failure so a later -event never overtakes an earlier one: - -- A row is delivered when the guest answers 202. Every READY transition - sends the Dot's rows again, and a periodic pass retries rows whose retry - time came. -- Whether a `task.created` may still go out is decided in the statement that - marks its send begun: a task that stopped being active meanwhile (a cancel - that won the race with a wake) is dropped there and never sent. A cancel of - a task whose send never began drops that `task.created` in its own - transaction; otherwise the guest may hold the task, so the cancel stores a - `system.event` behind it. -- A send whose outcome is unknown (a timeout, a reset connection) may have - reached the guest: it is sent again with the same id, which the guest - ignores if it has it, and it never counts as a failure. Only failures - with a known outcome count: the guest refused the event (it is dropped), - or nothing listened. After 3 such failures, or one refusal, a `task.created` - gives its task up as FAILED, logged as `task.failed` in the same transaction - (a task cancelled meanwhile stays cancelled); a message or a decision is never - given up and waits for the next READY. - -### 9.3 Lifecycles - -VM states: `PROVISIONING, STARTING, RUNNING, STOPPING, STOPPED, ERROR, DELETING`. - -Dot states: `CREATING, READY, IDLE, RUNNING, WAITING_APPROVAL, ERROR`. A Dot whose -computer sleeps is IDLE and its computer STOPPED: there is no idle state of a VM. - -Task states: `PENDING, RUNNING, WAITING_APPROVAL, COMPLETED, FAILED, CANCELLED`. - -A Dot is READY when all of these hold: -the QEMU process is running, `GET /v1/health` answers through the port forward with -`agentd: "ok"` and the agent `status: "ok"`, the OpenRouter key has been pushed -(`openrouter_configured: true`), and the guest's own checks pass (filesystem -writable, network reachable, browser layer installed) as reported in the -health answer. - -The READY procedure runs under the Dot's lock, like every operation that -changes a computer's state, and is the one place that starts a Dot's event -pump; it first stops a pump of an earlier start, which would read a guest -port that is gone. The key and the config are read from the database when -they are pushed. A key or a config stored while the procedure runs bumps the -Dot's generation, and the procedure pushes again until it completes on an -unchanged generation, so a tightened permission or a rotated key is never -lost in that window; pushes to a READY guest run one at a time per Dot, so the -last one sent is the newest. Every stop, reboot and delete takes the Dot out -of READY before anything else, so a delivery that skipped the lock because -the Dot was READY can only meet a stop that has not begun. - -### 9.4 Create - -`POST /api/dots` -> insert `dots` (CREATING) -> generate a token -> -`qemu-img create -f qcow2 -F qcow2 -b disk.qcow2 ` -> write -`seed.iso` -> pick a guest port -> spawn QEMU and write `qemu.json` (section -3.4) -> wait for the forward -> wait for health -> push the secret -> `PUT -/config` -> open the event stream -> READY. - -### 9.5 Sleep and wake - -When a Dot has no PENDING task that is due, no RUNNING or WAITING_APPROVAL -task, nothing undelivered in its outbox and no automation due within the wake -lead time (below), its agent state is IDLE, and -nothing happened for `idle_timeout`: `POST /v1/agent/prepare-sleep` -> -`POST /v1/system/poweroff` -> QEMU exits (killed after 60 s, section 3.4) -> -STOPPED. The idle stop checks all of it again under the Dot's lock, after -taking the Dot out of READY, and is called off when work arrived meanwhile. -Disk, identities and memory stay. A new task or message for a STOPPED Dot -starts the VM, waits for READY and then delivers it; so does a scheduled task -when its `scheduled_at` comes. A STOPPED Dot whose due work waits behind a task -its guest has not finished (it was stopped mid-task) is started too, so its -guest finishes that task and the next one can be claimed. - -A Dot's automations (section 8.8) run in its guest, which is off while the -Dot sleeps, so the host starts the computer for them. The guest reports, with -`automation.next_run` (section 5.4), when its earliest enabled automation is due, -and the host keeps it in `computers.next_automation_at`. The scheduler's pass -(every `dispatchIntervalMs`, 5 s) starts every STOPPED computer that the person did not stop (below) and whose Dot is not -in ERROR or CREATING and whose next automation is due within the -wake lead time (`lifecycle.automationWakeLeadMs`, 90 s by default: a start -reaches READY with some to spare) or is already past, a run missed while the -host or the computer was down, which the guest makes as it starts. The same -condition is part of "has work" for the idle sleep, in the idle check and -again under the Dot's lock, so a computer is not put to sleep when a run is -due within the lead time, and with an automation every minute it stays up. -The start is the ordinary one (the key and the config are pushed), and the run -itself is the guest's: the host decides only when the computer starts. - -An explicit stop by the person wins over the automations. Why a computer is off -is recorded with it (`computers.stop_reason`: `idle` for the sleep above, -`user` for the person's stop through `POST /computer/stop`, the CLI or the -web, `exited` for a VM that stopped by itself), kept while the computer is -STOPPING or STOPPED and cleared by any start. A computer the person stopped is -not started for its automations, missed or due, until the person starts it -again: they are paused while it is stopped, and the CLI (`status`, `computer -stop`) and the web (the stop confirmation, and the note below) say so. A message or a task for it -still starts it, as it does for any stopped Dot, and from that start on it -sleeps and wakes for its automations like any other. The person's stop of a -computer that is already asleep is recorded too. A stop that the control plane -was interrupted in is finished for the reason it was asked for, without the idle -check again: the shutdown had begun, and the guest may already have taken -prepare-sleep. Work that came due meanwhile starts the computer again from the -next pass, as for any stopped Dot. - -"Has work" is one definition, `Lifecycle.keepsAwake`: a task or inbound work for -the Dot, or an automation due within the lead time (a past one included). The -idle check, the idle stop under the lock and the restart after an unexpected -exit all read it. - -A VM that stops -without being asked (the guest powered itself off, QEMU crashed) is recorded -as STOPPED, and started again at once when `keepsAwake` says the Dot still has -work. - -`next_automation_at` arrives only from a guest that ran. A Dot that was asleep -when migration `0008` was applied has none stored, so it is not woken for its -automations (and misses their runs) until it starts for another reason: a -message, a task, or the person. Its guest then reports its next run at that -start, and the automations are woken for from there on. The host does not boot -every sleeping Dot once at an upgrade to ask. - -### 9.6 API - -All routes require `Authorization: Bearer ` (from -`~/.invisible-dots/config/api.token`, created at first start). The server -binds `127.0.0.1:8787` by default (`INVISIBLE_DOTS_LISTEN`). Every reader of -the token (the server, the command, the web server) reads it through one -function, `readApiToken()` in `packages/shared/src/api-token.ts`: -`INVISIBLE_DOTS_TOKEN` when set, otherwise the first line of `api.token`, -trimmed, and at least 16 characters. - -```text -POST /api/dots body: { config: | } -GET /api/dots -GET /api/dots/:id -PATCH /api/dots/:id body: { config, expected_config_version? } (pushed to the guest if running; with `expected_config_version`, the `config_version` of the Dot as read, a Dot whose config changed since is a 409 `dot_changed` and nothing is saved) -DELETE /api/dots/:id destroys the VM and its disk, then deletes the Dot, its rows and its own secrets - -POST /api/dots/:id/messages body: { text } -GET /api/dots/:id/messages ?limit=&order=asc|desc&before= conversation, from the event log (a user message carries `origin` when it came through a channel); oldest first by default, at most 500 a page; `order=desc` is the newest first and `before` (the `event_id` of the oldest message of the previous page, desc only) goes on, older, from there -POST /api/dots/:id/tasks body: { description, priority?, scheduled_at? } -GET /api/dots/:id/tasks ?limit=&before= the newest created first, at most 200 a page; `before` (the id of the last task of the previous page) goes on, older -GET /api/tasks ?status=COMPLETED|FAILED|CANCELLED&finished_after= { tasks }: the tasks of every Dot that ended in that state after that time, the latest end first, at most 200 -GET /api/tasks/:id -POST /api/tasks/:id/cancel - -GET /api/dots/:id/computer -POST /api/dots/:id/computer/start -POST /api/dots/:id/computer/stop -POST /api/dots/:id/computer/reboot -GET /api/dots/:id/computer/screenshot - -GET /api/dots/:id/browser-identities -POST /api/dots/:id/browser-identities -GET /api/dots/:id/browser-identities/:identityId -DELETE /api/dots/:id/browser-identities/:identityId -GET /api/dots/:id/browser-identities/:identityId/frame image/jpeg, only while the identity is open (409 not_open, 503 busy, 502 frame_failed or crashed: the engine's own answers pass through) -POST /api/dots/:id/browser-identities/:identityId/close 204; the browser ends, the profile stays - -GET /api/dots/:id/channels { channels: [{ kind, enabled, status, status_detail, account, settings, peers, created_at }], available: [kind] }; never a token; `available` is what this server runs (WhatsApp only when it was started with it, section 9.8) -GET /api/channels { channels: [{ dot_id, kind, enabled, status, ... }] }: every channel of every Dot, each with its Dot's id; never a token -PUT /api/dots/:id/channels/telegram body: { token } links the Dot to the bot (201), or gives the linked bot a new token (200); the token is checked with Telegram, stored encrypted, never returned -PATCH /api/dots/:id/channels/:kind body: { settings?: { approvals?, notify_tasks?, show_arguments? }, enabled? } enabled false pauses the channel, its people and token stay -POST /api/dots/:id/channels/whatsapp/link 202 the channel's record, waiting: starts linking WhatsApp (400 when the server does not run it, 409 `already_linked`) -GET /api/dots/:id/channels/whatsapp/qr server-sent events of `ChannelLinkFrame` (`waiting`, `code`, then `linked` or `failed`, and the stream ends); never cached, never stored -DELETE /api/dots/:id/channels/:kind unlink: the channel stops, its token (WhatsApp: the linked device's keys) and its people are deleted -POST /api/dots/:id/channels/:kind/pairing 201 { code, deep_link, message, expires_at }: a one-time code, valid ten minutes; `message` is what to send the account to pair -DELETE /api/dots/:id/channels/:kind/peers/:peer revoke a paired person - -GET /api/approvals ?status=&dot_id=&limit=&order=asc|desc&before= `dot_id` keeps the approvals of one Dot, in the database; `status` is one or several of pending|approved|rejected|expired; oldest first by default, `order=desc` is the newest first by the time of the last change (the answer, for an answered one) and `before` (the id of the last row of the previous page, desc only) goes on from there -POST /api/approvals/:id/approve body: { note?, always?: true } `always` also sets the approval's permission to `allow` in the Dot's config in the same transaction (then pushed like a PATCH, `dot.updated` logged); `approval.resolved` carries `always: true` -POST /api/approvals/:id/reject body: { note? } - -GET /api/dots/:id/events ?after=&before=&limit=&types=&tools=&task_id=&order=asc|desc `types` are event type names (an unknown one is a 400), `tools` narrows `tool.called` to those tools (`data.tool`) and leaves other types alone, `task_id` keeps the events whose `data.task_id` it is, `order=desc` is the newest first so that a `limit` keeps the newest, and `before` (the id of the oldest event of the previous page, desc only) goes on, older, from there -GET /api/dots/:id/files/list ?path= { path, entries: [{ name, type, size, mtime }] }: a directory under /home/dot (home when omitted) -GET /api/dots/:id/files ?path= the bytes of a file under /home/dot, at most 16 MiB (413 `file_too_large`) -GET /api/dots/:id/tools { tools: [{ name, permission, offered, description }], mcp_servers: [{ name, state, error, tools }] }: the engine's tool table and what the model is offered now, and where each declared MCP server is; needs the computer running -GET /api/dots/:id/mcp-secrets { dot_id, secrets: [{ server, name, set }] }: every secret the config's MCP servers name, and whether it is set; never a value -PUT /api/dots/:id/mcp-secrets/:server/:name body: { value } sets it (404 for a secret the config does not name, 400 for a value that breaks MCP_SECRET_RULE) and pushes the secrets to a running guest, which starts the server again with it -DELETE /api/dots/:id/mcp-secrets/:server/:name clears it -GET /api/dots/:id/skills { skills: [{ name, description, source, path, content }] }: the Dot's skills (section 8.6); needs the computer running -GET /api/dots/:id/usage ?since= { dot_id, since, spent_usd } -GET /api/stream SSE: every event, ?dot_id= to filter -PUT /api/secrets/openrouter body: { value, dot_id? } -GET /api/health -GET /api/doctor { ok, checks: [{ id, label, status: ok|missing|failed, detail, fix? }] }: the host report of section 11.1, run on the machine the server runs on -``` - -`GET /api/health` answers `{ status: "ok", database: "ok", version, -openrouter_configured, database_kind, data_dir, logs_dir }`; `openrouter_configured` -is whether a global OpenRouter key is stored, which `invisible-dots doctor` -reports, and the last three say where the state lives (`pglite` or `pg`, -`INVISIBLE_DOTS_HOME` as the server resolved it, and its `logs/`), so the web -client's host settings page can tell the person without their knowing the -environment variable. - -`GET /api/dots/:id/events` returns at most `MAX_EVENT_PAGE` (1000, in -`packages/shared`) events a call: the store clamps to it, the route refuses a -larger `limit` with a 400, and a client that pages through the log asks for -exactly that many, so a shorter page is the last one. - -`GET /api/doctor` answers the report of section 11.1 as the server's own host -sees it, the same rows in the same order as `invisible-dots doctor --json` -(`{ ok, checks }`, `ok` true only when every check is). It runs QEMU's -accelerator probe, so it takes a moment and it is not polled. - -`GET /api/dots/:id/usage` answers the model spend the Dot's guest reported, in -USD, since `since` (the first event when omitted; a malformed `since` is a 400). -The event log is its one source: it sums `spent_usd` over the events that end a -unit of spend, a task's `task.completed` and `task.failed` (each carries the -whole task) and the chat's `message.assistant` (each carries what the chat spent -since its last answer), by the -time the host stored them. `task.progress` is left out because its value is the -running total of a task that ends with one of those events. Work that never -reported an end (a cancelled task) is not in the -total; the task's own `spent_usd` still shows what was heard of it. Like -`/events`, it reads the history of a deleted Dot by id. - -`GET /api/dots/:id/events` filters in the database: `types` is a comma-separated -list of type names, `tools` a comma-separated list of tool names that narrows the -`tool.called` events (the other types pass), `task_id` matches `data->>'task_id'` (the one place the -contract puts the task, so the host's `task.created` and `task.cancelled` and -the guest's `task.*`, `tool.called` and `approval.requested` of a task, and the -host's `approval.resolved` of one, all match), and all combine with `after` and `limit`; `order=desc` reads from the newest -event back (the limit then counts the newest that the filters keep, and the page is -newest first), and `before` (an event id, desc only) goes on from there: the events -older than it, which is how the Activity page pages through a long log. A chat turn's events carry -no task. Migration `0006_events_task.sql` adds the expression index the task -filter reads. A type name no event has (`tool.calls`) is a 400, so a typo does not -look like a quiet Dot. - -`GET /api/dots/:id/tools` and `GET /api/dots/:id/skills` pass through to the engine's -routes of section 5.3: the tool table and the skills are the engine's, and the control -plane keeps no copy of either. They need the computer running (`409 computer_stopped`), -and the engine's own refusals pass through with their code and status. - -`GET /api/dots/:id/files/list` and `GET /api/dots/:id/files` read the Dot's -computer through dot-agentd's `GET /v1/files/list` and `GET /v1/files`, and -only under `/home/dot`: `path` is absolute, relative to `/home/dot` or `~`, and -the control plane normalizes it (`checkHomePath` in `packages/shared`) so the -guest always gets an absolute one; a path outside home, or with any `..` segment, -is a `400 invalid_path` without a call to the guest. That check is lexical and -only the early answer; the rule is dot-agentd's: its TCP listener (the host's -door) resolves every path to its real location with the symbolic links followed -and refuses one that is not under home, `403 outside_home`, which passes through -(a link under home to `/proc//environ`, where the browser server runs as `dot` -with the proxy password in its environment, shows nothing; the session file the -server saves the proxy in is not under home at all, section 4.2). The same goes for the -`PUT` and the listing of that listener; a link that stays in home works, one that -leaves it lists as `other`. The engine's socket is not confined. A read is one buffered answer of at most 16 MiB; a larger -file is a `413 file_too_large` and the host stops reading it as soon as it passes -the limit. The bytes are the Dot's own (a model wrote them, perhaps after reading -hostile text) and the web server answers from the page's origin, so the type is -never one a browser runs: images are served as `image/png`, `image/jpeg`, -`image/gif` or `image/webp` inline, source and markup (`.md`, `.json`, `.html`, -`.svg`, ...) as `text/plain` inline, everything else as a download, always with -`nosniff`, `Content-Security-Policy: default-src 'none'; sandbox` and `no-store` -(the web proxy passes those headers on). The guest's own refusals pass through -with their code (`404 not_found`, `400 is_a_directory`, `400 not_a_directory`). - -Browser identity and file routes need the Dot's computer running: on a stopped -Dot they answer `409 { error: "computer_stopped" }`. - -Errors are `{ error: , message }` with a 4xx or 5xx status. - -### 9.7 Web client - -`apps/web` is a Next.js server on `127.0.0.1:3000`. `invisible-dots server` -starts it: the build `npm run build --workspace @invisible-dots/web` leaves -(Next's standalone server and the browser files beside it, assembled by -`apps/web/scripts/standalone.mjs`) is run with `node` as a child process of the -server, on `INVISIBLE_DOTS_WEB_LISTEN` (default `127.0.0.2:3000`), with -`INVISIBLE_DOTS_URL` set to the control plane's address; the setting is read -before anything starts, so a malformed value or port 0 (the person has to be -told where to go) fails the command at once. The child gets an -allowlisted environment, not the server's: the data directory (so it reads -`api.token` itself on every request), `INVISIBLE_DOTS_TOKEN` only when the -server was given the token that way, and `INVISIBLE_DOTS_WEB_ALLOWED_HOSTS`. -It also gets the server's pid (`INVISIBLE_DOTS_WEB_PARENT_PID`) and exits when -that process is gone (`apps/web/src/instrumentation.ts`), so a `kill -9` of the -server does not leave an orphan holding the port: a stop signal ends the child -through the server, anything else through the child itself. -The web client is a companion, not a dependency: when it is not built, its -port is taken or it exits, the server logs why and the control plane and the -command line go on; `server --no-web` does not start it. A stop signal closes -the web client first, then the control plane, and a Ctrl+C while the web -client is still starting ends that start. It answers `/api/...` with -the control plane's own paths, so the browser uses the SDK unchanged, and -adds the API token on the way, so the browser never sees it. It holds that -token, and it has no login: it listens on `127.0.0.2:3000` -(`INVISIBLE_DOTS_WEB_LISTEN`), a loopback address no guest reaches. QEMU's user -network maps `10.0.2.2` to the host's `127.0.0.1` only (section 3.6), so a Dot's -VM reaches the API on `127.0.0.1:8787`, which needs the token, and never the web -client. The Host, Origin and `Sec-Fetch-Site` checks stay, as a defence against -DNS rebinding and cross-site pages: the Host must be a loopback name (any -`127.x.y.z`, `localhost`, `::1`) or listed in `INVISIBLE_DOTS_WEB_ALLOWED_HOSTS`. - -The pages are React with Tailwind CSS 4. `src/app/tokens.css` is the one place -that holds a color, a radius or a typeface (a light and a dark set, each pair -checked for WCAG AA contrast by a test), `globals.css` maps it into Tailwind, -and the primitives in `src/components/ui/` are shadcn/ui source, copied in -(`THIRD_PARTY_NOTICES.md`). Every page sits in one frame: the rail -(the Dots with a ring around each avatar that says what it is doing, the Inbox -with what needs the person, the state of the API and of the live stream, the theme) -and, for a Dot, its header and tab bar. One live stream serves all of -it; `lib/attention.ts` is the single owner of what needs the person, and the -rail badges, the avatar ring, the tab title and the favicon all read it. There -is no other stylesheet: every screen is utilities over the tokens. The browser tests (`apps/web/e2e`, Playwright) start the -real control plane in-process over the fake VM layer and the built web client -against it (`e2e/harness.ts`), so a test drives what a Dot's computer does. - -Home (`/`) is a card per Dot: its avatar ring and name, the state with the -recorded reason beside an ERROR, the model, what it spent today, its last -activity (the newest chat, task, tool, approval or browser event, one -`order=desc&limit=1` read and then the stream), the approvals that wait, Open chat and the computer's power menu. Each card's spend pill is -scoped to its own Dot, so a message of one Dot re-reads only that Dot's usage. -Search appears above six Dots, and with no Dot the page invites the first, and, -while this computer is not ready for a Dot, shows the setup checklist under the -invitation: each check of `GET /api/doctor` and `GET /api/health` that is not -ready, with the command that fixes it and a button that copies it (setup needs -an elevated terminal and an image build takes long, so they stay commands the -person runs), and the key field when no key is stored. The checks run again -when the person returns to the window while something is missing, and the -checklist stays, turned green, once the last thing is done. A ready host shows -none of it. - -Settings (`/settings`, in the rail) are the host's, not a Dot's: the same checks -(`useHostChecks` is the one list the checklist, this page and the create form's -preflight draw), the OpenRouter key (a write-only field over -`PUT /api/secrets/openrouter`; whether one is stored comes from the health -answer, which the shell shares and asks again after a save, so the rail agrees -at once; the answer says how many running Dots got it), the theme and About: -the version, the database kind, the data -directory and the logs directory, which `GET /api/health` reports for the -purpose (`database_kind`, `data_dir`, `logs_dir`), and `invisible-dots logs -` for a Dot's own logs. - -Create a Dot (`/new`) is a form in three steps (Identity, Brain, Computer and -safety) with sliders inside `CONFIG_BOUNDS`, the idle timeout, the Careful, -Balanced and Autonomous permission presets, the cost cap and the optional -summary model; or the same config as YAML, where the form refuses to take back -YAML that sets what it has no control for instead of dropping it. The shared -config schema is the only judge: `lib/dot-form.ts` maps its issues onto the -controls, and a name another Dot has is refused at once. A preflight panel shows -what `GET /api/health` reports (the control plane, its database, the OpenRouter -key) and, below it, the host report of `GET /api/doctor` (QEMU, the accelerator, -disk, the images), each failing row with the command that fixes it; the doctor's -own key row is left out, so the key is said once. Submitting is `POST /api/dots`, and then the Dot's chat opens. - -A Dot's settings (`/dots//settings`) are the same config, edited in place: -one draft of the whole `DotConfig`, as a form (General, Model, Permissions and -tools, Computer, Limits) or as the same YAML, never a second -model of it. `lib/config-fields.ts` is the one table of what can change (where -each field lives, its words, when a change reaches the Dot) and gives what the -page needs from it: the list of changes, the notice a save ends with, and the -rebase below. Nothing is written until the person has seen the review, which -lists each changed field from what it is to what it will be; the save is a -`PATCH` conditional on the `config_version` the edit began from, so a config -changed meanwhile (another tab, an "Always allow") is refused with 409 -`dot_changed` and never undone. When the host's config moves on under an edit -that is under way, the edits are kept on top of the new config -(`lib/config-draft.ts`) and the page says so; a field only the other change -touched keeps what that change says. The permission editor has one row per -permission of `PERMISSIONS`, with its words and risk, a three-way allow, ask or -deny control, the default said, and the tools of the Dot's own table (`GET -/api/dots//tools`, so a tool the engine adds shows with no change here) -with whether the model is offered each: those are as the Dot has its config -now, so a changed row says its tools show the change once it is saved. A -permission set to what its default is is saved as no entry. A stopped computer -has no table: the page says to start it, and the permissions can be set -meanwhile. A change reaches a running Dot from its next turn (`PUT /config`); -the computer's size (processors, memory, disk) applies the next time the -computer starts, and a disk can grow but never shrink. The Danger zone deletes -the Dot after its name has been typed. - -The Dot header's error banner gives the reason and "Open settings", and one way -back: Reboot while the computer is up (the host reboots only a running -computer), or Start the computer when the computer itself is in ERROR. Both -use the power menu's one action, which asks before a stop and, while a task runs, before a reboot. - -The Tasks page (`/dots//tasks`) shows a Dot's tasks in four sections: -Running (the newest `task.progress` line of each task, what it has spent from -the task row's `spent_usd`, how long it has run, Cancel after a question), -Scheduled and Queue (in the order the dispatcher takes them: priority, then -age, then id, which a test pins to the dispatcher's `ORDER BY`), and History (filtered by how the task ended, twenty at a time). A task -has its own address, `/dots//tasks/`, which opens a drawer over the -list with its state, its result as markdown or the reason it failed, and its -story. The task route knows a task by its id alone, so a task of another Dot -opened under this Dot's address is shown as missing. The story and the progress lines come from the Dot's event log, which -the page reads from its start, asking only for the event types of a task's -story (`readEventLog` of the SDK, `packages/sdk/src/event-log.ts`, is the one -function that pages through the route's `types` filter) only once a task is running -or open, and then keeps current from the live stream. The newest progress line of a running task is also under the name in -the Dot header. A task that waits for an answer shows its approval as a card on its own card and in its -drawer, answerable there (see the Inbox below). - -The chat (`/dots//chat`) is the conversation: the person's messages as -bubbles (one that came through Telegram or WhatsApp says "via Telegram", from -the `origin` of the message), the Dot's as markdown (no raw HTML, links open in a new tab without a -referrer, an image is shown as a link to it, a fenced block has a copy button). -Between them it shows what the Dot did to answer, read from the same event -log: each `tool.called` that names no task as one quiet line (the words of -`TOOL_LABELS` in `packages/shared/src/labels.ts`, which a test keeps equal -to the engine's tool table, then the call's `target`, and how it ended when that was not well; more -than three in a row fold into one line that opens), and each `approval.requested` that names no task, where it -was asked, as the approval card while it waits (answerable there) and as a receipt -line once answered ("Allowed for good" when the answer was "always", with the -person's note). A call and an approval are read into a step one way for the -chat, a task's story and the Activity page, with one set of words for how a -call ended and one format for how long it took (`lib/steps.ts`). The messages route and the log are two -views of one log, so a step is placed between two messages by event id -(`lib/chat-thread.ts`). A message shows as soon as it is sent and is replaced by -the logged one, which `POST .../messages` names by its `event_id`; one that was -`queued` says the computer is waking up until the agent reports. While the agent -thinks or runs a tool a row says so, with the last step. The box grows with the -text, Enter sends and Shift+Enter adds a line, and an unsent draft is kept per Dot -in this browser. The computer panel is always beside the thread (from 1024 px; a -strip above it below that), and both reach the bottom of the window: the page column -is as tall as the window, a Dot's header and tabs stay and the tab's body fills the -rest. The panel shows the desktop, an open browser being a window on it, as -the pictures the host reads from the guest every few seconds while the page is -visible, with a LIVE badge, a warning when a frame is more than 15 s old, and the -words "The Dot has control", because nothing the person does there reaches the -computer. - -The Inbox (`/inbox`, in the rail) is where everything that needs the person is, -from every Dot. Its address holds its whole state: `?tab=history`, `?dot=` and `?permission=` (the old `/approvals` and `/dots//approvals` -redirect to it). "Needs you" lists the waiting approvals, the one that has waited -longest first, then the Dots in ERROR and the tasks that failed in the last 24 -hours (each dismissable; the dismissals are kept in this browser only, so a -cleared browser shows them again). The shell reads them for every Dot at once -(`GET /api/tasks?status=FAILED&finished_after=`, a day back) and again on -`task.failed`, and says when they could not be read. -`lib/attention.ts` counts the three, and the rail's Inbox badge, the tab title and -the favicon all read that one count. The approval card (S7) says what the Dot -wants to do from the engine's tool table (`TOOL_LABELS`), the -permission and its risk (`PERMISSION_INFO`, `permissionPhrase`), the Dot's reason, and the call's -arguments in the form that reads best: a command, a diff for `write_file`, -`edit_file` and `apply_patch`, an address, a schedule, with the raw arguments -under Details (a proxy's password is never shown). It is destructive for a -command, the deletion of a browser identity and a change to a file outside the -workspace. The answers are Allow once, Always allow and Deny with an optional -note. Always allow asks first, naming the permission, what it can do and the tools -it covers (`GET /api/dots//tools`, when the computer answers), and then -`POST /api/approvals//approve` with `always: true`, which changes the Dot's -config (section 9.6). The host answers 409 `already_resolved` to the second -answer; the card says the approval was answered somewhere else. An answered card -stays in place as a receipt until the person leaves the page. Keys, while the -focus is in the list of the cards (WCAG 2.1.4: a single key is never the whole -page's): `j` and `k` move between the cards, `a` allows the selected one once and -`d` denies it, and a card the focus enters is the one selected; a destructive -card is not allowed by a key, `a` moves the focus to its Allow once button, -whose press is the confirmation. History lists every approval that is no -longer waiting, the last answered first: it asks `GET /api/approvals` for the -answered statuses with `order=desc` and `limit=50`, and "Show older answers" asks -for the next page with `before` the last id it holds, so the newest answer is -listed however many approvals the Dots have asked for. A live refresh reads the -newest page again and keeps the older rows it reaches (when more was answered in -between than a page holds, the older rows are dropped and read again on request). -A channel that needs the person (its login was refused: a revoked Telegram token, a WhatsApp device removed on the phone; one the person paused does not count) is a -card under "Channels to link again" with what the host says and a link to the Channels page, counts in the Inbox's number, the title and the favicon, and marks -the Dot in the rail and its Channels tab. The shell reads every Dot's channels at once (`GET /api/channels`) and again on `channel.status` and `channel.changed` (the host announces a pause, a resume and a removal, which no status says, so the marks follow them without a reload); channels -that cannot be read are said, not hidden. - -The Computer page (`/dots//computer`) has three views, named in the address -(`?view=screen|files|usage`, Screen when it says nothing, and so for an address of the -Browser view that was). Screen and Files read the guest and need the computer running: -when it is not, they say why and offer Start, and a route's own 409 `computer_stopped` -gets the same answer. Screen is the desktop as the chat's panel shows it (one -`FrameView` draws both), with "The Dot has control"; the Dot's browsers have no view of -their own, an open one is a window of that desktop, and the Dot makes, opens and closes -them with its tools. Files is a read-only walk through -`/home/dot` (`GET /files/list`, `GET /files`): the folder and the open file are in -the address, a text file is shown as text and an image as a picture, and -everything else, and any file over 1 MiB (text) or 8 MiB (image), is a download -only. What counts as text or an image is `fileType` of `packages/shared`, the one -table the API serves a file by too (markup and svg are text, so a file the Dot -wrote is never run); a file named text that holds a NUL byte is not shown. Usage -reads while the computer is off: what it was given, what it uses (`GET /computer` -embeds the guest's `system` while it runs), the images, why the last start failed, -the model spend today and in total, an Automations card, and Start, Reboot and Stop. -Stop always asks and says that the automations do not run while the computer is stopped -(Reboot asks only while a task runs). The Automations card reads the host's record of the computer (`GET -/computer`: `stop_reason` and `next_automation_at`), which holds both while the computer is -off: "Paused: you stopped this computer" when the person's stop paused them, otherwise when -the next one is due (and that a computer asleep starts shortly before), or that none is due; -it follows `computer.*` events and `automation.next_run`. - -The Skills page (`/dots//skills`, a tab between Computer and Activity) lists the -Dot's skills (`GET /skills`, section 8.6), read only: each with its name, its -description and whether it is built in or written by the Dot, and the open one (in the -address, `?skill=`) shown as Markdown by the chat's renderer, its frontmatter left -out and its path above it. It reads the Dot's computer, so a computer that is not -running is said with Start, and the list is read again when a turn ends, which may have -written a skill. - -The Channels page (`/dots//channels`) is one card per channel the server runs (`available` in `GET /api/dots/:id/channels`): Telegram always, -WhatsApp only when the server was started with `INVISIBLE_DOTS_WHATSAPP=1`. Telegram not connected asks for the bot's token (a password field, sent once over -`PUT .../channels/telegram`, emptied at once and never read back); connected, the card shows the bot, pairs a chat (`POST .../pairing`: a link that opens the -chat with the code ready, its QR code, the words to type, and the ten minutes counting down; the panel closes when `channel.peer.paired` arrives), lists the -people paired with a Revoke each, and has the three switches of the channel's settings (approvals here, tell me when a task ends, show what the Dot wants to -run), each saved as it is flipped, a Pause and a Disconnect that says the token and the people go. A refused token shows "Telegram needs a new token" with -the field to replace it, the people staying paired. WhatsApp first says that the client is unofficial and can get the number banned, then links by scanning: -the page follows `GET .../whatsapp/qr` (the frames of `ChannelLinkFrame`, reduced to one view by `lib/channel-link.ts`), draws each code itself as an SVG -from the QR modules (`qrcode`, dark on light in either theme), and ends linked with the number, or failed with the host's reason and "Link again"; a link -already going on when the page opens is followed. A number counts as linked when the host holds its account, not by the word `connecting` (a server start, -a Resume or a credential refresh report it again for a number that stays linked): only a login the host says has to be redone, or a link with no account yet, -goes through the scan. Cancel removes the channel the page's own start made, and only pauses one that holds a linked number or people (Link again on a -number that needed it), so nothing is deleted without the question that Unlink asks. The page follows `channel.status`, `channel.changed` and -`channel.peer.paired` live. The browser tests run the real Telegram and WhatsApp adapters against the hub's own fakes (`FakeBotApi`, `FakeWhatsAppConnector`). - -The Activity page (`/dots//activity`; the old `/timeline` address redirects -to it) is the whole event log of the Dot as readable lines, for the person who -wants to know exactly what happened. `lib/events/view.ts` describes every type -the log can hold, written against the data the contract gives that type (a type -or field added to `packages/shared/src/events.ts` does not compile until it is -described, and a test fails for a type with no family): a tone, a title, one -line of detail (cut at 280 characters; under "Data" the event is shown as -stored), and, for a message that came through a channel, "via Telegram". A -`tool.called` line names the tool in words, what it acted on (`target`), how it -ended (ok, failed, denied, interrupted) and the policy decision; `agent.started` -says "The engine started (the key was sent again)". The families (chat, tasks, -tools, approvals, browser, computer with the agent, memory, channels, the Dot's -own config) are chips: the ones chosen become the `types` of the request, so a -family not chosen is never read, and a live event is kept only if it is of a -chosen type. The page reads the newest 200 events first (`order=desc`) and -"Load older events" goes on with `before`, the id of the oldest one held, so the -cost does not grow with the age of the Dot, and says where the log starts. The -search is over the lines read so far (title, detail, type, channel), never a -request of its own, and says so; the order switch only turns the list over. The -export saves the events on screen, after the search, as JSON Lines (one stored -event per line, oldest first) named for the Dot and the range of ids. - -### 9.8 Messaging channels - -A person can talk to a Dot from a chat (Telegram, WhatsApp). This is the control -plane's business only: the Dot never sees a channel, no guest route or event -type names one, and the Dot has no tool that sends a message anywhere. -`packages/channels` holds the **channel hub**, built in `startServer` right after -the Scheduler, started after `scheduler.start()` and closed before it. It runs in -the server process because the database is single-process (section 9.1) and the -credentials live in it. It uses only what the Scheduler offers: `sendMessage` -(with an origin), `resolveApproval`, `requireDot` and the event log. - -An adapter (`Channel`) is transport only: `run(sink, signal)` connects and -delivers until aborted, `sendText`, `sendApproval` and `editApproval`, optionally `typing`. A `ChannelType` makes the -adapter for a binding and names the secrets a binding of its kind keeps (and which -of them are credentials a log line could hold, `scrubNames`). A kind that is -linked by scanning a code on a phone (`scanned`: WhatsApp) is linked with `link` -and never given credentials; the others with `add`. The hub owns every policy: - -- **Who may talk.** Only a person paired with a one-time code, by the channel's - stable id (a Telegram numeric user id, a WhatsApp phone number, never a mutable - name). The code is eight - symbols (40 bits), valid ten minutes, used once, stored as a SHA-256 hash with - the binding id, and pairs the sender as an `owner` together with their chat. - Anyone else, a chat that is not a private one, an empty message: dropped - before anything is written, so a stranger costs no row and no model call. A - paired person is held to a token bucket (10 messages at once, then 20 per - minute, told once) and to 8000 characters per message. -- **Inbound.** A message becomes `Scheduler.sendMessage(dot, text, {channel, - binding_id, chat_id, external_id})`; the guest receives `{text}` only. The - channel's own message id is part of the `user.message` event, which a unique - index allows once per Dot, binding and channel id, so the message and the proof - that it was handed over commit together. `sink.inbound` resolves only after the - control plane has answered, so an adapter commits its offset after the hub is - done with the message. A redelivered message, after a failure at any point or - across a restart, finds the first one: the Dot gets it once, and the repeat is - answered with the first message. -- **Outbound.** One `events.stream({dotId}, {after: event_cursor})` per binding. - `message.assistant` goes to the chat of the `user.message` its `in_reply_to` - names, when that message came through this binding and its person is still - paired; an answer that answers nothing (an automation's) goes to every owner's - chat; an answer to a message from the web or another channel is not mirrored. - `task.completed`, `task.failed` and an answer that answers nothing go to the - owners unless `notify_tasks` is off. - `agent.state` THINKING shows typing in the chat of the last message while the - Dot has not answered it. Text is split at the adapter's `maxText` on paragraph, - line and word boundaries. The cursor moves after a send succeeded (events that - need no send are written in batches), so a restart resumes where it stopped; - a crash between a send and the cursor write sends that message again. A send - that fails is retried with backoff (a channel's `retry_after` is honoured); one - the channel refuses for good (the person blocked the bot) is dropped. -- **Approvals.** When the Dot asks (`approval.requested`) and the binding's - `approvals` setting is on, every owner's chat gets a prompt: what the Dot wants - to do and the permission with its risk, in the words the web client's approval - card uses (`packages/shared/src/labels.ts`), the reason and the - arguments, each cut to 300 characters (the - arguments can hold private data, and a chat is read by a third party; with - `show_arguments` off the prompt leaves them out), with an - Approve and a Reject button. Only a paired owner, in their private chat, can - answer; the button's id proves nothing, so the hub also checks that the - approval belongs to the binding's own Dot and that approvals are still asked - in chats. The answer is `Scheduler.resolveApproval`, the one way an approval - is answered, so a second press, or a press after the web answered, is the - scheduler's 409 and the person is told it was answered already. The person - always gets a short notice for the press. Every answer, whoever gave it, - arrives as `approval.resolved` and edits the prompts to the outcome with the - buttons taken away; `channel_prompts` remembers where each prompt is. The - prompts are brought up to date when the channel starts, when a person pairs - and when `approvals` is switched on: a prompt whose approval was settled - meanwhile (answered elsewhere, or its task ended: "No longer needed") is - edited, and a pending approval with no prompt in an owner's chat is sent one. - Delivery is at least once like every send: a crash between sending a prompt and - recording it sends it again. An approval over a chat is as strong as the - person's Telegram account; switching `approvals` off keeps the answer in the - app. -- **No link previews.** Nothing a Dot names is fetched on its behalf, by this - process or by a messaging service: the Dot's only way to the web is its own - browser (section 8.3). An approval prompt shows the URL a call is about to open, - query included, so a preview would send that URL to Telegram's crawler (or, with - Baileys, request it from this machine) before anybody approved it. Every Telegram message and edit says `link_preview_options.is_disabled`, - and every WhatsApp text and edit `linkPreview: null`; the adapters' tests assert - it against the fake Bot API (which records a link it would have fetched) and the - library's own content builder. -- **Answers in words.** A channel without buttons (`approvalByText`: WhatsApp) - ends its prompt with `Reply "yes ap-xxxxxx" to approve or "no ap-xxxxxx" to - reject`, where the token is the last six characters of the approval's id. The - hub, not the adapter, reads a message that is exactly that, from a paired owner - only (a stranger's message is dropped before anything is read, so a stranger is - never answered), and resolves it with the same checks and the same - `Scheduler.resolveApproval` as a button; the person is told "Approved.", - "Rejected.", "It was answered already.", "That request does not exist." or that - two requests share the code. Anything else, including a bare `yes`, is an - ordinary message to the Dot: an answer given by a misread sentence would be an - approval nobody meant. -- **Failure.** An adapter that fails is started again after an exponential backoff - (1 s up to 60 s, with jitter) on a fresh instance; `ChannelNeedsRelinkError` - (a revoked token, a logged-out device) stops it until the person relinks. - Every change of status is a `channel.status` host event, once; the reason is - cut to 300 characters and has the binding's credentials replaced, whatever - the adapter wrote. A new binding starts after the Dot's latest event: history - is not replayed into the chat. When the Dot is deleted its bindings go with it - (foreign keys) and the hub stops the adapter on `dot.deleted`. -- **Credentials.** Stored as secrets scoped to the Dot, under the names the - channel type declares, encrypted like the OpenRouter key, never returned, - never pushed to the guest, deleted with the binding and with the Dot. A channel - type may check credentials before anything is stored (`check`): Telegram asks - `getMe`, so a wrong token is a 400 `invalid_credentials` (the message never - holds the token), an unreachable Telegram a 502 `channel_unreachable`, and a - bot another Dot already uses a 409 `account_in_use` (a bot serves one Dot: two - pollers on one token take turns failing). Giving a linked channel a new token - (`PUT`) starts it again with the people kept; it is the way back from - `needs_relink`. - -#### Telegram - -`packages/channels/src/telegram/` is the adapter, on grammY (the Bot API client, -MIT). It is transport only, like every adapter. - -- **Long polling**, because the control plane listens on a local address behind - NAT and polling needs only outbound HTTPS. Telegram keeps an update that was - not confirmed for 24 hours: a PC that is off for longer loses what was sent - meanwhile. A webhook the bot had is deleted on connect (the bot is the Dot's - own). Only `message` and `callback_query` updates are asked for. -- **An update is confirmed to Telegram** (the `offset` of the next poll) only after - the hub dealt with it. When the hub could not record a message the adapter - fails, the hub starts it again, and Telegram offers the update once more; the - hub recognises one it already gave the Dot by its id, `:` - (update ids of two bots overlap, and a Dot's bot can be replaced). No offset is - stored: a restart is the same as a failure. -- **Pairing.** `/start ` in a private chat is a pairing attempt; the deep - link `https://t.me/?start=` sends exactly that. A bare `/start` and - everything a stranger sends get no answer. Only private chats are served. - Authorization is the sender's numeric user id, never a username. -- **Approval buttons.** `callback_data` is `ap1:y:` or - `ap1:n:` (47 bytes for `appr_`): versioned, checked to fit - Telegram's 64 bytes when the prompt is made (an id that does not fit is - refused for good) and again when parsed, and parsed strictly, so data from - another version or a forged shape is answered "This button is out of date" - and goes no further. A press is confirmed to Telegram only after the hub dealt - with it, like a message. The press is always answered (`answerCallbackQuery`) - with the hub's notice, best effort: Telegram refuses an answer that is too - old. An edit of a prompt that is gone or already says the same counts as done. -- **Messages.** Plain text, no formatting; a long answer is split at 4000 - characters. A message without text (a photo, a voice note) is answered "not - supported yet" to a paired person and dropped. Typing shows while the Dot - thinks. -- **Failures are told to the hub in words.** A 401 is a revoked token - (`needs_relink`); a 409 on polling says another process polls the same bot; a - 403, 400 or 404 on send is final (the person blocked the bot), a 429 carries its - `retry_after`, anything else is retried. The token is in every request URL, so - no message the adapter makes carries a URL, and the hub replaces the binding's - token in whatever it stores or logs. Retries and backoff are the hub's alone. -- **Not private.** Telegram bot chats are not end-to-end encrypted: Telegram can - read what a person and the Dot write there. The CLI says so when a bot is linked. - -From the CLI, `invisible-dots channel add telegram --dot ` (token asked for in -a terminal, where what is pasted is not echoed, or read from stdin, never from arguments), `channel list [--dot]`, -`channel pair --dot ` (prints the deep link and the words to send) and -`channel remove --dot `. - -#### WhatsApp (opt-in, unofficial) - -`packages/channels/src/whatsapp-baileys/` is the adapter, on Baileys -(WhiskeySockets, MIT), a client of the WhatsApp Web protocol. **It is not an -official way to use WhatsApp.** It links the Dot as a device of a personal account, -which WhatsApp's terms do not allow for automation, and WhatsApp can answer by -restricting or banning the account. The library is a release candidate pinned to -one exact version (`7.0.0-rc14`; a test keeps `package.json` and the lock file at -the same exact version) because the protocol moves under it: when WhatsApp stops -accepting that release, WhatsApp stops working until the pin is moved. Use a number -of its own (a spare SIM or eSIM), never the one a person lives on. The official -Cloud API (a business account and a public webhook) is a later adapter on the same -hub. - -- **Licenses and the opt-in install.** Baileys is MIT, but it depends on - `libsignal`, which is GPL-3.0, and the default `npm install` holds nothing GPL. No - workspace declares Baileys (not as a regular, dev, optional or peer dependency), - so the root `package-lock.json` holds none of it and `npm ci` installs no GPL - package (`tests/repo/default-install-licenses.test.ts` scans the lock file's - licenses). The client lives apart in `optional/whatsapp/`: its own - `package.json` (Baileys at one exact version, a release candidate) and - `package-lock.json` (the integrity hash of every package), outside the - workspaces. One command installs it, `npm run whatsapp:install` from the - repository root: `npm ci --prefix optional/whatsapp --ignore-scripts`, a clean - install of that lock file without install scripts. `client.ts` is the one file - that names the library: it looks for it in that folder when a connection opens - (`createRequire` from the folder, then one `import()` of the file's URL), loads it - only if it is there and is the pinned release, and otherwise throws - `WhatsAppClientMissingError` (not installed) or `WhatsAppClientVersionError` (the - installed `node_modules/baileys` is not the exact version - `optional/whatsapp/package.json` declares, which stays the one owner of the pin: - a pull that moves it leaves the old release installed until the install is made - again). Both messages name the command and the variable that turn WhatsApp on. - The error is the channel's status detail, the server logs it at start when - `INVISIBLE_DOTS_WHATSAPP=1` is set and `whatsappClientProblem` finds one (the - same check the loader makes), and the "it is off" answer of linking carries the - same words (`WHATSAPP_ENABLE_HELP`). Nothing imports the - library by name, not even as types: the adapter writes down what it uses of it - (`WhatsAppClient`, with the library's credentials and key data as type - parameters) and compiles without it; `npm run typecheck:whatsapp` (a CI job, - with the client installed) proves the real module has that shape, that the - options of the socket the adapter builds (`socketConfig`, typed - `WhatsAppSocketConfig`) are each an option of the library with a type it takes - and none is a name it lacks, that the adapter's auth state is one the library - accepts, and that the key groups are the library's own. (The methods of - `WhatsAppClient` are compared in both directions, so those two are asserted - separately, one way.) `npm run test:whatsapp` runs the tests that need - the real library (`packages/channels/test-optin/`, left out of the default - suite). The command bundle has no import of it, so no build of ours embeds GPL - code. `THIRD_PARTY_NOTICES.md` says what that means for whoever distributes an - installation that has `optional/whatsapp/node_modules`. -- **Off by default.** The server runs WhatsApp only when started with - `INVISIBLE_DOTS_WHATSAPP=1` (`defaultChannelTypes` in `apps/api/src/start.ts`). - Otherwise the type does not exist: `GET .../channels` lists only the kinds it - runs (`available`), and linking answers 400 with how to turn it on. Baileys is - loaded by `client.ts` when a connection opens, so a server that never - links WhatsApp never loads it, and no file but `client.ts` names it (a test reads - the sources). -- **One port.** `port.ts` is what the channel needs of a connection (messages in, - text out, a code to scan, why it ended); `baileys.ts` implements it over - the network and `FakeWhatsAppConnector` for tests, because WhatsApp cannot be - faked. Everything WhatsApp-specific that is a decision is in `whatsapp.ts` and - tested against the fake; the glue is tested for what it decides alone (how a - message is read, how a close is understood) and for surviving a socket that - cannot connect. The real network is not reached by any test. -- **Linking by a code.** `POST .../whatsapp/link` creates the binding without - credentials and starts the adapter, which opens a connection that is not - linked; WhatsApp sends a code every few seconds, shown on the phone under - Settings, Linked devices, Link a device. `GET .../whatsapp/qr` streams the codes - and the end as `ChannelLinkFrame`. The code is a way into the account for as long - as it is shown, so it lives in memory only (`LinkSessions`): not in the - database, not in an event, not in a log, not in a status, and the stream is - `no-store`. A watcher that joins late gets the code on show now. The scan ends - with `linked` and the number; a code that ran out before the scan, a device WhatsApp - rejects or one removed on the phone ends with `failed` and `needs_relink` - (`ChannelNeedsRelinkError`); a connection that is lost is retried by the hub with - backoff. Linking again after a failure deletes every key of the old device first. - WhatsApp asks for a new connection when a link finishes (status 515); the adapter - opens it at once, and gives up after three in a row. -- **The keys are secrets.** The linked device's identity and Signal keys are an - account takeover if they leak, so they are not the plaintext JSON files of - Baileys' own helper: `AuthStore` implements Baileys' `AuthenticationState` over - the encrypted `secrets` of the Dot, one secret for the credentials and one per - group of keys (eleven; a `Record` over the library's own list of groups makes a - group added by an upgrade a compile error), under the same AES-256-GCM and the - same row-bound associated data as the OpenRouter key. A change is written (the - credentials and the groups that changed, in one transaction, in order) before it - is acknowledged to Baileys, so a crash leaves the state as it was or as it is - now, never a step apart; a failed write is kept and tried again with the next. - A closed store refuses every write, so a delete by the hub is final, and the - write itself holds the binding row, so a session still open when the channel or - its Dot is deleted cannot write its keys back afterwards (the write is refused - as `ChannelGoneError`). They are never sent to the guest, and deleted with the - channel and with the Dot. -- **Who is who.** WhatsApp addresses a person by phone (`@s.whatsapp.net`) - or by LID (`@lid`), and may switch. The peer is the phone number when it is - known from the address, from its twin address in the same message, or from what - the account learned earlier (a local lookup that sends nothing to WhatsApp), and - `lid:` otherwise, so one person is one peer whichever address is used; the - chat to answer is the matching address. The one edge: someone paired while only - their LID was known, whose number is learned later, appears under the number and - pairs again. Device and agent suffixes are dropped. -- **Reply-only.** Nothing is sent to a chat that did not write first: the hub sends - only to paired people, and a person pairs by writing the code. A stranger is never - answered, told they are refused, or sent a read receipt (the unread message of a - chat is marked read just before the Dot answers that chat, from a bounded memory), - so no stranger learns that the number is alive. Each send is preceded by a typing - indicator and a pause of 0.4 to 1.5 seconds. The account does not announce itself - online. No groups (WhatsApp is told to ignore every address that is not one - person's, so those messages are neither decrypted nor seen), no broadcast, no - channel posts, no messages the account wrote itself, no attachments (a paired - person is told "not supported yet", as on Telegram). There is no way to send to a - number that has not written. -- **Pairing.** `pair ` as a whole message; the link - `https://wa.me/?text=pair%20` opens the chat with exactly that - ready to send. The number is the account the adapter reported when it connected. -- **Approvals.** In words, as described above: the prompt is a message with the - reply to send, and when the approval is settled the prompt is edited to the - outcome (WhatsApp limits how long a sent message can be edited, about fifteen - minutes, so the prompt of an old approval may keep its question; a late answer is - told it was answered already). -- **At most once on the way in.** WhatsApp confirms a message to its sender when - it arrives, not when the hub dealt with it, so unlike Telegram a message the hub - could not record cannot be offered again. The adapter then drops the connection, - the channel shows `error`, the hub reconnects, and the person writes again. - Messages come one at a time, in order. -- **Not private, and not stable.** WhatsApp and the account's other devices see - what the Dot writes; the account's owner sees the Dot as a linked device. Baileys - follows a protocol WhatsApp does not publish: a release of it can stop working - without notice, and nothing here can prevent a ban. - -From the CLI, `invisible-dots channel link whatsapp --dot ` prints the risk, -then each code as a QR for the terminal until the number is linked, then -`channel pair whatsapp --dot ` prints the link and the words that pair. - -## 10. Out of scope for this version - -Snapshots and rollback, remote desktop -and interactive terminal, artifacts, backups, quotas, network policies, -multiple hosts, organisations and RBAC, macOS hosts. The tables and states -above leave room for them; nothing here pretends to implement them. - -Scheduled and recurring jobs are in scope: they are the Dot's automations -(sections 8.8 and 9.5), which run in its guest and, with the computer off, -start it. - -## 11. Getting a host ready - -The same commands on every host: - -```text -invisible-dots setup --all everything below in one run, in order, that can be run again (section 11.4) -invisible-dots setup get QEMU and its accelerator ready (may ask for administrator rights once) -invisible-dots doctor check everything, print one line per check and the command that fixes a failure -invisible-dots image build build the golden image and the runtime ISO (section 3.3) -invisible-dots server run the control plane and the web client in the foreground (section 9.7) -``` - -`invisible-dots server` is the one entry point of the control plane; no -other program starts it. It stops cleanly (closes the database, releases -`server.lock`) on Ctrl+C, a service manager's SIGTERM, a closed terminal -(SIGHUP, which Node also raises on Windows when the console window closes) -and Ctrl+Break on Windows (SIGBREAK): the same handler for all four. - -The OpenRouter key is then stored with `invisible-dots secret openrouter`, -the same line on every host and in every shell: in a terminal it asks for the -key and reads one line, and piped it reads standard input. The key is never an -argument, so it never lands in a shell history or a process list. - -### 11.1 doctor - -Checks, in this order, each with `ok` / `missing` / `failed` and a fix line: -Node version; QEMU found (and its version, 8.2 or newer); `qemu-img` found; -the accelerator usable (Linux: `/dev/kvm` opens read-write; Windows: the -`HypervisorPlatform` optional feature is enabled, read without administrator -rights through `Get-CimInstance Win32_OptionalFeature`), confirmed by actually -running QEMU with `-nodefaults -no-user-config -machine q35 -accel --cpu host,-vmx,-svm -display none -no-reboot -boot reboot-timeout=0` and seeing it exit -with code 0; the data directory, `INVISIBLE_DOTS_HOME`, a path QEMU can be -given (plain ASCII, no comma, section 3.2) with enough free space; -the golden image and runtime ISO present and matching their manifests; the -web client built (the files `invisible-dots server` serves, section 9.7; the -fix is its build command; a row of `invisible-dots doctor` and `setup` only: the -report of `GET /api/doctor` is read in the web client, which is built by then); the OpenRouter key stored. `doctor` never changes anything and creates nothing. -Exit code 0 only when every check is `ok`. On a host invisible_dots does not -run on, the accelerator rows say so, and the report still prints. - -- The probe runs guest code instead of holding the machine before its first - instruction: the empty machine's firmware finds nothing to boot, - `reboot-timeout=0` makes it reset at once, and `-no-reboot` turns the reset - into QEMU exiting with code 0. So exit code 0 means the accelerator opened - AND ran the CPU model the Dots use; there is no monitor, the same as for a - Dot. Measured with QEMU 8.2.2 and KVM: exit 0 after 0.2 s, and the firmware's - debug port reads "No bootable device. Retrying in 0 seconds." then - "Rebooting."; without either flag QEMU never exits. A probe still running - after 30 s is killed and reported as a virtual CPU that does not run, with - QEMU's first line of error output. Not measured with WHPX yet. -- The probe uses the Dots' machine type, not `-machine none`: QEMU 11.1 with - WHPX aborts on `-machine none` (`X86_MACHINE` assertion, measured). -- The probe has the last word: when the host-side check says `missing` but - QEMU starts with the accelerator, the accelerator row is `ok` and `setup` - enables nothing. Measured on Windows 11: WHPX initialised while the - `HypervisorPlatform` feature read as disabled (Virtual Machine Platform on). -- The OpenRouter key is read from the running server's `GET /api/health`, - because only the server may open the embedded database (section 9.1). While - the server is down that row is `failed`, with the command that starts it. - Inside the server (`GET /api/doctor`) the row asks the Scheduler directly. -- One code runs the report in both places: `runDoctor` in - `apps/vm-manager/src/doctor.ts`, over the real machine's - `hostDoctorDeps()` and the images' rows of `apps/api/src/doctor.ts`; the CLI - only renders it (`apps/cli/src/doctor/render.ts`). The wire types - (`DoctorCheck`, `DoctorAnswer`) are in `packages/shared/src/api.ts`. - -### 11.2 setup - -`setup` runs `doctor`, then fixes only what is missing: - -- **Windows**: downloads the official QEMU installer for the version pinned in - `virtualization/qemu/windows.json` (URL and SHA-256) into a temporary - directory as the normal user, verifies the hash, then asks for elevation - ONCE (`Start-Process -Verb RunAs` on a generated PowerShell script) to run, - in that single elevated session: `dism /online /enable-feature - /featurename:HypervisorPlatform /all /norestart` when the feature is off, and - the QEMU installer silently (its arguments, `/S`, come from the pin) when - QEMU is missing. Nothing the elevated session runs or writes is in a place - the normal user can change, because a program running as that user could - swap an installer between its hash check and its start, plant a DLL next to - it, or turn the result file into a link to anywhere. So the session creates - a new directory directly under ProgramData whose ACL, set in the same call - that creates it, lets only Administrators and SYSTEM write (the person may - read), and refuses to go on if it existed or is not empty; it copies the - installer there, hashes the COPY and runs the copy from that directory; it - finds dism through the system directory Windows reports, never through an - environment variable the user can set; and it writes the exit codes to a - result file in that directory, which the normal session reads and then - removes with the directory. Exit code 3010 from dism means a restart is - required: `setup` says so and stops with exit code 5. The person restarts - and runs `invisible-dots doctor`. The installer's Authenticode certificate is - outside its validity period, so the pinned hash is the only trust anchor. -- **Linux**: prints and runs `sudo apt-get install -y qemu-system-x86 - qemu-utils` on apt-based systems (and prints the equivalent `dnf` / `pacman` - line elsewhere instead of guessing), then checks `/dev/kvm`; if it is not - accessible it prints `sudo usermod -aG kvm $USER` and that a new login is - needed. `setup` refuses to run as root: it would check `/dev/kvm` as root and - name root in that line, not the person who runs the server. It calls `sudo` - itself for the one step that needs it, as Windows elevates its one step. -- The minimum of section 3.1 is 8.2 because Ubuntu 24.04's own - `qemu-system-x86` is 8.2.2, so `setup` on the most common LTS installs a QEMU - that `doctor` accepts; 8.2.2 was measured against the argv of section 3.4 - and the probe of section 11.1 (with KVM). Debian 13 ships 10.0. - -The project never redistributes QEMU binaries: Windows gets the official -installer, Linux gets the distribution's package. - -### 11.3 Licensing - -The root LICENSE (MIT) covers this repository, except everything under -`invisible_engine_dots/`, which is under `invisible_engine_dots/LICENSE` (MIT) and the -nested notices next to the code they cover; `THIRD_PARTY_NOTICES.md` at the -root carries that license, the notices of the earlier TypeScript engine whose -text the history still holds (Open Multi-Agent, MIT), and names every nested -notice, and `invisible_engine_dots/UPSTREAM.md` records the version and commit -the fork comes from. The engine's source goes -onto the runtime disk with the fork's `LICENSE` and `UPSTREAM.md` -(`/opt/invisible-dots/engine/`). No list is written by hand, so it cannot -drift from the lockfile. QEMU (GPL-2.0) is installed -from the official Windows installer or the distribution's package and is only -ever run as a separate program; it is never linked into, bundled with or -shipped by this project. The guest operating system (the Ubuntu cloud image), -`uv`, the browser engine, `invisible-playwright-mcp` with its Python -packages, and the Python packages the engine's lock -(`guest/image-builder/builder/engine-requirements.lock`) names are -downloaded from their publishers when the golden image is built, each under -its own license, as the wheels the publishers released. The golden image is -published as built by CI (the release `golden-`, made by -`.github/workflows/golden-image.yml`; `image build` downloads it when its -inputs are the checkout's, `guest/image-builder/src/prebuilt.ts`, and builds the -image itself otherwise); the sources of its Ubuntu packages are in Ubuntu's -archive, and whoever copies it takes on the license terms of the components -inside it. - -The default `npm install` holds nothing under the GPL (section 9.8, WhatsApp); that is a -statement about the npm lock file and no more. The few LGPL packages of the lock file -are the prebuilt image libraries (`@img/sharp-*`) that Next.js may install; the notices -name them. The VM image is a different thing: it carries the Ubuntu guest operating -system and the apt packages the golden image installs on it (`apt_packages` of -`guest/image-builder/pins.json`: Xvfb, XFCE and their libraries, over the Linux kernel, -bash and the rest of Ubuntu's base), most of it GPL or LGPL software under its own -licenses and source offers, downloaded from Ubuntu's archive on the machine that builds -the image, which is CI when the image is the published release of the paragraph above. Whoever copies a golden image to another -machine takes those licenses with it (the paragraph above). - -### 11.4 setup --all - -`invisible-dots setup --all` is what the quick start used to have the person -type after `npm ci` and the build of the command line, as one command -(`apps/cli/src/setup/all.ts`). Every step is the code of the command that -owns it, run as a function, and skips what is already done, so the same -command is also how a run that stopped is continued. In order: - -1. **The guest daemon.** `go build` of `guest/dot-agentd` for linux/amd64 with - no cgo (`agentdBuildCommand()` in the image builder, which also names where - the runtime disk takes the binary from). It comes first because it needs - nothing but Go: a missing Go stops the run here, with the install command of - each host in one line (`GO_INSTALL_HINT` in `setup/build.ts`: winget on - Windows, snap on Linux; one text, so that no platform check is added to - section 1.1), before anything is changed. It is not skipped: Go's build - cache makes an unchanged daemon take seconds, and the image builder turns the - same bytes into "already built". -2. **QEMU and its accelerator.** `prepareHost()` of `setup/setup.ts`, the code - `setup` itself runs (section 11.2), so the one elevated step (UAC on - Windows, `sudo apt-get` on Linux) stays one step and `setup` as a command - prints what it printed before. It comes before the long builds so that a - restart or a new login it asks for costs nothing already done. Where it asks - for a restart (exit code 5) or for `usermod -aG kvm`, `setup --all` says to run - itself again and stops with `setup`'s exit code. -3. **The web client**, unless `locateWebBuild()` finds it complete: node runs - `apps/web/scripts/build.mjs` (the script `npm run build --workspace - @invisible-dots/web` runs; node starts it directly because npm is a `.cmd` - file on Windows and the CLI never starts a program through a shell), its output - on the person's terminal. Next's telemetry is off in `apps/web/next.config.ts` - itself, so this build and any other send none. -4. **The guest images**, `image build` as it is (the runtime disk first, then - the golden image). - -A step that fails ends the run with exit code 1 and the reason, and nothing -after it is run; the message ends with the command to run again. At the end it -prints how to start the server and the address of the web client -(`INVISIBLE_DOTS_WEB_LISTEN` or its default), which has no login: the web -server adds the API token itself (section 9.7). It does not -start the server itself: `server` runs in the foreground until Ctrl+C. Ctrl+C -asks `image build` to stop (it kills its builder VM) and ends the run between -steps; a second Ctrl+C exits at once. - -Not in it, on purpose: installing Node, Git and Go (the Windows installers and -snap ask for administrator rights of their own, and `setup --all` keeps to -one), the clone, `npm ci` and the build of the command line (the command is -their product), and the OpenRouter key (the web client's Home page takes it -once the server runs, or `invisible-dots secret openrouter`). diff --git a/docs/guide.md b/docs/guide.md deleted file mode 100644 index d100ab18..00000000 --- a/docs/guide.md +++ /dev/null @@ -1,842 +0,0 @@ -# The invisible_dots guide - -Everything about running invisible_dots, in one page. The short version is the [README](../README.md). - -A **Dot** is a persistent AI agent with a computer of its own. You give it a -name, a model and a set of permissions; invisible_dots gives it a QEMU virtual -machine on your own PC, with a disk that persists, a Linux desktop, a shell, its -own files, memory and skills, and browser identities that keep their cookies and -logins from one task to the next. You talk to it from a web UI, the command -line, an HTTP API, Telegram, or (opt-in) WhatsApp. The model is any model on -OpenRouter, paid with your key. - -## Highlights - -- **A computer, not a sandbox for one command.** Each Dot has a - hardware-accelerated VM (KVM on Linux, Windows Hypervisor Platform on - Windows) with a persistent disk. It powers off after a quiet spell - (`idle_timeout`, 15 minutes in the sample) and starts again for the next - message, the next task or one of its own automations; disk, identities, - memory and skills stay. -- **A browser built not to look automated.** The only browser of a Dot is - [invisible-playwright-mcp](https://github.com/feder-cr/invisible_playwright_mcp), - on [invisible_playwright](https://github.com/feder-cr/invisible_playwright): a - Firefox patched in C++, with the fingerprint set inside the engine instead of - injected into the page. Each identity keeps its own profile and fingerprint. -- **It learns, as Claude Code does.** The Dot keeps its own memory, notes in - `/home/dot/memory`, and its own skills, how it does a kind of task, in - `/home/dot/skills`. There is nothing to set: it writes them with its file - tools, and its prompt names them. -- **You set what it may do.** Every tool belongs to a permission, and each - permission is allow, ask or deny. An ask waits for your answer, survives a - restart, and lets that one call run once. -- **A crash loses nothing you saw accepted.** A restart of the server, a crash - of the Dot's engine or a `kill -9` keeps every message and task you were told - was accepted, and a tool call cut short is never run twice: the model is told - its outcome is unknown and checks before trying again. -- **Local and private.** The reasoning happens inside the Dot's VM; the control - plane on your PC runs the VMs, the queue, the events, the approvals and the - channels. invisible_dots has no server of its own and no telemetry. - -> [!IMPORTANT] -> invisible_dots is alpha. No package or hosted service is published: the one -> thing published is the golden image, built by CI from this repository's pinned -> inputs and downloaded by `image build` when its inputs are yours; everything -> else you build on your own machine. The acceptance run (`tests/e2e/run.ts`) -> passes its 16 steps on a Linux host with KVM (the linux-host container on -> WSL). On Windows a Dot's VM boots under the Windows Hypervisor Platform with the CPU -> model `host,-vmx,-svm` (measured; plain `-cpu host` pauses it), and the rest -> of the lifecycle there was driven by hand through the web client, not by an -> automated run ([architecture: VM definition](architecture.md#34-vm-definition)). - -**On this page:** [Quickstart](#quickstart) · [What to ask a Dot](#what-to-ask-a-dot) · -[What a Dot can do](#what-a-dot-can-do) · [The web UI](#the-web-ui) · -[The command line](#the-command-line) · [Approvals](#approvals) · -[The browser](#the-browser) · [Talk to it from your phone](#talk-to-it-from-your-phone) · -[How it works](#how-it-works) · [Security](#security-model-and-known-limits) · -[Privacy](#privacy) · [Configuration](#configuration) · -[Troubleshooting](#troubleshooting) · [Updating](#updating-and-uninstalling) · -[Status](#status-and-what-is-not-done-yet) · [Development](#development-and-tests) - -## Quickstart - -Install Node, Go and Git, clone, build the command line, run `setup --all`, run -`server`, and open the web UI to make the first Dot. - -### Requirements - -- Linux or Windows, on x86-64, with hardware virtualization on. macOS is not - supported. -- Node 24 or newer, Git, and Go 1.25 or newer (to build the guest daemon). -- QEMU 8.2 or newer; `setup --all` installs it and enables the accelerator. -- At least 20 GiB free for the data directory (`doctor` checks it). Each Dot's - disk is a copy-on-write overlay that grows as it writes, up to its - `computer.disk` (40 GB in the sample). -- Memory: `image build` boots a builder VM with 2 vCPUs and 4 GiB, and each - running Dot takes its `computer.memory` (4 GB in the sample). Inside a Dot, - each open browser takes roughly 0.8 GB, and at most `max_open` (default 3) - are open at once. -- A data directory path in plain ASCII with no comma. The default is - `~/.invisible-dots`, under your account name, so an account named `José` - fails at `doctor`; set `INVISIBLE_DOTS_HOME` to a path such as - `C:\invisible-dots` before every command, the server included. -- An [OpenRouter](https://openrouter.ai) key. - -### 1. Get the tools and the code - -**Windows**, in PowerShell: - -```powershell -winget install -e --id OpenJS.NodeJS.LTS; winget install -e --id GoLang.Go; winget install -e --id Git.Git -$env:Path = [Environment]::GetEnvironmentVariable("Path", "Machine") + ";" + [Environment]::GetEnvironmentVariable("Path", "User") -git clone https://github.com/feder-cr/invisible_dots -cd invisible_dots -npm ci -npm run build --workspace @invisible-dots/cli -``` - -**Linux** (Ubuntu 24.04), in bash: - -```bash -curl -fsSL https://deb.nodesource.com/setup_24.x | sudo -E bash - -sudo apt-get install -y nodejs git -sudo snap install go --classic -git clone https://github.com/feder-cr/invisible_dots -cd invisible_dots -npm ci -npm run build --workspace @invisible-dots/cli -``` - -### 2. Get the host ready, in one command - -The same command in PowerShell and bash, from the repository folder: - -```sh -node apps/cli/dist/invisible-dots.mjs setup --all -``` - -Run it as yourself, not as root or administrator: it asks for the rights it -needs once. It runs four steps in order, and each one skips what is already -done; you can still run any of them alone: - -| step | what it does | by hand | -|---|---|---| -| 1. The guest daemon | Builds `dot-agentd`, the program that runs inside every Dot's VM, for Linux whatever your host is. It stops here, before changing anything, when Go is missing, and prints the command that installs it. | `go -C guest/dot-agentd build -trimpath -o bin/dot-agentd ./cmd/dot-agentd` with `CGO_ENABLED=0 GOOS=linux GOARCH=amd64` | -| 2. QEMU and its accelerator | Checks the host and fixes only what is missing. On **Windows** it enables the Windows Hypervisor Platform and installs QEMU in one elevated step (one UAC prompt). On **Linux** it installs QEMU with `sudo apt-get` (sudo asks for your password). | `invisible-dots setup` | -| 3. The web client | Builds it, unless it is built already. | `npm run build --workspace @invisible-dots/web` | -| 4. The guest images | Builds the runtime disk (the daemon and the engine), and downloads the golden image (Ubuntu 24.04, the desktop, the browser) that CI built for exactly these inputs, or builds it here when none is published (a changed pin, a fork, no network). A second run with unchanged inputs does nothing. | `invisible-dots image build` (`--no-download` always builds) | - -Two things can stop it on purpose, and both end with the same advice: run the -same command again and it carries on. - -- On **Windows**, if enabling the accelerator needs a restart, it says so and - exits with code 5 after step 2. Restart, come back to this folder and run it - again. -- On **Linux**, if you are not in the `kvm` group yet it prints - `sudo usermod -aG kvm $USER`. Log out and in again, then run it again. - -Anything else that fails (a missing Go, a failed download, a build error) -stops the run with the reason and with nothing after that step run. `doctor` -prints one line per check and the command that fixes each failure; `setup ---all` ends by telling you what to do next. Ctrl+C stops the image build -cleanly, and a second Ctrl+C ends the command at once. - -### 3. Start the server and open the web UI - -```sh -node apps/cli/dist/invisible-dots.mjs server -``` - -It runs in the foreground until Ctrl+C, and serves the web client too. Open -http://127.0.0.2:3000: there is no login. The web client listens on 127.0.0.2, -a loopback address of your PC that no Dot's VM reaches (a VM reaches only -127.0.0.1, as 10.0.2.2), and adds the API token to each API call itself. - -On a computer that is ready except for the key, Home shows a field for your -OpenRouter key; then Create a Dot makes the first Dot. The web client is a -companion: if it was not built or its port is taken, the server says why and -the API and the command line carry on (`server --no-web` leaves it out). - -
-Your first Dot from the command line instead - -In a second terminal in the same folder: - -```sh -node apps/cli/dist/invisible-dots.mjs secret openrouter -node apps/cli/dist/invisible-dots.mjs init -node apps/cli/dist/invisible-dots.mjs create dot.yaml -node apps/cli/dist/invisible-dots.mjs message my-first-dot "List the files in your home folder and say how much disk is free." -node apps/cli/dist/invisible-dots.mjs logs my-first-dot -``` - -`secret openrouter` asks for the key (it is never a command-line argument, and -what you paste at the prompt is not shown on the screen). `init` writes -`dot.yaml`, the Dot's name, instructions, model, resources and permissions, to edit -before `create`. The first Dot takes a while to boot; the message waits in the -queue until it is ready. `logs` prints the Dot's events and keeps following new -ones until Ctrl+C (`--no-follow` prints and exits). It worked when `logs` shows -a `message.assistant` event with the answer and what it cost (`spent_usd`). - -In the rest of this page, `invisible-dots` stands for -`node apps/cli/dist/invisible-dots.mjs`, and `invisible-dots --help` lists every -command. In **bash**, `alias invisible-dots="node '$PWD/apps/cli/dist/invisible-dots.mjs'"` -in the repository folder makes it so (again in each new terminal); in -**PowerShell**, type `npx invisible-dots` from the repository folder. - -
- -## What to ask a Dot - -Anything that needs a computer and a person's judgement, done the way a person -would do it, for as long as it takes: - -> Open a browser identity called research, go to https://example.com and tell -> me the page's title and its first sentence. Leave the browser open. - -> Every morning, check one-way fares from Milan to Lisbon for the next two -> weeks, write them to ~/workspace/fares.csv and tell me the cheapest day. - -> Log in to the shop with the shopping identity, download this month's invoices -> to ~/documents, and remember where the invoices page is for next time. - -The first is the conversation in the picture above. For the second, the Dot sets -up an automation (it asks you first), and its computer is started each morning -for it and sleeps again afterwards. For the third, its prompt tells it to keep what it -learned as a note or a skill of its own, which the next prompt names. - -## What a Dot can do - -Its tools, each behind a permission -([architecture: tools](architecture.md#83-tools)): - -- **Run commands** on its own computer: a shell with a timeout, background - jobs, and programs on a pseudo-terminal that it reads as a screen of text. It - installs a missing Ubuntu package with `sudo dot-install `. -- **Read and write files**: read, list, find, grep, write, edit, apply a patch. -- **Use its browsers**: create, launch and close identities; navigate (http - and https only), read the page, take screenshots, click, type, select, - scroll, go back and forward. -- **Look at its desktop** with a screenshot. -- **Use the MCP servers you give it**: a program on its computer or a URL, each - one declared in its settings, with its own permission (asked by default). See - [MCP servers](#mcp-servers). -- **Schedule itself**: add, list and remove its own automations (at a time, - every interval, or a cron expression). This one asks you first by default; - to pause or remove one, ask the Dot. - -Work reaches it three ways: a message in its one persistent conversation, a -queued task (with a priority and an optional start time), or one of its own -automations firing. Each task has limits on steps, on context and on cost -(`max_cost_per_task_usd`). Long threads are summarized, by the Dot's model or -by another model you name (`models.summary`). - -**Memory and skills.** The Dot keeps its long-term memory itself, one note per -file in `/home/dot/memory`, and finds it again with its file tools; the newest -notes are named in its prompt. A skill says how to do a kind of task: a folder -with a `SKILL.md`, the layout Claude Code reads, whose name and one-line -description are in every prompt and whose steps the Dot reads before a task the -skill covers ([architecture: memory](architecture.md#86-memory)). The -built-in `invisible-playwright` skill teaches it the browser: one identity per -person it acts as, a selector before coordinates before a screenshot, the real -pointer and keyboard only, and saying a task is impossible rather than getting -blocked. The Dot writes skills of its own in `/home/dot/skills` as it learns, -and one of its own replaces a built-in one of the same name. - -> [!NOTE] -> Automations run inside the Dot's computer, and the control plane starts the -> computer for them. A Dot that is asleep is started shortly before an -> automation is due (90 seconds ahead by default), and it is not put to sleep -> while one is due, so a job every minute keeps it up. A run whose time passed -> while the computer or the server was down is made once, late, when the -> computer starts, never once for each occurrence it missed. A computer you -> stopped yourself (`invisible-dots computer stop`, or Stop in the web UI) -> stays off, and its automations are paused until you start it again; a message -> or a task still starts it. - -A Dot has no web search or fetch tool, no sub-agents and no tool to message -anyone: it reads the web through its own browser, and only the control plane -talks to chats. - -## The web UI - -The web UI is the same control plane as the command line, with a page for what -you would otherwise read as text. It is set in Geist (bundled with it), has a -light and a dark theme that follows your system unless you choose, and fits a -phone screen. - -**First run.** On a computer that is not ready for a Dot yet, Home shows a -setup checklist: each check of `doctor` that fails, with the command that fixes -it and a button that copies it, and a field for the OpenRouter key while none is -stored. The Settings page shows the same checks, takes a new key (and pushes it -to the Dots that are running), and has the theme. - -**Create a Dot** is a form in three steps (Identity, Brain, Computer and -safety) or the same configuration as YAML, and it refuses what the -configuration schema refuses before anything is sent. The permissions start -from one of three presets: - -| preset | what it does | -|---|---| -| Careful | Asks you before it runs a command or changes a file. | -| Balanced | The defaults: works freely on its own computer, and asks before it deletes a browser identity or adds an automation. | -| Autonomous | Never asks, except before it deletes a browser identity. | - -**The Inbox** is everything that needs you, from every Dot: the approvals that -wait (the one that has waited longest first), a Dot in an error state, a task -that failed in the last 24 hours, and a channel that has to be linked again. An -approval says what the Dot wants to do, under which permission and why, shows a -command, a diff of a file change, an address or a schedule in the form that -reads best, and answers with Allow once, Always allow or Deny with a note. With -the keyboard, once the list of approvals has the focus (click it or Tab to it), -`j` and `k` move between the cards, `a` allows the selected one once and `d` -denies it; a command or a deletion is not allowed by a key press alone. - -**A Dot's page** has seven tabs: - -- **Chat**: the one persistent conversation, with a quiet line for each tool the - Dot used to answer and approvals you can answer in place. The Dot's computer - is always beside it (above it on a narrow screen); both fill the window to - the bottom. -- **Tasks**: what is running (with its latest progress line and its cost), what - is scheduled and queued, and the history. -- **Computer**: Screen, Files and Usage. Screen is the Dot's desktop, where an - open browser is a window. Files is a read-only walk through `/home/dot`. Usage - shows what the computer was given and what it uses, the model spend, the - next automation, and Start, Reboot and Stop. -- **Skills**: the built-in skills and the Dot's own, read only. -- **Channels**: Telegram and WhatsApp, below. -- **Activity**: the whole event log as readable lines, filtered by kind, - searchable, and exportable as JSON Lines. -- **Settings**: the Dot's configuration, as a form or as YAML. Nothing is saved - until you have seen the list of what changes, and a configuration changed - somewhere else in the meantime is never overwritten. - -The desktop is shown as pictures that the control plane reads from the guest -every few seconds while the page is in view, with a LIVE badge and a warning -when a picture is more than 15 seconds old. It is a view and not a remote -desktop: it says "The Dot has control", because nothing you do in it reaches the -computer. Stop asks first, and says that the computer's automations do not run -while you have stopped it. - -## The command line - -| command | what it does | -|---|---| -| `invisible-dots list` | every Dot, with its state | -| `invisible-dots status ` | a Dot, its computer and its recent tasks | -| `invisible-dots message ` | a message in its one persistent conversation | -| `invisible-dots task [--priority N] [--at ISO-8601]` | a queued task; a higher priority runs first, `--at` sets the earliest start | -| `invisible-dots tasks ` | its tasks and their status | -| `invisible-dots cancel ` | stop a task that has not finished | -| `invisible-dots logs [--tail N] [--no-follow]` | its events, then new ones as they come | -| `invisible-dots approvals [--all]` | pending approvals (all of them, the newest first, with `--all`) | -| `invisible-dots approve [--always] [--note text]` | let that one call run; `--always` also allows its permission for the Dot from now on | -| `invisible-dots reject [--note text]` | refuse it; the model is told no | -| `invisible-dots computer start\|stop\|reboot` | power its computer on or off by hand | -| `invisible-dots browser identities` | its browser identities | -| `invisible-dots mcp ` | its MCP servers: where each is, and which secrets are set | -| `invisible-dots secret mcp --dot [--clear]` | set a secret of one of its MCP servers, asked for like the key | -| `invisible-dots channel list [--dot ]` | linked chats and who is paired | -| `invisible-dots channel remove telegram\|whatsapp --dot ` | unlink a chat; its token or keys and paired people are deleted | -| `invisible-dots secret openrouter --dot ` | a key for one Dot instead of the global one | -| `invisible-dots update ` | replace its configuration; refused if it changed since the command read it | -| `invisible-dots delete --yes` | delete it: its computer, its disk and its channels; without `--yes` it only says what would go | - -`` is a Dot's name or id, and `--json` gives machine-readable output. - -## Approvals - -- Every tool call goes through one policy gate. `allow` runs it, `deny` tells - the model no, `ask` stops the turn and records the call with its arguments - ([architecture: policy](architecture.md#84-policy)). -- The defaults are permissive for the Dot's own computer: running commands, - files and the browser are allowed, except deleting a browser identity, which - asks; automations ask; anything unknown is denied. Set any permission to `ask` - or `deny` in the YAML to be asked first or to forbid it. -- An approval can be answered in the web UI, the command line, a Telegram - button or a WhatsApp reply. The first answer wins, a second is ignored, and - the approved call runs at most once. `approve --always` turns that ask into - an allow for the Dot. -- Pending approvals survive a restart of the server or of the Dot. - -## The browser - -Each browser identity is a separate Firefox profile under -`/home/dot/browsers//`, with its own cookies, storage, logins and -fingerprint, the same fingerprint at every launch. It runs on the Dot's desktop, -so it appears in the Screen view and in screenshots of the desktop. The Dot -creates, opens and closes identities with its tools, and `invisible-dots -browser identities` lists them. - -A browser has no proxy of its own by default, and nothing asks for one: it -inherits the egress of the Dot's VM, and its time zone, language and location -follow the exit it actually uses. A proxy for one identity is an explicit -option, set when that identity is created, as the URL the browser library reads -(`http://user:pass@host:port` or `socks5://host:port`); the library, not this -project, judges it when the identity launches. - -A Dot can have a VM proxy: the whole computer of the Dot, browser, commands and -the engine's requests to OpenRouter included, then goes out through it. Set it -in the Dot's settings (VM proxy) or with -`invisible-dots secret proxy --dot ` (asked for like the key; `--clear` -removes it). It is a SOCKS5 URL, `socks5://user:password@host:port`, stored -encrypted and never shown again, and the Dot uses it from its next start. If the -proxy cannot be reached, nothing leaves the VM. When a VM proxy and an identity -proxy are both set, the identity's proxy is reached through the VM's tunnel. - -Launching is explicit: a page action on an identity that is not open fails, so -denying `browser.identity.launch` cannot be undone by navigating. At most 3 -identities are open at once (roughly 0.8 GB each), the least recently used one -closing first, and at most 20 exist. Screenshots go to the model and are never -stored. - -How well the browser holds up against bot checks is measured in -[invisible_playwright's README](https://github.com/feder-cr/invisible_playwright), -on the suites it names. invisible_dots adds no claim of its own: a site can -still refuse a Dot for what it does, and the terms of the sites you point it at -still apply. - -## MCP servers - -A Dot uses the tools of any MCP server you declare, the way Claude Code and -Codex do: in its settings (MCP servers), or in its YAML. - -```yaml -mcp_servers: - time: - command: uvx # runs on the Dot's computer, as the Dot's user - args: [mcp-server-time] - search: - url: https://search.example/mcp - secrets: [Authorization] # its value is set apart, never in the YAML -``` - -A server's tools are named `mcp__`, and one permission covers -them all, `mcp.`, which asks you before each call until you allow it -(or "Always allow" an approval). A secret, an API key or a token, is set in the -settings next to its server or with -`invisible-dots secret mcp --dot ` (asked for like the -key; `--clear` removes it): it reaches the server as an environment variable or -a header, never through the YAML or a command line. - -The Dot can add a server itself, as NanoClaw's agents can: it installs the -program (`uvx`, which is there, or from GitHub with `uv tool install -git+https://...`; for `npx` it runs `sudo dot-install nodejs npm`), then calls -`add_mcp_server`. That asks you first (`mcp.servers.add`): the approval shows -what the server runs or where it is, the secrets you will set, and why; approved, -the server is in the Dot's settings and its tools are the Dot's from its next -message, each asking you until you allow it. Only you change or remove a server. -A server that cannot start says why, in its settings, in `invisible-dots mcp ` -and to the Dot itself, and it is started again when the Dot's next message or task -begins. - -## Talk to it from your phone - -**Telegram.** Make a bot with @BotFather (one bot per Dot), then: - -```sh -invisible-dots channel add telegram --dot my-first-dot -invisible-dots channel pair telegram --dot my-first-dot -``` - -`channel add` asks for the bot's token and stores it encrypted. `channel pair` -prints a one-time link, valid ten minutes: open it in Telegram and press Start, -and that account is the Dot's owner from then on. Messages from anyone else are -dropped before they cost anything. Approvals arrive with Approve and Reject -buttons, the arguments cut to 300 characters. Telegram bot chats are not -end-to-end encrypted. Text only. - -A channel has three settings, all on by default: `approvals` (ask approvals in -the chat), `show_arguments` (show the tool's arguments in them) and -`notify_tasks` (what the Dot says on its own: finished and failed tasks, and -the answers to its automations). The Channels tab of the web UI has a switch -for each, and the API sets them too: - -```bash -curl -X PATCH http://127.0.0.1:8787/api/dots/my-first-dot/channels/telegram \ - -H "Authorization: Bearer $(head -1 ~/.invisible-dots/config/api.token)" \ - -H "Content-Type: application/json" \ - -d '{"settings": {"show_arguments": false}}' -``` - -**WhatsApp** is opt-in, and it needs a warning first. - -> [!WARNING] -> This is not an official way to use WhatsApp. It links the Dot as a device of -> a personal account through an unofficial client (Baileys, pinned to one -> release candidate), which WhatsApp's terms do not allow for automation and -> which can get the account restricted or banned. Use a number of its own, such -> as a spare SIM or eSIM, never the one you live on. - -Install the client once, from the repository folder, then start the server with -`INVISIBLE_DOTS_WHATSAPP=1`: - -```sh -npm run whatsapp:install -invisible-dots channel link whatsapp --dot my-first-dot -invisible-dots channel pair whatsapp --dot my-first-dot -``` - -`channel link` shows a QR code to scan under WhatsApp > Settings > Linked -devices; the linked device's keys are stored encrypted. `channel pair` prints a -link that opens a chat with the number and `pair ` ready to send. The Dot -never writes first, to anyone, answers only people who paired, in private -chats, and takes text only. Approvals are answered in words -(`yes ap-xxxxxx` or `no ap-xxxxxx`); a bare `yes` is an ordinary message. - -Baileys depends on `libsignal`, which is GPL-3.0, so nothing of it is installed -by default: `npm ci` installs neither, and neither is in the bundled command. -`npm run whatsapp:install` installs them into `optional/whatsapp/`, pinned to -one release by a lock file with the integrity of every package, and the server -loads them from there only when WhatsApp is linked -([THIRD_PARTY_NOTICES.md](../THIRD_PARTY_NOTICES.md), -[architecture: messaging channels](architecture.md#98-messaging-channels)). - -## How it works - -The web UI and the command line call the HTTP API; the scheduler behind it keeps -the queue in a database and starts and stops one QEMU VM per Dot; inside the VM, -the engine reasons with OpenRouter, runs the model's commands through the -computer daemon and drives its browsers on the VM's desktop; Telegram and -WhatsApp reach the scheduler through the channel hub. - -```mermaid -flowchart LR - subgraph PC["Your PC: one invisible-dots server process"] - WEB["Web UI
127.0.0.2:3000"] - CLI["CLI"] - API["HTTP API and SSE
127.0.0.1:8787"] - SCH["Scheduler
queue, outbox, approvals, sleep and wake"] - HUB["Channel hub"] - VMM["vm-manager
the QEMU driver"] - DB[("PostgreSQL
embedded PGlite by default")] - end - subgraph VM["One QEMU VM per Dot"] - AGD["dot-agentd
the computer daemon, user dotagentd"] - ENG["Engine, a nanobot fork
user dotengine, key in memory"] - DESK["Xvfb and XFCE desktop"] - BR["invisible-playwright-mcp
one per open identity"] - end - WEB -->|"API calls with the token"| API - CLI -->|"API calls with the token"| API - API -->|"messages, tasks, answers"| SCH - HUB -->|"chat messages and approvals"| SCH - SCH -->|"queue, events, approvals"| DB - SCH -->|"start, stop, reboot"| VMM - VMM -->|"starts and stops"| VM - SCH -->|"loopback port forward
token after an HMAC proof"| AGD - AGD <-->|"proxy, and the model's commands"| ENG - ENG -->|"browser tools over MCP"| BR - BR -->|"windows on"| DESK - ENG -->|"model requests"| OR["OpenRouter"] - HUB <-->|"bot messages"| TG["Telegram"] - HUB <-->|"linked device"| WA["WhatsApp, opt-in"] - BR -->|"pages, from the VM's egress"| SITES["The web"] -``` - -- **One process on the host.** `invisible-dots server` runs the API, the - scheduler, the VM driver and the channel hub, and starts the web client as a - child (`--no-web` leaves it out). The database is an embedded PostgreSQL - (PGlite) in the data directory, or an external PostgreSQL 16 or newer through - `DATABASE_URL`. -- **One VM per Dot.** QEMU with KVM or the Windows Hypervisor Platform; never - software emulation. A copy-on-write disk over a golden image, plus a - read-only runtime disk with our code. Both images are built from pinned, - hashed inputs - ([architecture: two images](architecture.md#33-two-images-two-lifetimes)). -- **The engine** is a hard fork of [nanobot](https://github.com/HKUDS/nanobot) - in Python, cut down to its core: the turn runner, OpenRouter, the tools above, - automations, skills, and an MCP client that serves only the browser - ([invisible_engine_dots/UPSTREAM.md](../invisible_engine_dots/UPSTREAM.md)). -- **Durable on both sides.** The host writes what it tells a guest in the - same transaction as the decision, so a restart loses nothing you saw - accepted ([architecture: durable queue](architecture.md#92-durable-queue)). - The engine keeps its state in one SQLite file and commits each step with the - events that describe it; its event stream keeps its order across a - `kill -9`. A task cut short by a crash is started again with a note that the - last attempt was interrupted, and fails once it has been started three times - ([architecture: crash recovery](architecture.md#87-crash-recovery)). -- **The server can restart without stopping Dots.** QEMU runs detached, and a - new server adopts the VMs it finds running. A VM that dies while its Dot has - work is started again. - -The full contract every part is written against: -[docs/architecture.md](architecture.md). - -## Security model and known limits - -What protects you: - -- The OpenRouter key is never written to the image, the disk or the seed. It - is pushed to the Dot after every start and held in the engine's memory only - ([architecture: secrets](architecture.md#43-secrets)). -- The engine and the computer daemon each run as a user of their own, and the - model's commands as a third, `dot`, which can reach neither: it cannot read - the engine's state or the Dot's token, cannot connect to the engine's socket - (so it cannot change the Dot's permissions or approve its own calls), cannot - stop either daemon and cannot become root - ([architecture: processes](architecture.md#41-processes-systemd-each-unit-as-the-user-it-names)). - The one thing it may do as root is install Ubuntu packages it is missing, - `sudo dot-install ...`: the command takes package names and nothing - else, so no option reaches apt; the packages' own install scripts run as root, - and they come from the Ubuntu archive. -- The channel between host and guest goes one way: the host calls the guest - on a loopback port, with the Dot's token, and only after the guest proves it - holds that token. The API needs a bearer token (the command line reads it - from `config/api.token` itself); the web UI needs none and listens on - 127.0.0.2, which no Dot's VM reaches. -- Files the host shows from a Dot come only from `/home/dot`, at most 16 MiB, - never served as a type a browser would run. -- The proxy of an identity, when it has one, is not shown by the host: an - identity says only that it has one, an approval shows it as `***`, and no - event carries it. -- Secrets on the host (the OpenRouter key, channel tokens, WhatsApp keys) are - stored encrypted; see [Configuration](#configuration). - -What it does not protect against, by design or not yet: - -- The model's commands run as `dot`, which can read the proxy of an identity - that was given one, password included, from the open browser's environment - and from the browser server's own session file, and what the browser server - says of a proxy it refuses reaches the model and the engine's log as it is. - An identity with no proxy, the default, has nothing to read. -- The split between the three users is the guest operating system's. A flaw in - the guest's kernel, or a way to become root that this repository does not - know of, would end it. The VM is what stands between the Dot and your PC - then, with the reach into your PC's loopback listed below. -- The computer daemon holds three capabilities (`CAP_SETUID`, `CAP_SETGID` and - `CAP_KILL`, to run the model's work as `dot`) and may run one command as - root, the power-off. A flaw in the daemon would be worth more than one in an - ordinary program. Its one door, a TCP port inside the guest, needs the Dot's - token, which `dot` cannot read. -- Nothing limits how much disk, memory or processor time the model's commands - use of their own VM, so a runaway command can starve the Dot's own daemons - until the VM is stopped or restarted. -- When typing in the browser is set to `ask`, the approval shows the text being - typed, password or not. -- Each Dot can reach services on your PC's loopback through `10.0.2.2`. - Everything invisible_dots listens on there needs a credential the Dot lacks, - but nothing else should listen there unauthenticated - ([architecture: networking](architecture.md#36-networking-and-its-limits), - [architecture: browser identities](architecture.md#6-browser-identities)). - -## Privacy - -invisible_dots runs on your machine and has no server of its own. What leaves -it, and to whom: - -- **OpenRouter** gets the conversation, the tool results and what the Dot - reads, under your key. Requests carry the headers - `HTTP-Referer: https://github.com/feder-cr/invisible_dots` and `X-Title: invisible_dots`. -- **Telegram and WhatsApp** carry the messages and approval prompts of a - linked channel, and can read them. Link previews are off, so neither - service fetches a link of a message on its own; a person who taps a link - opens it themselves. -- **The sites a Dot visits** see its browser, coming from the egress of the Dot's - VM, or from the identity's proxy when you gave that one identity a proxy. -- **Publishers of what the images are made of** (Ubuntu, uv, the Python - packages, the browser engine) serve the downloads when you build them; on - Windows, `setup` downloads the official QEMU installer. -- **The browser layer.** - [invisible-playwright-mcp's privacy policy](https://github.com/feder-cr/invisible_playwright_mcp#privacy-policy) - says each browser launch fetches a one-line counter file from a GitHub - release, carrying no identifier. -- **Address-echo services** (`api.ipify.org`, `icanhazip.com`, - `checkip.amazonaws.com`) are asked by the browser library at each launch for - the public address the browser exits from, to set the time zone and locale, - and it keeps its GeoIP database current from its GitHub release. -- **The web UI** loads nothing from other sites: its pages, scripts and styles - come from the server on your PC, its fonts (Geist) included. A link you - press (to @BotFather, to an OpenRouter page) opens that site. -- **No telemetry of invisible_dots.** Nothing of its own reports on how you use - it: Next.js's build telemetry is off in the web client's own config, so no - way of building it sends any, and the Dots' Ubuntu has no crash reporter and - no pollinate. The browser layer's launch counter, above, is that library's. - -Your Dots' disks, memory and conversations stay in the data directory and -inside their VMs. - -## Configuration - -A Dot is one YAML file, checked by one schema: name, instructions, -model, computer (cpu, memory, disk, idle timeout), permissions, limits and MCP -servers. Every field and its range is in -[architecture: Dot configuration](architecture.md#7-dot-configuration); -`invisible-dots init` writes a sample. - -One server runs per data directory. Secrets (the OpenRouter key, channel -tokens, WhatsApp keys, MCP servers' secrets) are stored with AES-256-GCM under -`config/master.key`; anyone who can read the data directory can decrypt them, -so it is kept private to your user. - -
-The environment variables the host reads - -| variable | default | what | -|---|---|---| -| `INVISIBLE_DOTS_HOME` | `~/.invisible-dots` | the data directory: keys, database, images, VM disks, logs. Plain ASCII, no comma | -| `INVISIBLE_DOTS_LISTEN` | `127.0.0.1:8787` | where the server's API listens | -| `INVISIBLE_DOTS_URL` | `http://127.0.0.1:8787` | where the CLI finds the server | -| `INVISIBLE_DOTS_TOKEN` | first line of `config/api.token` | the API token, instead of the file | -| `INVISIBLE_DOTS_WEB_LISTEN` | `127.0.0.2:3000` | where the web client listens; keep it off 127.0.0.1, which every Dot's VM reaches | -| `INVISIBLE_DOTS_WEB_ALLOWED_HOSTS` | loopback only | other host names the web client answers to | -| `INVISIBLE_DOTS_QEMU_DIR` | official installer's folder, then `PATH` | the one folder QEMU is looked for in | -| `INVISIBLE_DOTS_WHATSAPP` | off | `1` offers the WhatsApp channel (after `npm run whatsapp:install`) | -| `DATABASE_URL` | embedded PGlite | an external PostgreSQL 16 or newer | -| `INVISIBLE_DOTS_DEBUG` | off | `1` adds debug lines to the server's log | - -
- -## Troubleshooting - -- `invisible-dots doctor` first: one line per check, and the command that fixes - a failure. -- The server logs to the terminal it runs in; `INVISIBLE_DOTS_DEBUG=1` adds - debug lines. -- A Dot that does not start: QEMU's own output is in - `~/.invisible-dots/logs/qemu-.log`, and the guest's serial console in - `~/.invisible-dots/vms//serial.log` (rewritten at each start). - `invisible-dots list` shows each Dot's id. -- What a Dot did and why it stopped: `invisible-dots logs ` and - `invisible-dots tasks `, or the Dot's Activity tab in the web UI. -- The web UI does not open: `invisible-dots server` says why it did not start - it (not built, port taken) in its output. Its address is - http://127.0.0.2:3000, not 127.0.0.1. - -## Updating and uninstalling - -**Updating.** Stop the server (Ctrl+C), then in the repository folder: -`git pull`, `npm ci`, the builds of the command line -(`npm run build --workspace @invisible-dots/cli`), of the web client -(`npm run build --workspace @invisible-dots/web`) and of dot-agentd (step 1 of -`setup --all` above), `invisible-dots image build`, and `invisible-dots server` -again. Running Dots keep -running meanwhile: QEMU is not a child of the server. Each VM start takes the -newest runtime image (our code), so `invisible-dots computer reboot` -moves a Dot to it. A Dot keeps the golden image it was created on, and new Dots -get the newest one; this version has no way to move a Dot to a new golden image. - -**Uninstalling.** Stop each Dot's computer (`invisible-dots computer -stop`) while the server runs, since a VM outlives the server, then stop the -server and delete the data directory (`~/.invisible-dots`, or your -`INVISIBLE_DOTS_HOME`) and the repository folder. What `setup` installed stays -until you remove it yourself: QEMU, the Windows Hypervisor Platform feature on -Windows, and your membership of the `kvm` group on Linux. - -## Status and what is not done yet - -Alpha. Nothing is released yet; the golden image is the one thing published. - -- **The real-VM acceptance run is not in CI.** It - (`tests/e2e/run.ts`: build, create, browse, approve, kill the VM, restart, - scan the disk for the key) passes its 16 steps on a Linux host with KVM, run - by hand; CI keeps its contract in step with the product and tests the guest - with the container smokes, which run its daemons without a VM. -- **Windows is measured by hand, not by an automated run**: Dots on WHPX were - driven through the web client (chat, tasks, approvals, files, browser - identities, sleep and wake, reboot). It runs the same code path as Linux, and - every difference between the two is listed in - [architecture: one mechanism on every host](architecture.md#11-one-mechanism-on-every-host). -- **The install is one command after the checkout, not before it.** Node, Go - and Git, the clone, `npm ci` and the build of the command line itself are - still typed by hand, because the command needs them to exist before it can - run. -- **Channels carry text only**, one Telegram bot per Dot; the official - WhatsApp Cloud API is a later adapter on the same hub. -- **Not in this version**: snapshots and rollback, backups, quotas, network - policies, controlling the desktop or an - interactive terminal for you (the web UI shows the desktop as pictures, and - nothing you do there reaches the computer), artifacts, several hosts, - organisations and roles, macOS hosts - ([architecture: out of scope](architecture.md#10-out-of-scope-for-this-version)). - -## Development and tests - -Besides the quickstart's tools, development needs Python 3.11 or newer for the -engine, and Docker for the smokes. - -```sh -npm ci -npm run typecheck -npm test # vitest over the TypeScript workspace and tests/repo -git config core.hooksPath .githooks # the pre-push gate: typecheck, vitest, go test, prose checks -``` - -The pre-push gate's prose checks need Python with the pinned invisible-core. The -hook looks for it in `tmp/gates-venv` first: -`python -m venv tmp/gates-venv`, then -`tmp/gates-venv/bin/python -m pip install -r .github/gates-requirements.txt` -(on Windows the interpreter is `tmp\gates-venv\Scripts\python.exe`). - -- The engine: `pip install -e ".[dev]"` and `pytest` in - `invisible_engine_dots/` (Linux only: it uses unix sockets). -- dot-agentd: `go test ./...` in `guest/dot-agentd/`. -- The web client's browser tests: build it (`npm run build --workspace - @invisible-dots/web`), install the browser they drive (`npx playwright install - chromium`) and run `npm run test:e2e --workspace @invisible-dots/web`. Each run - starts the real control plane over a fake VM layer and the built web client as - the product runs it. -- The engine smoke and the browser smoke run the guest's two daemons, and the - real invisible-playwright-mcp with its Firefox, in Linux containers: - `bash guest/image-builder/test/smoke/run.sh` (add `--suite browser`); only - Docker is needed - ([the smoke tests README](../guest/image-builder/test/smoke/README.md)). -- The acceptance run on a real VM, `node tests/e2e/run.ts`, needs a Linux host - with KVM and an OpenRouter key, and is not part of CI - ([the acceptance run README](../tests/e2e/README.md)). - -
-What CI runs, and where the code lives - -CI runs on pushes to main and on pull requests: typecheck and vitest on Linux -and Windows, the database layer against a real PostgreSQL, dot-agentd's Go -tests and static build, the engine's pytest, the ISO images read by the Linux -kernel, both smokes, the web build with its Playwright tests, the opt-in -WhatsApp client against its real library, and two prose -gates (English only, and a README that promises nothing it cannot support). The -vitest, Go, pytest and Playwright runs are counted against the floors in -`.github/test-floors.json`, and the smokes fail on a skipped check, so a test -that stops running fails the build. - -| folder | what | -|---|---| -| `apps/` | `api`, `scheduler`, `vm-manager`, `web`, `cli` | -| `packages/` | shared types and schemas, database, events, channels, the ISO writer, the SDK | -| `guest/` | `dot-agentd` (Go) and the image builder | -| `invisible_engine_dots/` | the Dot's engine (Python) and its built-in skills | -| `virtualization/` | the pinned QEMU, cloud-init templates, base image metadata | -| `tests/` | checks over the whole repository, the real-VM run, and the agent benchmark | - -
- -## Acknowledgements - -- [nanobot](https://github.com/HKUDS/nanobot), the agent the Dot's engine is a - fork of. -- [invisible_playwright](https://github.com/feder-cr/invisible_playwright) and - [invisible-playwright-mcp](https://github.com/feder-cr/invisible_playwright_mcp), - the Dot's browser. -- [QEMU](https://www.qemu.org), [Ubuntu cloud images](https://cloud-images.ubuntu.com), - [PGlite](https://pglite.dev), [Next.js](https://nextjs.org) and - [Baileys](https://github.com/WhiskeySockets/Baileys). -- The Agent Skills layout of [Claude Code](https://github.com/anthropics/claude-code), - which the Dot's skills follow. - -## License - -MIT ([LICENSE](../LICENSE)). The engine under `invisible_engine_dots/` is a fork -of nanobot, also MIT, under its own [LICENSE](../invisible_engine_dots/LICENSE); -parts of this repository's history come from Open Multi-Agent, also MIT. Their -notices are in [THIRD_PARTY_NOTICES.md](../THIRD_PARTY_NOTICES.md). QEMU (GPL-2.0) -is installed from its official installer or your distribution and only run as -a separate program, never bundled. The guest operating system, the browser -engine and the packages in the golden image keep their own licenses; the -published golden image is Ubuntu with those packages, and whoever copies it -takes on those licenses -([architecture: licensing](architecture.md#113-licensing)). Nothing under -the GPL is installed by the default `npm install`; the WhatsApp client, which -brings a GPL-3.0 dependency, is an opt-in install. - -invisible_dots is an independent project, not affiliated with OpenRouter, -Telegram, WhatsApp or Meta. diff --git a/docs/hero/README.md b/docs/hero/README.md deleted file mode 100644 index aa8ba90a..00000000 --- a/docs/hero/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# The README animation - -The six `docs/images/hero-*.gif` are built from this folder, in three layers, each -made from the one before: - -| layer | where | what it is | -|---|---|---| -| photos | `photos/` | the pictures the animation shows | -| frames | `html/` | one page per layout that composes the photos; `renderLoop(t)` draws the frame at second `t` | -| GIFs | `docs/images/` | the frames, 12 a second, one palette per loop | - -The repository's social preview, `docs/images/social-preview.png`, comes from the -same photos through one more page, `html/social.html`, as a single frame. - -```sh -node docs/hero/build.mjs # all six GIFs and the social preview -node docs/hero/build.mjs wide # only hero-light.gif and hero-dark.gif -node docs/hero/build.mjs social # only the social preview -``` - -The social preview is not read from the tree: after a build it is uploaded by hand, -in the repository's Settings, under Social preview. - -It needs the repository's `node_modules` (Playwright comes with `apps/web`), Python -with Pillow, NumPy and SciPy, and `ffmpeg` on `PATH`. The pages use Segoe UI and -Consolas, so the GIFs come out identical only on Windows. The frames land in -`docs/hero/frames/`, which is not tracked. - -## The photos - -| file | what it is | edit it? | -|---|---|---| -| `photos/characters.png` | the four characters on a white background | yes: this is the one to replace | -| `photos/characters-light.png` | the same, cut out for the light theme, shadows kept | no, `cut_out.py` writes it | -| `photos/characters-dark.png` | the same, cut out for the dark theme, shadows removed | no, `cut_out.py` writes it | -| `photos/trending.png` | the GitHub Trending page in the last scene's third card | yes | - -New characters are a new `characters.png`, drawn on plain white, all four in one -row and about the same height. The build cuts them out again on every run. - -## The pages - -| page | builds | size | -|---|---|---| -| `html/wide.html` | `hero-light.gif`, `hero-dark.gif` | 1280 x 470 | -| `html/mobile.html` | `hero-mobile-light.gif`, `hero-mobile-dark.gif` | 720 x 620 | -| `html/small.html` | `hero-small-light.gif`, `hero-small-dark.gif` | 720 x 620 | -| `html/social.html` | `social-preview.png` | 1280 x 640 | - -The three animated pages read `?theme=light` or `?theme=dark` and, opened in a browser, play -the whole loop as the GIF will: the scenes, their words and their timing are all -in the page. With `&frames` they stay still and wait for `renderLoop(t)`, which is -how the build takes one frame at a time. A change to a scene is a change to the -three pages, then a build. - -The README picks the GIF by the reader's width and theme, in its `` at -the top. diff --git a/docs/hero/build.mjs b/docs/hero/build.mjs deleted file mode 100644 index f2a2a730..00000000 --- a/docs/hero/build.mjs +++ /dev/null @@ -1,97 +0,0 @@ -// The README animation, rebuilt from its pieces: photos -> frames -> GIFs. -// -// 1. photos cut_out.py turns photos/characters.png into one cut-out per theme -// 2. frames each page in html/ composes the photos; renderLoop(t) draws the -// frame at second t, and Chromium takes a screenshot of it, 12 a second -// 3. GIFs ffmpeg turns each run of frames into docs/images/hero-*.gif -// -// html/social.html is a page of one frame: the repository's social preview, written -// to docs/images/social-preview.png. -// -// node docs/hero/build.mjs everything -// node docs/hero/build.mjs wide social only those pieces (wide, mobile, small, social) -// -// Needs the repository's node_modules (Playwright comes with apps/web), Python with -// Pillow, NumPy and SciPy, and ffmpeg on PATH. The pages use Segoe UI and Consolas, -// so the GIFs come out identical only on Windows. -import { spawnSync } from "node:child_process"; -import { mkdirSync, rmSync } from "node:fs"; -import { createRequire } from "node:module"; -import path from "node:path"; -import { fileURLToPath, pathToFileURL } from "node:url"; - -const HERE = path.dirname(fileURLToPath(import.meta.url)); -const ROOT = path.resolve(HERE, "..", ".."); -const { chromium } = createRequire(path.join(ROOT, "package.json"))("@playwright/test"); - -const FPS = 12; -const LAYOUTS = { - wide: { page: "wide.html", width: 1280, height: 470, gif: "hero" }, - mobile: { page: "mobile.html", width: 720, height: 620, gif: "hero-mobile" }, - small: { page: "small.html", width: 720, height: 620, gif: "hero-small" }, -}; -const THEMES = ["light", "dark"]; -// Pictures of one frame. The social preview is uploaded by hand in the repository's -// Settings, so it is written where it can be found for that. -const STILLS = { - social: { page: "social.html", width: 1280, height: 640, out: "social-preview.png" }, -}; - -function run(cmd, args) { - const r = spawnSync(cmd, args, { stdio: "inherit" }); - if (r.status !== 0) throw new Error(`${cmd} failed (${r.error ? r.error.message : `exit ${r.status}`})`); -} - -const wanted = process.argv.slice(2); -const known = [...Object.keys(LAYOUTS), ...Object.keys(STILLS)]; -for (const name of wanted) if (!known.includes(name)) throw new Error(`unknown piece ${name}: ${known.join(", ")}`); -const layouts = (wanted.length ? wanted : Object.keys(LAYOUTS)).filter((n) => LAYOUTS[n]); -const stills = (wanted.length ? wanted : Object.keys(STILLS)).filter((n) => STILLS[n]); - -// 1. photos -run(process.platform === "win32" ? "python" : "python3", [path.join(HERE, "cut_out.py")]); - -const browser = await chromium.launch(); -try { - for (const name of layouts) { - const { page: file, width, height, gif } = LAYOUTS[name]; - for (const theme of THEMES) { - // 2. frames - const dir = path.join(HERE, "frames", `${name}-${theme}`); - rmSync(dir, { recursive: true, force: true }); - mkdirSync(dir, { recursive: true }); - const page = await browser.newPage({ viewport: { width, height }, deviceScaleFactor: 1 }); - // ?frames stops the page's own playback: every frame is the one renderLoop(t) drew - await page.goto(`${pathToFileURL(path.join(HERE, "html", file)).href}?theme=${theme}&frames`); - await page.waitForLoadState("load"); - const seconds = await page.evaluate(() => window.LOOP_SECONDS); - const count = Math.round(seconds * FPS); - for (let i = 0; i < count; i++) { - await page.evaluate((t) => window.renderLoop(t), i / FPS); - await page.screenshot({ path: path.join(dir, `f${String(i).padStart(5, "0")}.png`) }); - } - await page.close(); - - // 3. GIF: one palette for the whole loop, built from every frame - const out = path.join(ROOT, "docs", "images", `${gif}-${theme}.gif`); - run("ffmpeg", [ - "-v", "error", "-y", "-framerate", String(FPS), "-i", path.join(dir, "f%05d.png"), - "-vf", "split[a][b];[a]palettegen=max_colors=256:stats_mode=full[p];[b][p]paletteuse=dither=sierra2_4a:diff_mode=rectangle", - "-loop", "0", out, - ]); - console.log(`${path.relative(ROOT, out)}: ${count} frames, ${seconds}s`); - } - } - for (const name of stills) { - const { page: file, width, height, out: picture } = STILLS[name]; - const page = await browser.newPage({ viewport: { width, height }, deviceScaleFactor: 1 }); - await page.goto(pathToFileURL(path.join(HERE, "html", file)).href); - await page.waitForLoadState("load"); - const out = path.join(ROOT, "docs", "images", picture); - await page.screenshot({ path: out }); - await page.close(); - console.log(`${path.relative(ROOT, out)}: ${width}x${height}`); - } -} finally { - await browser.close(); -} diff --git a/docs/hero/cut_out.py b/docs/hero/cut_out.py deleted file mode 100644 index 65f2b16c..00000000 --- a/docs/hero/cut_out.py +++ /dev/null @@ -1,88 +0,0 @@ -"""photos/characters.png -> photos/characters-light.png and photos/characters-dark.png. - -The four characters arrive as one picture on a white background, the way an image -model draws them. The animation needs them without it, and differently per theme: - - light only the white connected to the picture's edge turns transparent, through - the edge's own shades, so the soft shadows under the feet stay and the - fur fades into the page the way it faded into the white; - dark the same, and the pale grey of those shadows goes too, since a light - shadow on a dark page reads as a stain. - -Only background CONNECTED TO THE EDGE is touched: the whites of the eyes and a lens -are as white as the background and stay opaque. Both outputs are cropped to the -characters, so the pages size them by width alone. - - python docs/hero/cut_out.py -""" -from pathlib import Path - -import numpy as np -from PIL import Image -from scipy import ndimage - -HERE = Path(__file__).parent -SRC = HERE / "photos" / "characters.png" - -#: How far from pure white a pixel may be and still be background (0-255, on the -#: channel furthest from white). Covers the paper-white of the picture and its -#: anti-aliased edge; fur is far past it. -PALE = 60 -#: A pale pixel with less colour than this is a shadow, not fur fading into white. -GREY = 18 -#: How many pixels past the background the fur still carries some of its white. -BAND = 3 -#: How far from pure white the paper may be: within this it is no picture at all. -PAPER = 6 - - -def cut(rgb: np.ndarray, keep_shadows: bool) -> np.ndarray: - a = rgb.astype(np.float64) - far = (255.0 - a).max(axis=2) # 0 = pure white - spread = a.max(axis=2) - a.min(axis=2) # 0 = no colour at all - labels, _ = ndimage.label(far <= PALE) - edge = np.unique(np.concatenate([labels[0], labels[-1], labels[:, 0], labels[:, -1]])) - bg = np.isin(labels, edge[edge > 0]) - # The fur's outermost pixels are part fur, part white, and past PALE: left - # opaque they draw a thin light outline on a dark page. They are the band just - # outside the background, and colour to alpha is exact for them too. - band = ndimage.binary_dilation(bg, iterations=BAND) - - # Colour to alpha against white: the least alpha that explains the pixel as - # some colour over white, and that colour with the white taken out. - alpha = np.where(band, far / 255.0, 1.0) - # The paper itself is not pure white (an image model draws it at 253-254), and - # colour to alpha turns that into an opacity of 1 or 2 out of 255 with made-up - # colours: a faint box around the characters on any page that is not white. - alpha = np.where(bg & (far <= PAPER), 0.0, alpha) - if not keep_shadows: - alpha = np.where(bg & (spread < GREY), 0.0, alpha) - safe = np.maximum(alpha, 1e-6)[..., None] - colour = np.where(band[..., None], 255.0 - (255.0 - a) / safe, a) - out = np.dstack([np.clip(colour, 0, 255), alpha * 255.0]).round().astype(np.uint8) - out[out[..., 3] == 0, :3] = 0 - return out - - -def box(rgba: np.ndarray, margin: int = 8) -> tuple: - """The characters' bounding box. ONE box for both themes: cropped apart, the - light one (shadows included) would be taller, and at the same width on the - page the characters would change size between the two themes.""" - ys, xs = np.nonzero(rgba[..., 3] > 8) - return (max(0, ys.min() - margin), min(rgba.shape[0], ys.max() + margin + 1), - max(0, xs.min() - margin), min(rgba.shape[1], xs.max() + margin + 1)) - - -def main() -> None: - rgb = np.asarray(Image.open(SRC).convert("RGB")) - cuts = {"light": cut(rgb, keep_shadows=True), "dark": cut(rgb, keep_shadows=False)} - y0, y1, x0, x1 = box(cuts["light"]) - for theme, rgba in cuts.items(): - out = rgba[y0:y1, x0:x1] - path = HERE / "photos" / f"characters-{theme}.png" - Image.fromarray(out, "RGBA").save(path, optimize=True) - print(f"{path.name}: {out.shape[1]}x{out.shape[0]}") - - -if __name__ == "__main__": - main() diff --git a/docs/hero/html/mobile.html b/docs/hero/html/mobile.html deleted file mode 100644 index ec4dc480..00000000 --- a/docs/hero/html/mobile.html +++ /dev/null @@ -1,225 +0,0 @@ - - - - - - - - - -
-
-
Thinking...
-
-
- -
-
its computer · Firefox
-
-
-
built to be undetectable by anti-bots
- -
-
Add an automation?
-
Monthly: the shop's invoices, the total to Telegram
-
Allow
Reject
-
-
- -
- - - - - -
-
Terminal
-
~/documents
14 PDFs
-
Firefox
Shop · ⬇ Invoices
-
- -
~/skills/invoices.md
-
~/memory · the total, not the list
- -
invoices
the shop's invoices, monthly
own VM
-
fare-watch
cheapest flight, every morning
own VM
-
researcher
GitHub Trending → Telegram
own VM
- - - - - - diff --git a/docs/hero/html/small.html b/docs/hero/html/small.html deleted file mode 100644 index a9241bc1..00000000 --- a/docs/hero/html/small.html +++ /dev/null @@ -1,226 +0,0 @@ - - - - - - - - - -
-
-
Thinking...
-
-
- -
-
its computer · Firefox
-
-
-
built to be undetectable by anti-bots
- -
-
Add an automation?
-
Monthly: the shop's invoices, the total to Telegram
-
Allow
Reject
-
-
- -
- - - - - -
-
Terminal
-
~/documents
14 PDFs
-
Firefox
⬇ Invoices
-
- -
~/skills/invoices.md
-
~/memory · the total, not the list
- -
invoices
the shop's invoices, monthly
own VM
-
fare-watch
cheapest flight, every morning
own VM
-
researcher
GitHub Trending → Telegram
own VM
- - - - - - diff --git a/docs/hero/html/social.html b/docs/hero/html/social.html deleted file mode 100644 index fb083392..00000000 --- a/docs/hero/html/social.html +++ /dev/null @@ -1,30 +0,0 @@ - - - - -invisible_dots social preview - - - -
-
invisible_dots
-
Open-source, self-hosted alternative to OpenAI Dots, Manus, Claude Cowork & co.
-
AI agents with a computer of their own. Built to be undetectable by anti-bots.
- - - diff --git a/docs/hero/html/wide.html b/docs/hero/html/wide.html deleted file mode 100644 index 98cacb4b..00000000 --- a/docs/hero/html/wide.html +++ /dev/null @@ -1,288 +0,0 @@ - - - - - - - -
 invisible_dots
-
- -
-
I
invoices
-
-
I
Thinking
-
-
- -
-
 researcher's computer · Firefox
-
-
-
built to be undetectable by anti-bots
- -
-
invoices asks
-
Add an automation
-
On the 1st of every month, 09:00
Download the shop's invoices, send the total to Telegram
-
Allow once
Always allow
Reject
-
-
- -
-
I
invoices
-
08:00
-
-
- - - - - -
-
Applications  ·  invoices' computer
-
~/documents/invoices/2026-10
-
0412.pdf
0413.pdf
0414.pdf
-
Terminal
-
Firefox · identity "shop"
Ceramiche Rossi · Admin
Invoices · October 2026
#0412 · €84.00⬇ PDF
#0413 · €126.50⬇ PDF
#0414 · €310.00⬇ PDF
-
- -
~/skills/shop-invoices.md
-
~/memory · Federico wants the total, not the list.
- -
-
-
I
invoices
wakes on the 1st
-
~/documents
📄 0412.pdf
📄 0413.pdf
-
Monthly
the shop's invoices → Telegram
-
-
F
fare-watch
running · its own VM
-
$ fares.py
07/10 €38
09/10 €31
-
Every morning
cheapest flight, Milan → Lisbon
-
-
R
researcher
running · its own VM
-
-
Every day 08:00
GitHub Trending → Telegram
-
- - - - - - diff --git a/docs/hero/photos/characters-dark.png b/docs/hero/photos/characters-dark.png deleted file mode 100644 index c3ea9fbb..00000000 Binary files a/docs/hero/photos/characters-dark.png and /dev/null differ diff --git a/docs/hero/photos/characters-light.png b/docs/hero/photos/characters-light.png deleted file mode 100644 index 37acb087..00000000 Binary files a/docs/hero/photos/characters-light.png and /dev/null differ diff --git a/docs/hero/photos/characters.png b/docs/hero/photos/characters.png deleted file mode 100644 index 21f19d37..00000000 Binary files a/docs/hero/photos/characters.png and /dev/null differ diff --git a/docs/hero/photos/trending.png b/docs/hero/photos/trending.png deleted file mode 100644 index 53e851e4..00000000 Binary files a/docs/hero/photos/trending.png and /dev/null differ diff --git a/docs/images/hero-dark.gif b/docs/images/hero-dark.gif deleted file mode 100644 index 5d2d55cf..00000000 Binary files a/docs/images/hero-dark.gif and /dev/null differ diff --git a/docs/images/hero-light.gif b/docs/images/hero-light.gif deleted file mode 100644 index 74fbe2b8..00000000 Binary files a/docs/images/hero-light.gif and /dev/null differ diff --git a/docs/images/hero-mobile-dark.gif b/docs/images/hero-mobile-dark.gif deleted file mode 100644 index ea51b8a2..00000000 Binary files a/docs/images/hero-mobile-dark.gif and /dev/null differ diff --git a/docs/images/hero-mobile-light.gif b/docs/images/hero-mobile-light.gif deleted file mode 100644 index 7ecbc39a..00000000 Binary files a/docs/images/hero-mobile-light.gif and /dev/null differ diff --git a/docs/images/hero-small-dark.gif b/docs/images/hero-small-dark.gif deleted file mode 100644 index 4d3688fb..00000000 Binary files a/docs/images/hero-small-dark.gif and /dev/null differ diff --git a/docs/images/hero-small-light.gif b/docs/images/hero-small-light.gif deleted file mode 100644 index b291fe68..00000000 Binary files a/docs/images/hero-small-light.gif and /dev/null differ diff --git a/docs/images/social-preview.png b/docs/images/social-preview.png deleted file mode 100644 index 6c6b2072..00000000 Binary files a/docs/images/social-preview.png and /dev/null differ diff --git a/docs/wiki/agent-memory-systems-compared.md b/docs/wiki/agent-memory-systems-compared.md deleted file mode 100644 index 9c6c3954..00000000 --- a/docs/wiki/agent-memory-systems-compared.md +++ /dev/null @@ -1,134 +0,0 @@ ---- -title: "Agent memory systems, measured on the same questions" -description: "Mastra Observational Memory, LangMem, Hindsight, an agentic dream pass and a single request, run on the same LongMemEval questions on real agents, with accuracy and cost per pass. Plus what changes with years of history." -parent: "Studies" -nav_order: 2 ---- - -# Agent memory systems, measured on the same questions - -A Dot keeps a profile of the person, `MEMORY.md`, in every prompt (see -[How a Dot remembers](how-a-dot-remembers.md)). Something has to write it from -the conversations. Many open-source projects do this; most publish their own -scores on their own setups, and the setups differ. So the candidates were run -here on the same questions, the same agent and the same model. - -## The setup - -- **Questions:** the 30 LongMemEval questions whose answer depends on what the - person likes ("can you suggest a hotel for my trip?"), each with about 48 - dated chats, 115,000 tokens, as its history. These are the ones a profile is - for; the other kinds were checked separately on 116 mixed questions. -- **Agent:** a new Dot per question, with the history as its conversation - files. Each method writes `MEMORY.md` from that history; then the question - goes to the Dot as a task. The Dot can also grep the files, so every method - is measured on top of the same baseline. -- **Model:** z-ai/glm-5.3-flash for everything, through OpenRouter; GPT-4o - grades with LongMemEval's own prompt. -- **Runs:** two per method where the budget allowed. Two identical runs with no - profile scored 19 and 20 out of 30, and others moved by up to three - questions, so a single run of 30 cannot tell methods apart; a difference of - two questions out of 60 is noise. -- **Faithfulness:** each project's own prompts, copied verbatim (18 of 18 - checked against the sources), its own call pattern, and its own benchmark - settings where it publishes them. Every adaptation is marked in the adapters, - which are kept in commit `de3d9a1b` of the repository. - -## Results - -| Method | Correct | % | Cost of a pass over 115K tokens | Calls | Needs | -|---|---|---|---|---|---| -| No profile (the Dot greps the files) | 39/60 | 65% | 0 | 0 | nothing | -| **One request (ours)** | **50/60** | **83%** | **~$0.015** | **1** | nothing | -| Hindsight (MIT) | 25/30 | 83% | ~$0.25 | ~255 | Postgres + pgvector, ~1.9 GB RAM | -| Mastra Observational Memory (Apache-2.0) | 48/60 | 80% | ~$0.05 | ~6 | (TypeScript) | -| LangMem profile manager (MIT) | 48/60 | 80% | ~$0.02 | 1 | LangChain | -| Agentic pass with file tools (nanobot's Dream, MIT) | 44/59 | 75% | ~$0.08 | ~20 steps | nothing | -| Memobase (Apache-2.0) | stopped | | ~$0.25-0.40 | ~200 | Postgres + Redis | -| Mem0 (Apache-2.0) | not run | | ~$0.40 | ~250 | a vector store | -| Letta sleep-time agents (Apache-2.0) | not run | | ~$1.10-1.60 | ~120, growing | a server + Postgres | - -The costs of the last three are estimated from their own call patterns, -measured against a stand-in model: running them through would have cost more -than the rest of the study together, for designs the measured ones already -covered (Letta's is an agentic pass like Dream's, which was measured). - -## What the numbers say - -**The profile matters more than the method.** Every method that writes one -moves the score from 65% to about 80%. Between the one request, Hindsight, -Mastra and LangMem the differences are two questions out of 60 or less: -inside the noise. - -**At equal quality, cost decides, and it varies seventeenfold.** The single -request reads the conversations once and answers with the whole new profile. -Mastra reads them in 30,000-token chunks and rewrites its notes when they grow. -Hindsight extracts facts from every ~1,800 characters, one call each. The -agentic pass opens files one at a time with tools, and every step sends the -whole growing context again: 115,000 tokens of history become about 500,000 -tokens paid. Letta's sleep-time agent does the same while keeping its own -transcript, which is why its estimate is the highest. - -**Simpler is also more robust.** The single request has no tools to call and -no state of its own; the agentic pass failed in 13 of 30 runs before a token -sizing bug was fixed (see -[Token counts are not portable between models](token-counts-across-models.md)), -and Memobase's run ended when the credit did. - -**On the 116 mixed questions the profile helped where expected and hurt -nowhere:** updates from 78% to 94% (the profile keeps the newest version of a -fact, with its date), preferences from 57% to 71%, the other kinds within one -question. - -## What was chosen - -The single request, as a background pass: once the Dot has been quiet for -five minutes, the messages said since the last pass go to the summary model -with the current `MEMORY.md`, and the answer is the new `MEMORY.md`. Run in the product on the 116 mixed questions it -scored 90.4% (104 of 115 that ran), at $0.025 a pass. From Mastra it -takes two ideas rather than code: reading in pieces when the new conversations -do not fit one request, and saying that a changed fact replaces the old one. -From LangMem, waiting for a quiet spell rather than running after every -message. - -## With years of history (reasoning, not measured) - -A Dot used every day for a year holds on the order of ten million tokens of -conversation. What changes: - -- **The files and grep hold.** Searching 40 MB of text is instant, and nothing - is lost, because the files are the words as said. What degrades is - precision: a common word matches hundreds of times. The first remedy costs no - model call: a ranked keyword index (BM25) inside the Dot's computer. -- **One profile stops being enough.** A year of a working life does not fit in - 25,000 characters. The pass then needs two levels: `MEMORY.md` as the core, - and notes by subject (clients, health, home) that it writes when the core is - full, each named by a line in the core. That is the part of Mastra's design - (its Reflector) worth taking later. -- **The cost of the pass stays flat**, because it reads only what is new: - around a cent a day, a few dollars a year. A monthly rebuild of the profile - from the files, about $2 at that size, would stop small errors from - compounding over hundreds of rewrites. -- **Hindsight gains, and so does its bill.** Retrieving facts for each question - from millions of tokens is its strength, and the gap to a bounded profile - would grow for specific details. Its cost grows with the history (about $22 a - year of extraction at this rate, plus a database per Dot), and its facts are - extractions, lossy where the files are verbatim. For specific details the - files already do that job. -- **The agentic designs and per-message extraction get worse**: the first - reread more every pass, the second pay on every message. -- **Contradictions are the real risk** ("I live in Milan", then months later "I - moved to Rome"). A rewrite with dated facts handles them; systems that only - accumulate facts keep both. - -## Sources - -- LongMemEval: [paper](https://arxiv.org/abs/2410.10813), [code](https://github.com/xiaowu0162/LongMemEval) -- Mastra Observational Memory: [research](https://mastra.ai/research/observational-memory), [code](https://github.com/mastra-ai/mastra) -- LangMem: [code](https://github.com/langchain-ai/langmem) -- Hindsight: [paper](https://arxiv.org/abs/2512.12818), [code](https://github.com/vectorize-io/hindsight), [benchmark harness](https://github.com/vectorize-io/agent-memory-benchmark) -- Memobase: [code](https://github.com/memodb-io/memobase) -- Mem0: [paper](https://arxiv.org/abs/2504.19413), [code](https://github.com/mem0ai/mem0) -- Letta: [code](https://github.com/letta-ai/letta), [sleep-time compute](https://arxiv.org/abs/2504.13171) -- nanobot (Dream): [code](https://github.com/HKUDS/nanobot) -- Is Grep All You Need?: [arXiv 2605.15184](https://arxiv.org/html/2605.15184) diff --git a/docs/wiki/benchmarking-agents-on-real-vms.md b/docs/wiki/benchmarking-agents-on-real-vms.md deleted file mode 100644 index 76ec2157..00000000 --- a/docs/wiki/benchmarking-agents-on-real-vms.md +++ /dev/null @@ -1,92 +0,0 @@ ---- -title: "Benchmarking an agent on real VMs" -description: "How Dots are tested as a person meets them: Harbor tasks in a new VM each, long builds of two to three hours, eight Dots on one host, and the defects these runs found." -parent: "Studies" -nav_order: 5 ---- - -# Benchmarking an agent on real VMs - -A Dot is an agent with its own computer: a QEMU virtual machine on the host, -reached through an API. Unit tests and a stand-in model check the engine's -logic; they cannot tell whether a Dot gets real work done. These runs do, on -real VMs and a real model, and every one of them found something. - -## The harness - -[Harbor](https://github.com/harbor-framework/harbor) (Apache-2.0), the harness -of Terminal-Bench, runs a task in an environment and grades what is left. Here -the environment is a Dot: a new one per trial, created through the product's -API, with the task's Dockerfile replayed inside it as the Dot's own user (there -is no root: `apt-get install` becomes the Dot's `sudo dot-install`), and the -task's paths moved under its home. The agent is the Dot itself, given the -instruction as a task. A task counts only once its reference solution scores 1 -and doing nothing scores 0 **in a Dot**: a task that needs root fails its -oracle, which is how such a task is found. Code: `tests/bench/`. - -## Short tasks - -25 tasks written for this, in shell, files and documents, coding, data -analysis, the computer's settings, questions with one answer, two long -pipelines, and three about what a Dot is for (its own automation must run -twice; clean up leftovers and nothing else; say a request is impossible). The -data is generated with a fixed seed and the expected answers are computed by -the authoring script, never by an agent. - -- Oracle 25/25, doing nothing 0/25. -- A Dot on z-ai/glm-5.3-flash: **73 of 75 trials** (each task three times). - Both misses were the same task, a Bash script to port to Python. - -## Long tasks: two to three hours - -Five tasks where the Dot builds or researches something whole, each graded by -checking the result itself, never by asking a model: - -| Task | What the Dot does | What the grader checks | -|---|---|---| -| chess-perft | a chess move generator, every rule included | published perft counts of five positions to depth 4 and 5 (up to 4.9 million), and four positions it has not seen | -| lisp-interpreter | an interpreter for a Scheme subset | 30 programs it has not seen: closures, tail calls a million deep, recursion 5000 deep, errors | -| kv-store | a key-value server over HTTP | the API, TTLs, 1000 writes from 20 clients at once, every acknowledged write after `kill -9` | -| language-history | research on 18 programming languages, on the web | years and creators, influences, a cited 1500-word report from five sites | -| fraud-investigation | an investigation of 292,467 generated card transactions | exactly the planted cases of four kinds of fraud, a script that finds them again | - -These found the token limits described in -[Compacting a thread at the model's own window](compacting-at-the-models-window.md): -with a 4,096-token answer cap, a Dot writing a whole program in one call looped -until it failed. After the fix all five passed, the server and the interpreter -two out of two, in runs of 18 to 75 minutes. - -## Memory: LongMemEval - -The same harness runs [LongMemEval](https://github.com/xiaowu0162/LongMemEval) -on Dots: a history of 48 chats becomes a Dot's past conversations, the question -is a task, GPT-4o grades outside the Dot. The results and what they changed are -in [How a Dot remembers](how-a-dot-remembers.md) and -[Agent memory systems, measured on the same questions](agent-memory-systems-compared.md). -This run found two defects of its own: a token estimate OpenRouter refuses -(see [Token counts are not portable between models](token-counts-across-models.md)), -and in the harness a call to a Dot that could wait forever, which once held -four questions for seven hours; every call now has a bound. - -## Many Dots on one host - -`tests/e2e/scale.ts` starts 1, 2, 4 and 8 Dots at once on one host; each must -come up, answer a message and do a task, within budgets that are ratios to one -Dot. All four steps passed. Measured: the time to READY at the 95th percentile -went from 21 s with one Dot to 45 s with eight; the API answered in 6 to 10 ms -at the 95th percentile throughout; each Dot's QEMU took about 700 MB. - -## What the other tests found - -- **Approvals**: every tool under allow, deny, ask-then-approve and - ask-then-reject, in the chat and in a task (258 cases), plus a race of an - approve and a reject sent at the same moment, 50 times on PostgreSQL: one - wins, the other is refused, the Dot hears one decision. It found that a task - cancelled while a call waited for approval left the approval pending forever, - so the Dot never went back to READY and never slept. -- **Upgrades**: the databases a Dot and a host of a released version left - behind open under the new build and keep working. - -Every run here costs real money on a real model; the numbers above were bought -for a few dollars each, and the harness keeps each Dot's events, engine log and -memory with its result so a wrong answer can be read after the Dot is gone. diff --git a/docs/wiki/browser-calls-that-lead-nowhere.md b/docs/wiki/browser-calls-that-lead-nowhere.md deleted file mode 100644 index 72074ff3..00000000 --- a/docs/wiki/browser-calls-that-lead-nowhere.md +++ /dev/null @@ -1,119 +0,0 @@ ---- -title: "Browser calls that lead nowhere" -description: "45 browser tasks of real Dots, call by call: 90 of 773 page calls led nowhere and took a third of the time spent on pages. Grouped by cause, with what was changed for each and what was not." -parent: "Studies" -nav_order: 6 ---- - -# Browser calls that lead nowhere - -Two Dots in front of their owner spent their steps on calls that went nowhere: -an address that answered 404, then another, then a search. Instead of fixing -the case in view, every browser call of the saved runs was counted and each -wasted one given a cause. The numbers came first, then the rules. - -## The data - -All the tasks that drove a page: 41 from the benchmark runs of 7 to 9 October -2026 (LongMemEval questions and the long language-history task, mostly -z-ai/glm-5.3-flash) and 4 from a test Dot on 10 October. That is 45 tasks, -1,403 calls, 773 of them on a page (355 navigations), 2,823 seconds spent in -page calls. The events keep, for each call, the tool, its target, whether it -succeeded and how long it took, not what it returned. So a navigation -abandoned without being read was checked afterwards: its address was fetched -again on 10 October and compared with the addresses that were read. - -## What led nowhere - -90 page calls (12%), 961 seconds: a third of the time spent on pages. - -| Cause | Calls | Seconds | What it was | -|---|---:|---:|---| -| An address from memory | 58 | 416 | 54 navigations dropped without reading the page, 4 to domains that no longer exist. Of the 54, 27 answer 404 (or have no archived copy), 5 redirect to an index, 7 land on another date of the Wayback Machine. Of 40 addresses that were read, 1 answers 404. | -| The first page after a launch | 5 | 155 | The first navigation failed at 31 seconds and the same one, again, loaded. Only on a loaded host, where the launch itself took 36 to 115 seconds. | -| A page over 45 seconds | 5 | 238 | Three Wayback Machine captures, and one Starbucks page tried twice. | -| A read right after the navigation | 10 | 50 | `browser_read_text` failed once and worked when called again. Four came right after a Google search. One passed a `text=` selector, which is not CSS. | -| A click under Google's consent dialog | 4 | 66 | The click on a result waited its 15 seconds and failed. The model then answered the dialog and the same click worked. | -| Other clicks | 2 | 33 | A selector the model wrote itself instead of the snapshot's, and a link inside a closed menu. | -| `browser_back` | 3 | 1 | It failed three times out of three. | -| Other | 3 | 1 | A custom select, and a call with no arguments. | - -No snapshot was taken twice without something done in between. - -## Back, forward and reload never worked - -The Dot's `browser_back`, `browser_forward` and `browser_reload` pressed -Alt+Left, Alt+Right and F5 on the page. They were run on the library itself: - -| Key | What happened | -|---|---| -| Alt+Left, Alt+Right | `unknown key: 'Left'`: the library's names are `ArrowLeft` and `ArrowRight` | -| Alt+ArrowLeft, Alt+ArrowRight | Pressed, and the page stayed where it was | -| F5 | Pressed, and the page was not loaded again (a value set on the window survived it) | -| PageDown (`browser_scroll`) | Scrolled, from 0 to 913 pixels | -| A navigation to the page's own address | Loaded it again (HTTP 200, the value gone) | - -A key the server presses reaches the page, never the browser's own shortcuts. -So the three tools were removed: going back is a navigation to the address the -page came from, and loading a page again is a navigation to its own address. -The five "reloaded the page" answers in the runs were five reloads that did not -happen. - -## What was changed - -- **Finding a page** (PR #1438): an address comes from a link seen in a - snapshot, from the site's own search, or from a search engine, never from - memory; after a 404 the next address comes from a page that loaded. On a Dot - asked for three documentation pages whose addresses are not guessable, - 2 addresses from memory that answered 404 became 0; the task cost $0.037 - instead of $0.019, the difference being the snapshots of the results. Brave - is the engine named because it was the one that gave the Dot's Firefox - usable results: DuckDuckGo's HTML version wraps its links in a redirect, - and Bing gave an empty results area. -- **Back, forward and reload** removed, as above. -- **`browser_read_text` reads what the other tools find.** It resolved its - selector with `document.querySelector`, so a snapshot selector in Playwright's - own syntax (`:nth-match(...)`) or a field inside a shadow root failed there - while the same string clicked. invisible-playwright-mcp 0.71.0 resolves it - through the engine, as it already did for the diagnosis of a failed click. -- **A consent dialog is answered first.** The rule is now in the browser - server's own instructions. On the same Google task, run once with the old - rules and once with the new ones on a fresh Dot, both answered the dialog - before clicking. So this run shows no gain; the rule is for the four clicks - above. - -These rules, and the rest of what a page needs, are now written once, in -the browser server: see -[The browser server's own tools and words](the-browser-servers-own-words.md). - -## What was not changed, and why - -- **The snapshot with the navigation.** Another agent framework, OpenClaw, - returns the page's snapshot with every navigation, so a page costs one call. - Here, what follows a navigation that loaded is a text read 72% of the time, - another navigation 15%, and a snapshot 7%. An inline snapshot would save 25 - calls of 773. Not worth the tokens it would add to every navigation. -- **The two timeouts belong to the library.** The 31-second failure matches - the 30 seconds `invisible_playwright` gives `Browser.newPage`, which the first - navigation of a launch calls, while the navigation itself has 45. It is - reported there ([invisible_playwright#321](https://github.com/feder-cr/invisible_playwright/issues/321)), - not worked around in the engine. -- **The reads that fail right after a Google search** are not explained. The - events do not keep the error text, and a probe with the local copy of the - library (older than the Dots') did not reproduce it. - -## The method's limit - -The events say that a call failed, not why. The causes above come from the -target, the duration, the call that came next, and the addresses fetched -again; the error text is only in the Dot's own database, which the benchmark -does not save. - -## Sources - -- The analysis scripts read the saved events of `tests/bench/` runs - (`/work/bench-jobs/*/events/*.json`) and the test server's Dots. -- [invisible_playwright](https://github.com/feder-cr/invisible_playwright): - key names (`_juggler/keyboard.py`), `Browser.newPage` (`_juggler/server.py`). -- [invisible-playwright-mcp](https://github.com/feder-cr/invisible_playwright_mcp): - `browser_read_text` and `browser_snapshot` (`mcp/actions.py`). diff --git a/docs/wiki/compacting-at-the-models-window.md b/docs/wiki/compacting-at-the-models-window.md deleted file mode 100644 index 46223a77..00000000 --- a/docs/wiki/compacting-at-the-models-window.md +++ /dev/null @@ -1,83 +0,0 @@ ---- -title: "Compacting a thread at the model's own window" -description: "Why a Dot sets no token limit of its own, when it compacts a long thread, and why it clears old tool results before it summarizes anything: the rules taken from opencode, Codex, OpenHands and a SWE-bench study." -parent: "Studies" -nav_order: 4 ---- - -# Compacting a thread at the model's own window - -An agent working for hours fills its context. Something has to give, and -every choice costs something: a cap on the answer cuts work in half, a summary -loses detail, a truncation loses more. This page is how a Dot decides, and the -failures that decided it. - -## What went wrong with fixed limits - -Until October 2026 a Dot sent `max_tokens: 4096` on every request and kept each -thread under a configurable 32,000 tokens. On long benchmark tasks (writing a -Scheme interpreter, a key-value server: see -[Benchmarking an agent on real VMs](benchmarking-agents-on-real-vms.md)) this -failed in a specific way: the model wrote a whole program in one `write_file` -call, the answer was cut at 4,096 tokens in the middle of the call, the engine -asked it to "continue from its exact endpoint", and with nothing delivered to -continue from, the model made the same call again until the task failed. A -model with a one-million-token window was being summarized at 32,000. - -So the limits went, all of them: a Dot has no token setting. Every request uses -its model's own limits, read from OpenRouter at the start of every turn (the -widest window of the providers serving the model, and the longest answer at -that window; its requests go only to those providers, so none lands on a -narrower one), and -asks for the longest answer the window leaves room for. Without `max_tokens` -each provider would apply a default of its own, often far lower. A response -cut before it delivered anything is now told that nothing ran and to work in -smaller steps, instead of to continue. - -## When to compact - -A thread is compacted only when the next request would not fit: at the window, -less the room kept for the answer (20,000 tokens, or the longest answer when -that is shorter) and a safety margin. That is opencode's rule. How the prompt is -measured turned out to matter as much as the rule: see -[Token counts are not portable between models](token-counts-across-models.md). - -## How: clear first, summarize second - -1. **Old tool results are cleared.** Walking back from the newest message, the - first 40,000 tokens of tool results stay; older ones become a short note that - the tool can be called again, if that frees at least 20,000 tokens. The calls, - the reasoning and every message stay. No model call is needed. On SWE-bench - this costs about half as much as summarizing for the same solve rate - (JetBrains, "The Complexity Trap", arXiv 2508.21433); the thresholds are - opencode's. -2. **Only if that is not enough, a summary.** It is written as a handoff for the - model that resumes the work (Codex's framing), under the headings of - OpenHands' summarizing prompt, plus what only a Dot has: approvals given and - files written. After it, the person's latest messages are kept as they wrote - them (Codex keeps 20,000 tokens; a Dot keeps at most a quarter of the budget, - because a model with a small window still needs room after them). The summary - may be written by a cheaper model (`models.summary`), within that model's own - window; a thread too long even for it loses its oldest messages first, never - a call without its result. - -OpenRouter's own `context-compression` plugin was looked at and not used: it is -a middle-out truncation, which drops the middle of the thread without a trace. - -## What it changed - -On the five long tasks (two to three hours each), the key-value server and the -Scheme interpreter went from failing most runs to passing 2 out of 2; all five -passed in the final round, in runs of 18 to 75 minutes. - -One side effect is worth knowing: OpenRouter holds the cost of `max_tokens` up -front, so asking for the longest answer makes a nearly empty balance show -sooner. When it refuses for credit, the Dot now shows OpenRouter's own message -(how much the balance still allows, where to add credit) after its own. - -## Sources - -- opencode: [code](https://github.com/sst/opencode) (compaction thresholds) -- Codex: [code](https://github.com/openai/codex) (handoff summary, recent user messages) -- OpenHands: [code](https://github.com/All-Hands-AI/OpenHands) (summarizing condenser) -- JetBrains, "The Complexity Trap": [arXiv 2508.21433](https://arxiv.org/abs/2508.21433) diff --git a/docs/wiki/how-a-dot-remembers.md b/docs/wiki/how-a-dot-remembers.md deleted file mode 100644 index 10b04bac..00000000 --- a/docs/wiki/how-a-dot-remembers.md +++ /dev/null @@ -1,122 +0,0 @@ ---- -title: "How a Dot remembers" -description: "Past conversations kept as files the agent greps, a profile of the person in every prompt, and a background pass that keeps it current. Measured on LongMemEval: 7.8% to 90.4%." -parent: "Studies" -nav_order: 1 ---- - -# How a Dot remembers - -A Dot talks to one person for weeks: in its chat, and in the tasks it is -given. Three things make it remember what it was told, and each one was -added because a measurement asked for it. - -| Layer | What it holds | Who writes it | In the prompt? | -|---|---|---|---| -| Conversations | every word said, by day and by task | the engine, after every turn | no: the Dot searches it with grep | -| `MEMORY.md` | what the Dot should always know about the person | the Dot, and a pass in the background | always | -| Notes and skills | details by subject, and how to do things | the Dot | when it opens them | - -## The measure: LongMemEval - -[LongMemEval](https://github.com/xiaowu0162/LongMemEval) (ICLR 2025, MIT) -gives each question a history of about 48 dated chats, around 115,000 -tokens, and then asks something only that history answers: a detail the -person mentioned, one that changed later, how many days passed between two -events, what they would like, or something they never said (the right answer -is then that it is not known). The answer is graded by GPT-4o with the -benchmark's own prompts. - -On Dots it runs as a person would meet it: a new Dot per question, the -history written as its past conversations, the question sent as a task, the -grading done outside the Dot so the OpenRouter key never enters it. 116 -questions, drawn with a fixed seed in the benchmark's proportions; model -z-ai/glm-5.3-flash. The harness is `tests/bench/longmemeval.py`. - -| | Correct | % | -|---|---|---| -| No history at all (the control) | 9/116 | 7.8% | -| Conversations as files | 96/116 | 82.8% | -| + the fixes that run found | 102/116 | 87.9% | -| + `MEMORY.md`, written outside the Dot as the pass would | 105/116 | 90.5% | -| + the memory pass in the product | 104/115 | **90.4%** | - -The control only gets the four questions whose answer is "you never told -me", and five it can guess. The last row is the product as shipped: each Dot -made its own pass five quiet minutes after it started, then answered (one -question of the 116 did not run). - -## Conversations as files - -The transcripts live in the engine's database, which the model's commands -cannot read, and a long chat reaches the model only as a summary of its older -part. So after every turn the engine also writes what was said to the Dot's -own computer: - -``` -/home/dot/conversations/chat/2023-05-20.md one file a day of the chat -/home/dot/conversations/tasks/2023-05-21-t1.md one file a task -``` - -Each message sits under a heading with its time, and each call the Dot made -is one line (not what it returned). The prompt says to search there with grep -when something said before matters. This one change took the Dot from 7.8% to -82.8%. Plain files and grep were chosen over embeddings or a vector store -because that is what wins in the published measures: in "Is Grep All You -Need?" grep beat vector search in every harness and model tried, and on -LongMemEval-V2 the best result came from an agent searching files (74.9%) -rather than from extracted facts with retrieval (58.6%). - -## What the first run found - -Twenty answers were wrong, and reading each one showed where: - -- **4 of the 5 preference misses searched nothing.** Asked "can you suggest - accessories for my phone?", the Dot gave generic advice: nothing in the - question says "remember". The prompt now says to look in past conversations - before advice or a recommendation. -- **One task was refused by OpenRouter** for asking an answer 2,779 tokens - too long for the window: the token estimate was off. See - [Token counts are not portable between models](token-counts-across-models.md). -- The system prompt carried the time to the minute, so it changed on every - turn and the provider's cache never held. It now carries the day. - -Together: 82.8% to 87.9%. - -## MEMORY.md, and the pass that keeps it - -The remaining preference misses had a common cause grep cannot fix: the fact -that matters shares no word with the question. "I've been sneezing, could it -be my living room?" needs "my cat Luna sheds", and a search for "sneez|dust" -does not find the cat. What fixes that is a short profile of the person in -every prompt, so the Dot does not have to guess what to look for. - -`MEMORY.md` is that profile: who the person is, the people and animals in -their life, what they have and use, what they like and dislike, their -routines and plans, each fact with the day it was said, and a line for every -other note. The Dot sees it whole in every prompt, up to 25,000 characters -(cut there, with a word to shorten it, as Claude Code does with its own). - -A model often does not think to write it while it works, so the engine keeps -it current itself: once the Dot has been quiet for five minutes, it takes the -messages said since the last pass (of a long day, only the new ones) and has the -summary model rewrite `MEMORY.md` from them, in one request with no tools. A changed fact -replaces the old one with its date; nothing a web page or a command said is -copied in as an instruction. If the conversations do not fit one request they -go in several, oldest first. A pass that is cut, empty, unpriced, or races -with the Dot editing the file writes nothing, and the next one takes the same -conversations again. Its cost goes to the person's usage like any other turn. - -On the 30 preference questions it raised the right answers from 65% to 83%, -and on the 116 mixed ones updates went from 78% to 94% (the profile keeps the -newest version of a fact), with nothing else moving beyond the noise of a run. -How this was chosen among five methods is in -[Agent memory systems, measured on the same questions](agent-memory-systems-compared.md). - -## What it costs - -With glm-5.3-flash a pass over a whole 115,000-token history cost $0.025 on -average in the product run ($0.015 in the study, where the request was sent -from outside the Dot). In use a pass reads only what changed since the last -one, so a normal day of chat should come to a cent or two. Answering a question that needs the -history cost the Dot $0.003 to $0.005 on average. diff --git a/docs/wiki/index.md b/docs/wiki/index.md deleted file mode 100644 index 705cf8eb..00000000 --- a/docs/wiki/index.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -title: "Home" -nav_order: 1 ---- - -# invisible_dots Wiki - -invisible_dots runs Dots: autonomous AI agents, each with its own Linux -computer, that work for a person through chat and tasks. This wiki is where -the work behind them is written up: what was measured, how, what came out, -and what was chosen because of it. Every study has its numbers and its -method, so a reader can check them or run them again. - -The [README](https://github.com/feder-cr/invisible_dots#readme) is how to run -invisible_dots; [the architecture](https://github.com/feder-cr/invisible_dots/blob/main/docs/architecture.md) -is how it is built. - -## [Studies](studies.md) - -- [How a Dot remembers](how-a-dot-remembers.md): past conversations as files, - a profile in every prompt, and the pass that keeps it current. 7.8% to 90.4% - on LongMemEval. -- [Agent memory systems, measured on the same questions](agent-memory-systems-compared.md): - Mastra, LangMem, Hindsight, an agentic "dream" pass and a single request, - with their cost per pass. -- [Token counts are not portable between models](token-counts-across-models.md): - tiktoken against five models' own counts, and why a fixed safety margin got - a request refused. -- [Compacting a thread at the model's own window](compacting-at-the-models-window.md): - no token cap of our own, old tool results cleared before anything is - summarized. -- [Benchmarking an agent on real VMs](benchmarking-agents-on-real-vms.md): - Harbor tasks, two-to-three-hour builds, and eight Dots on one host. -- [Browser calls that lead nowhere](browser-calls-that-lead-nowhere.md): - 45 tasks call by call: addresses from memory, timeouts, and three tools that - never worked. -- [The browser server's own tools and words](the-browser-servers-own-words.md): - the page tools and instructions of invisible-playwright-mcp, offered as an MCP - host offers them. diff --git a/docs/wiki/mcp-servers-a-person-declares.md b/docs/wiki/mcp-servers-a-person-declares.md deleted file mode 100644 index 6d51ab13..00000000 --- a/docs/wiki/mcp-servers-a-person-declares.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -title: "MCP servers a person declares" -description: "A Dot uses the MCP servers its person declares or approves, each behind its own permission, run as the Dot's user, with secrets kept out of the config; how NanoClaw, Hermes, OpenClaw and others let an agent add one." -parent: "Studies" -nav_order: 8 ---- - -# MCP servers a person declares - -Until October 2026 a Dot had one MCP server, its browser's, and no way to use -another. The question was whether a Dot could install an application and then -use its MCP server, and what the standard way to let it do so is. - -## How other hosts do it - -| Host | Who declares a server | Where it is declared | Permission | Secrets | -|---|---|---|---|---| -| Claude Code | the person (`claude mcp add`, `.mcp.json`) | settings files | rules per server or tool (`mcp__`), asks by default | `env` in the file, or the shell's | -| Codex | the person | `[mcp_servers]` in `config.toml`, with `startup_timeout_sec` and `tool_timeout_sec` | approval policy | `env` in the file | -| nanobot (the engine's base) | the person | its config's `mcpServers` | none | `env` in the file | - -All three have the person declare a server, offer its tools as -``, and start it as a child process of the -host or reach it by URL. - -The agents that live on a computer of their own are closer to a Dot, and they -differ on whether the agent may add a server itself (their source, October 2026): - -| Host | The agent adds a server | Who approves | Usable | Secrets | -|---|---|---|---|---| -| NanoClaw | yes, `add_mcp_server` (any command or URL) | always an admin, on an approval card | after its container restarts | never in the agent: a credential gateway injects them | -| Hermes Agent | from a curated catalog only, `manage_connections` | a card; with no card, nobody | at once | entered on the card | -| OpenClaw | no; a plugin from the official catalog may bring one (owner-only) | the owner | at the next step | env, secret references, OAuth | -| nanobot | no | the person activates it | after a reload | `${VAR}`, OAuth | -| Goose | enables only extensions already configured | an allowlist | at once | its config | - -Hermes also shows what goes wrong: in June 2026 entries planted in its -`config.yaml` ran shell scripts that installed SSH keys, and it now refuses -entries of that shape. - -## What a Dot does - -- **The person declares a server, or approves one the Dot adds.** A server is - a `command` (with `args` and `env`) or a `url` (with `headers`), in the Dot's - settings or YAML. The Dot installs its program as it installs anything (from - GitHub with `uv tool install git+https://...`, from npm once it installed - Node.js), then asks with `add_mcp_server`, as NanoClaw's agents do, under the - permission `mcp.servers.add`, which asks by default. The person reads the - whole entry on the approval: what it runs or where it is, the secrets they will - set, and why. The host checks that approval again before it writes the server - and uses it up, so one approval adds one server once; the server starts at - once, with no restart. A stdio server runs as the - Dot's user, with what running a command already gives it, so what the Dot gains - are tools, each still asking. Only the person changes or removes a server. -- **One permission per server**, `mcp.`, asked by default and weighed - as high a risk as running commands: what a server's tools do is the server's - to say. -- **It runs as the Dot's user.** A command is started through `dot-agentd - relay`, as `dot`, in its home, like every program of the model. -- **Secrets are set apart** from the config, write-only, as Claude Code and - Codex do not do: `secrets: [NAME]` in the entry, the value set in the - settings or with `invisible-dots secret mcp`. It reaches the server as an - environment variable through the relay's own environment, never a command - line, or as a header; it is masked in the Dot's conversation files. -- **It starts when declared, and again when it can.** A server that failed - (its program missing, it exited) is started again when the next message or - task begins, and the turn waits for it, up to `startup_timeout_s` (60 s by - default, Codex's `startup_timeout_sec`). One that timed out, or lacks a - secret, waits for its settings to change, so it never holds up every turn. -- **The model is told why.** The prompt carries each server's `initialize` - instructions, or why it is not connected, quoting what it wrote on its - standard error. The settings and `invisible-dots mcp` show the same. - -## Measured - -A live Dot (`z-ai/glm-5.3-flash`) was given two servers, `npx -y -@modelcontextprotocol/server-everything` on a computer with no Node.js, and -`uvx mcp-server-time`, then asked to use the first one's `echo` tool and the -second one's time in Tokyo. - -| Task | What the Dot did | Calls | Cost | -|---|---|---|---| -| 1 | Read why `everything` was not connected (`exec: "npx": executable file not found in $PATH`), ran `sudo dot-install npm`, checked `npx`, answered the time from `mcp_time_get_current_time`, and said the server would be started again at the next message | 4 | $0.0028 | -| 2 | `mcp_everything_echo`: `Echo: dots-mcp-ok` | 1 | $0.0013 | - -The acceptance run on a real VM has a step for the same path (`uvx -mcp-server-time`: its tool asks, is approved always and runs as `dot`; a second -server waits for its secret, set through the command line), and the scan of -every log, row and disk file for the secret after it. It passes its 16 steps. - -## Limits - -- An HTTP server is reached with a header secret; its OAuth flow is not - supported (the engine's MCP client left OAuth out). -- The permission is per server, not per tool. -- A server's resources and prompts are offered as tools, as nanobot offers - them. - -## Sources - -- [Claude Code: MCP](https://docs.anthropic.com/en/docs/claude-code/mcp). -- [Codex: configuration, `mcp_servers`](https://github.com/openai/codex/blob/main/docs/config.md). -- [nanobot](https://github.com/HKUDS/nanobot): `nanobot/agent/tools/mcp.py`. -- The architecture, [sections 7 and 8.3](https://github.com/feder-cr/invisible_dots/blob/main/docs/architecture.md). diff --git a/docs/wiki/studies.md b/docs/wiki/studies.md deleted file mode 100644 index 537385ae..00000000 --- a/docs/wiki/studies.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -title: "Studies" -has_children: true -nav_order: 2 ---- - -# Studies - -What was measured to decide how a Dot works. Each page says the question, the -method, the numbers, and what was chosen; the code and the harness that -produced the numbers are in the repository, under `tests/bench/` and -`invisible_engine_dots/`. - -- [How a Dot remembers](how-a-dot-remembers.md) -- [Agent memory systems, measured on the same questions](agent-memory-systems-compared.md) -- [Token counts are not portable between models](token-counts-across-models.md) -- [Compacting a thread at the model's own window](compacting-at-the-models-window.md) -- [Benchmarking an agent on real VMs](benchmarking-agents-on-real-vms.md) -- [Browser calls that lead nowhere](browser-calls-that-lead-nowhere.md) -- [The browser server's own tools and words](the-browser-servers-own-words.md) -- [MCP servers a person declares](mcp-servers-a-person-declares.md) diff --git a/docs/wiki/the-browser-servers-own-words.md b/docs/wiki/the-browser-servers-own-words.md deleted file mode 100644 index bed0b29d..00000000 --- a/docs/wiki/the-browser-servers-own-words.md +++ /dev/null @@ -1,100 +0,0 @@ ---- -title: "The browser server's own tools and words" -description: "A Dot's page tools used to be rewritten in the engine and their rules copied into a skill. Now the engine offers invisible-playwright-mcp's tools and instructions as an MCP host does, and every rule about a page is written once, in the server." -parent: "Studies" -nav_order: 7 ---- - -# The browser server's own tools and words - -A Dot drives its browser through invisible-playwright-mcp. Until October 2026 the -model never saw that server's words: the engine wrote its own tool names and -descriptions, and its skill copied the server's instructions (the order to try -things in, never act by script, say when a task is impossible). The audit of the -browser calls ([Browser calls that lead nowhere](browser-calls-that-lead-nowhere.md)) -then added three more rules to the skill, all of them true of any page and any -host. Two copies of a rule drift apart, and the copy the model reads was not the -one the server's authors keep. - -## How other hosts do it - -| Host | Tool descriptions | The server's instructions | -|---|---|---| -| Claude Code | the server's, as `mcp____` | in the system prompt | -| opencode | the server's | in the system prompt, in `` | -| nanobot (the engine's base) | the server's, as `mcp__` | not used | -| a Dot, before | rewritten in the engine | not used; copied into a skill | - -## Why it could not be passed as it was - -The server serves one person and a helper, `main` and `support`, and its model -opens them itself: "OPEN `main` WITH browser_open BEFORE ANYTHING ELSE". A Dot -has many identities, each the engine opens and closes, within limits and -permissions the model does not control. Handed the server's instructions as they -were, a Dot's model would read about a tool it does not have. - -The standard answer is the one Microsoft's playwright-mcp gives with its flags: -the host starts the server configured for how it is used. invisible-playwright-mcp -0.71.0 reads `INVISIBLE_MCP_HOST_MANAGED=1`. The process then serves `main` -alone, no tool takes `browser` (`support` is refused if sent anyway), and its -instructions are the page rules only. Opening and closing stay tools of the -server, because the host needs them: a first launch answers with the engine's -download progress for minutes, and a close has to let Firefox write its profile -before the MCP client stops the process two seconds after closing its input. - -## What a Dot does now - -- The page tools are the server's: same names, descriptions and input schemas, - plus the `identity_id` of the open identity whose server the call goes to. - The permission table says which the model is offered; `browser_open`, - `browser_close`, `browser_list` and `browser_status` are the engine's identity - tools, and `browser_watch` is the frame the UI shows. -- The server's instructions go into the system prompt whenever a page tool is - offered. -- The engine reads both from the server its Dot has, when it starts - (`nanobot/dots/server_interface.py`): a Dot keeps the golden image it was made - from, so the engine offers what that server serves, whatever its version. The - image takes the newest release at each build; nothing pins it. -- What the engine still adds is what only a Dot knows: a URL that is not - `http://` or `https://` is refused before it reaches the server, and the skill - is about identities. - -The tools the Dot had invented went: `browser_scroll` (the server's -`browser_press_key` with `PageDown` does it) and the 214-character cap on -`browser_type`, which the server made unnecessary when it began to type a long -text in the background. Four tools of the server came in: `browser_read_html`, -`browser_evaluate`, `browser_upload_files`, and `browser_take_screenshot` under -its own name. An image the server answers with is shown, so `browser_click_at` -returns the page after the click. - -## browser_evaluate, measured - -On 0.71.0, four verification tasks (three documentation pages, a Google result, -a link and back, a PyPI release) made 66 calls, and `browser_evaluate` was the -most frequent at 14: mostly to read text, a heading or a link that -`browser_read_text` returns. 0.71.1 changed only that tool's words and the rung -that names it: last, and which tool reads text, markup or a control instead. -The same four tasks on 0.71.1 made 12 calls of it out of 78. The documentation -task went from 6 to 3, the link task from 3 to 1. The PyPI task used it 7 times, -and rightly: the release was 15 minutes old, the page showed only "15 minutes -ago", and the date was in the `datetime` attribute of a `