diff --git a/bridge/agentenv/computer.py b/bridge/agentenv/computer.py new file mode 100644 index 0000000000..fc1540e29b --- /dev/null +++ b/bridge/agentenv/computer.py @@ -0,0 +1,83 @@ +"""Keep one agentenv computer up for REA until stdin closes or SIGTERM arrives. + +REA runs ``computer.py [task]`` with the agentenv runner's own interpreter, so ``agent_env`` +imports. It deploys the bundle's task, ``smoke`` by default, on this machine with +``run_bundle(keep=True, emulate=False)``, prints one JSON line naming the computer's address, MCP URL, control +token and sandbox id, or ``{"error": ...}``, and tears the computer down once its stdin closes or it gets SIGTERM +or SIGINT. Everything else it or agentenv prints, a build's output included, goes to stderr. It exits 0 once +the teardown left nothing up, 1 when it did, and 2 when no computer came up. +""" + +from __future__ import annotations + +import json +import os +import signal +import sys +import threading + + +def _protocol(): + """A private copy of stdout for the JSON line; fd 1 then points at stderr, as do children that inherit it.""" + protocol = os.fdopen(os.dup(1), "w", encoding="utf-8") + os.dup2(2, 1) + sys.stdout = sys.stderr + return protocol + + +def _say(protocol, value: dict) -> None: + protocol.write(json.dumps(value) + "\n") + protocol.flush() + + +def _computer(result): + """The run's deployed computer, or why there is none.""" + run = result.runs[0] if result.runs else None + if run is None: + return None, "the bundle ran no task" + if run.error is not None or run.failed_steps: + return None, f"the task failed: {run.error or run.failed_steps}" + for env in run.context.deployed_envs: + if env.environment_url and env.mcp_url and getattr(env, "sandbox_id", None): + return env, None + return None, "the task deployed no computer" + + +def main(argv: list[str]) -> int: + protocol = _protocol() + bundle, task = argv[1], argv[2] if len(argv) > 2 else "smoke" + try: + from agent_env.bundle import run_bundle + except ImportError as error: + _say(protocol, {"error": f"agent_env does not import with {sys.executable}: {error}"}) + return 2 + try: + result = run_bundle(bundle, tasks=[task], keep=True, emulate=False, sandbox="local", + on_progress=lambda line: print(line, file=sys.stderr, flush=True)) + except BaseException as error: # noqa: BLE001 - a signal's RunInterrupted too; run_bundle tore its runs down + _say(protocol, {"error": f"{type(error).__name__}: {error}"}) + return 2 + stop = threading.Event() + for signum in (signal.SIGTERM, signal.SIGINT): + signal.signal(signum, lambda *_: stop.set()) + env, problem = _computer(result) + try: + if env is None: + _say(protocol, {"error": problem}) + else: + _say(protocol, {"base_url": env.environment_url, "mcp_url": env.mcp_url, + "control_token": env.control_token, "sandbox_id": env.sandbox_id}) + threading.Thread(target=lambda: (sys.stdin.buffer.read(), stop.set()), daemon=True).start() + while not stop.wait(0.5): + pass + finally: + torn_down = result.teardown() + still_up = [sandbox for run in torn_down.runs if run.torn_down is not None for sandbox in run.torn_down.still_up] + if still_up: + print(f"still up after the teardown: {still_up}", file=sys.stderr, flush=True) + return 1 + return 0 if env is not None else 2 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index a438e9257d..304218e341 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -78,6 +78,10 @@ export default defineConfig({ text: "Desktop application tasks", link: "/desktop-application-tasks", }, + { + text: "Desktop build computer", + link: "/desktop-build-computer", + }, { text: ".NET assemblies", link: "/managed-code-analysis" }, { text: "Python packaged applications", diff --git a/docs/desktop-build-computer.md b/docs/desktop-build-computer.md new file mode 100644 index 0000000000..e5be5c362c --- /dev/null +++ b/docs/desktop-build-computer.md @@ -0,0 +1,158 @@ +# Desktop build computer + +A build computer is a local agentenv computer that an agent drives while it +builds a desktop app environment. Claude Code cannot attach a new MCP server +in the middle of a session, so REA keeps the computer up and forwards the +agent's computer-use actions to it through three MCP tools: + +| Tool | What it does | +| ----------------------- | ------------------------------------------------------------------------------------------ | +| `start_build_computer` | Starts a computer from a request or a bundle folder; returns its id and a first screenshot | +| `act_on_build_computer` | Sends one action to the computer's driver and returns the driver's answer, images included | +| `close_build_computer` | Tears the computer down, by agentenv's teardown or by removing its sandbox's containers | + +The tools are MCP-only, like the debugger sessions: the computer lives as long +as the REA MCP server that started it, or until it is closed. + +## Requirements + +- Docker, and an agentenv-framework install whose `agent-env` program is named + by the absolute `REA_AGENT_ENV_COMMAND`. Bundles target agentenv + 0.9.1431. REA runs its helper, `bridge/agentenv/computer.py`, with that + runner's Python: the interpreter on the program's `#!` line, or the + `python3` beside it in its virtual environment. +- `REA_DOCKER_COMMAND`, the absolute path of the Docker CLI, lets REA remove a + computer whose agentenv teardown was not confirmed. Without it, REA names the + command to run instead. +- For a request, `REA_BUILD_COMPUTER_REA_ARCHIVE`: the absolute path of a tar + archive of a runnable REA, with its production dependencies, whose one top + folder holds `scripts/rea.mjs`. The computer runs REA from it. + +Set these where the MCP server starts; a shell export does not reach a server +that is already running. + +## Start a computer + +From a request, REA renders a bundle in a private temporary folder: a computer +app on the `ubuntu-24.04` base with the `reference` driver that installs the +app, opens it on demand with `gtk-launch ` and keeps no seed and +no data hooks. The app is not open when the computer starts: agentenv lists it +in the desktop's Applications menu under Other by the environment's name, such +as `mousepad`, and that entry launches it through the app's server. + +```json +{ + "source": { + "kind": "request", + "desktop_entry": "org.xfce.mousepad", + "window_class": "Mousepad", + "install": { "kind": "archive", "package": "mousepad" } + } +} +``` + +`install` takes one of: + +| `kind` | Installs | +| ---------------- | ---------------------------------------------------------------------------------------------------------- | +| `archive` | `package` from the Ubuntu 24.04 archive, on amd64 and arm64 | +| `apt_repository` | `package` from the repository at `uri` with `suite` and `components`, trusting the key file `key_file` | +| `package_file` | The `.deb` at `url`, checked against `sha256`, for one `architecture`, after its apt `dependencies` are in | + +Build steps run with the network off, so a `.deb`'s dependencies must be +listed. The bundle adds REA in a build-only layer: Node.js +24.22.1, downloaded per architecture and checked against its release checksum, +and REA from `REA_BUILD_COMPUTER_REA_ARCHIVE`, both unpacked into `/opt/rea` by +a root build step, with `node` and `rea` on `PATH`. An exported bundle carries +none of it. + +From a bundle folder, such as one `export-desktop-application` wrote, REA +starts that bundle's `tasks/smoke.json` as it is: + +```json +{ "source": { "kind": "bundle", "path": "/home/ada/mousepad-bundle" } } +``` + +REA runs the deploy with `run_bundle(keep=True, emulate=False)` on the local +sandbox provider, so only the host's own architecture is built. The first start +of a bundle builds its image and can take minutes; later starts reuse it. REA +then waits until the computer's readiness checks pass, opens one MCP session +to its driver and takes a screenshot, with the `region`, `format` and `quality` +given in `screenshot`. The result holds `computer_id`, agentenv's `sandbox_id`, +the environment name or bundle folder, and the screenshot's frame as structured +content beside its image. + +A failed start returns the last 16 KiB of the helper's stderr, where a failing +build prints its error, and its exit status; a cancelled one stops the helper, +which tears down whatever agentenv brought up. When that teardown is not +confirmed, the error's `computer_id` names what REA keeps for +`close_build_computer` to retry. Closing the server cancels starts still in +progress and waits for them before closing every computer. + +## Act + +`act_on_build_computer` takes `computer_id`, an `action` and the driver's own +arguments for `computer_`, which REA passes through: + +| `action` | Arguments | +| ------------ | --------------------------------------------------------------- | +| `screenshot` | `region`, `format` (`png` or `jpeg`), `quality` | +| `click` | `x`, `y`, `button`, `count` (2 double-clicks), `modifiers` | +| `drag` | `path` of `[x, y]` points, `button`, `duration_ms`, `modifiers` | +| `scroll` | `direction`, `x`, `y`, `amount`, `modifiers` | +| `type` | `text`, where `\n` presses Enter and `\t` Tab | +| `key` | `keys`, one chord pressed in order, and `repeat` | +| `wait` | `ms` | + +Pointer actions also take `frame_id`, `space` and `norm_scale`, and every +action but `screenshot` takes `screenshot: true` for an image afterwards. That +image is taken at once, so after an action that opens a menu or a window, ask +for it with a `wait` of a few hundred milliseconds instead. +Points are pixels of the latest frame, so take a screenshot first; with +`space: "norm"` they are on a `0..norm_scale` grid over it. Keys are W3C names +or one character; REA translates model spellings such as `ctrl+s`, `Return`, +`esc` and `cmd` and leaves other names for the driver to judge. An argument the +action does not take is refused before anything is sent. + +The result carries the driver's content unchanged, its text and images, after +REA's own JSON: `action_index` and the driver's structured result. A refusal, +such as a stale frame, is an error carrying the computer's message. A response +larger than the MCP budget is refused with how to ask for a smaller image: a +`region`, or `jpeg` with a lower `quality`. That error keeps REA's own result, +`computer_id` included, as `details.result`, since the action or start has +happened. + +Every action is logged on the computer in order with the arguments sent, its +start and end and whether it was applied, refused or failed. The log keeps the +latest 256 actions within 256 KiB of JSON, dropping the oldest first. Each +string argument, such as `type`'s `text`, is kept to its first 1 KiB and each +list, such as `drag`'s `path`, to its first 64 items; an entry names the +arguments it cut in `truncated_arguments`, and the driver gets them whole. The +log counts every action, so it says how many it dropped, and keeps the index of +the last applied action of each kind for rules over the whole session. The log +and the computer's recording hooks (`computer-recording/v1` start and end under +REA's own request id, and the segments fetched through the driver's `data/get` +one at a time, each read within its manifest size and checked against it) are +kept for later build stages; they are not tools. A recording larger than a +quarter of the V8 heap is refused before any segment is read. + +## Close + +`close_build_computer` ends REA's MCP session and closes the helper's stdin, +on which the helper runs agentenv's teardown. When agentenv confirms nothing is +left, the result says `teardown: "agentenv"`. Otherwise REA removes the +containers of that exact sandbox: agentenv runs a computer as +`agent-` without a label and labels its other containers +`agentenv.sandbox=`, so REA lists both with `docker ps -aq +--filter`, removes them with `docker rm -f`, and says +`teardown: "container_label"`. When neither confirms it, the error names what +is left and the computer stays listed, so closing it again retries. Closing the +MCP server closes every computer the same way, and a helper whose REA server +exits sees its stdin close and tears down. + +The control token that guards the computer's framework-only routes stays in +REA: no result, error or log carries it. Code running in the computer as its +user can still read it, which is acceptable for a local build. + +See the [specification](specs/desktop-app-environments/build-computer/spec.md) +for scope and verification. diff --git a/docs/index.md b/docs/index.md index 88c4ec2ec1..caabe1e59d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -63,6 +63,7 @@ For runtime instrumentation, see the [Frida guide](./frida-instrumentation.md). | Read a captured application's state | [Desktop application state](desktop-application-state.md) | | Export a captured application as an environment | [Desktop application export](desktop-application-export.md) | | Write and validate a task for an exported app | [Desktop application tasks](desktop-application-tasks.md) | +| Drive a computer while building an environment | [Desktop build computer](desktop-build-computer.md) | | Use CLI/MCP and retain evidence | [CLI](cli.md), [MCP contracts](mcp-contracts.md), [agent prompts](mcp-prompts.md), [obligation ledgers](reconstruction-obligation-ledgers.md) | For development, use [testing](testing.md), [client verification](client-verification.md), diff --git a/docs/specs/desktop-app-environments/README.md b/docs/specs/desktop-app-environments/README.md index 2da35c2485..befba07af7 100644 --- a/docs/specs/desktop-app-environments/README.md +++ b/docs/specs/desktop-app-environments/README.md @@ -39,6 +39,9 @@ The work lands as one PR per step: draft are told; apt lists from a user-space `apt-get update` are found; the application does not inherit the computer's control token; and the bundle holds neither the capture host's name nor its user's. +12. [Build computer](build-computer/spec.md): an agentenv computer REA keeps + up from a request or a bundle, with REA installed inside for building, + driven through REA's MCP tools while an agent builds the environment. Version 1 covers Linux with X11, applications whose state lives in files or SQLite under the home directory, and real applications from public packages. diff --git a/docs/specs/desktop-app-environments/build-computer/plan.md b/docs/specs/desktop-app-environments/build-computer/plan.md new file mode 100644 index 0000000000..35e1f6a7d1 --- /dev/null +++ b/docs/specs/desktop-app-environments/build-computer/plan.md @@ -0,0 +1,21 @@ +# Plan + +1. Domain: the build request schema and the bundle renderer beside the + export's renderers, reusing their env.toml, apt source, install script, + smoke task and naming helpers; the action schema, per-action argument + admission and key translation. +2. Helper: `bridge/agentenv/computer.py`, shipped in the package, with its + exit statuses (0 torn down, 1 left up, 2 nothing came up). +3. Adapter: `src/desktop/AgentEnvComputer.ts` finds the runner's Python, + spawns and supervises the helper, reads the handshake, waits for readiness, + holds the driver MCP session and the token, serves the recording hooks and + releases the computer, removing its containers by name and label when needed. +4. Application: `DesktopBuildComputers` renders request bundles into a private + folder, starts and holds computers, forwards actions with their log, and + closes them, keeping unconfirmed ones for retry. +5. MCP: contracts, effects, the `application` family, the capability service, + registration, image pass-through in `ToolResultDelivery`, and the MCP-only + list for CLI parity. +6. Docs and tests: the guide; renderer and key tests; boundary tests with a + fake driver and a fake `agent_env` behind the real helper; an opt-in real + test with agentenv and Docker. diff --git a/docs/specs/desktop-app-environments/build-computer/spec.md b/docs/specs/desktop-app-environments/build-computer/spec.md new file mode 100644 index 0000000000..6c0854061f --- /dev/null +++ b/docs/specs/desktop-app-environments/build-computer/spec.md @@ -0,0 +1,74 @@ +# Build computer + +## Outcome + +An agent that builds a desktop app environment, such as Claude Code, drives a +real agentenv computer through REA's own MCP tools: REA keeps the computer up, +forwards each computer-use action to its driver and returns the driver's answer, +screenshots included, then tears the computer down. The computer starts from a +request (the app's install source and desktop entry), with REA installed inside +it for building, or from an existing bundle folder. + +## Requirements + +- MCP-only session tools in the `application` family: `start_build_computer`, + `act_on_build_computer` and `close_build_computer`, held by one owner per + server and closed with it. +- Lifecycle: one helper process per computer, `bridge/agentenv/computer.py`, + spawned with REA's owned-process supervision and run with the agentenv + runner's Python, found from the absolute `REA_AGENT_ENV_COMMAND`. It calls + `run_bundle(, tasks=["smoke"], keep=True, emulate=False, + sandbox="local")`, prints one JSON line (base URL, MCP URL, control token, + sandbox id) on a private copy of stdout, sends everything else to stderr, + and tears down on stdin EOF, SIGTERM or SIGINT. REA waits for + `GET /agentenv/ext/computer-ready/status` with the token, then holds one MCP + session to the driver's `/mcp`. +- Teardown: closing stdin, then SIGTERM, then the process group. When the helper + does not confirm the teardown, REA removes the containers of the exact + sandbox id it reported, the one named `agent-` and any labelled + `agentenv.sandbox=`, through the absolute `REA_DOCKER_COMMAND`. An + unconfirmed teardown keeps the computer listed for a retried close. + Cancelling a start stops the helper, whose deploy tears down what it began; + a helper that reported its failure finishes its own teardown first. The owner + holds each start from its helper's spawn: a failed start whose teardown is + unconfirmed stays held under the `computer_id` its error names, and shutdown + cancels and joins starts in progress before closing every computer. +- Diagnostics: the helper's stderr is kept as its last 16 KiB, through the + shared supervisor's tail retention, so a failed build reports its error + rather than its first package downloads. +- The control token never appears in a result, an error or a log; the helper's + stderr is redacted of it. +- Build bundle from a request: the app from the Ubuntu archive, an apt + repository with its key file, or a `.deb` URL with its SHA-256, dependencies + and architecture, rendered with the export's install shapes and Python + helpers; `Launch(at="demand")` through `gtk-launch`; no seed and no data + hooks; REA in a build-only layer, with Node.js 24.22.1 downloaded per + architecture and checked, and REA's payload a parameter: a local archive + shipped in the environment folder (`REA_BUILD_COMPUTER_REA_ARCHIVE`) or, + later, a released download. +- Actions: one tool with an `action` (screenshot, click, drag, scroll, type, + key, wait) and the driver's own arguments, passed through, frame or `norm` + coordinates included. Common key spellings become W3C names. An argument the + action does not take is refused before sending. Results carry the driver's + content blocks unchanged after REA's JSON, which `ToolResultDelivery` measures + against the response budget as a whole; a response over it is refused with + REA's own result, `computer_id` included, kept in the error. The first + screenshot takes `region`, `format` and `quality`. Each action is logged per + computer within a fixed allowance: the latest 256 actions in 256 KiB of JSON, + oldest dropped first, each string argument cut to 1 KiB and each list to 64 + items with the cut arguments named, beside a count of every action and the + index of the last applied action of each kind. +- Recording hooks, internal: start and end `computer-recording/v1` under REA's + request id, and fetch the segments through the driver's `data/get` one per + request, each answer read within its segment's base64 size and the segment + checked against the manifest's size and SHA-256, before teardown. A + recording over the caller's allowance, a quarter of the V8 heap by default, + is refused before any read. + +## Non-goals + +- Capture checkpoints, the apt lists folder, the capture's launch environment + and the host scrub, which a sibling change owns. +- The witness and its record, which read the action log and recordings. +- Remote sandbox providers, emulated architectures, VNC, and reaping orphans + left by an earlier REA process. diff --git a/docs/specs/desktop-app-environments/build-computer/tasks.md b/docs/specs/desktop-app-environments/build-computer/tasks.md new file mode 100644 index 0000000000..43c330665c --- /dev/null +++ b/docs/specs/desktop-app-environments/build-computer/tasks.md @@ -0,0 +1,10 @@ +# Tasks + +- [x] Record the outcome, requirements and non-goals. +- [x] Render the build bundle from a request and admit actions. +- [x] Keep a computer up with the helper and tear it down, by name and label when needed. +- [x] Forward actions with images and log them; add the recording hooks. +- [x] Register the MCP tools and their catalog entries. +- [x] Document the tools. +- [x] Add boundary tests with a fake driver and an opt-in real test. +- [x] Run the opt-in real test on an amd64 host and record it here. diff --git a/docs/specs/desktop-app-environments/build-computer/verification.md b/docs/specs/desktop-app-environments/build-computer/verification.md new file mode 100644 index 0000000000..0d7579bdf6 --- /dev/null +++ b/docs/specs/desktop-app-environments/build-computer/verification.md @@ -0,0 +1,77 @@ +# Verification + +Run on 2026-11-05 on macOS 26.5 with Node v24.22.1 and Python 3.11. Docker and +agentenv were not used by the gates or the boundary tests. + +## Tests + +- `src/domain/desktop/appBuild.test.ts`: the bundle for an archive package, an + apt repository with its key and a `.deb` with a downloaded REA; the Node.js + pin matches REA's; refused desktop entries, credentialed or non-http URLs and + relative bundle paths. +- `src/domain/desktop/computerActions.test.ts`: key translation, argument + pass-through and refusal of an argument the action does not take. +- `tests/boundary/mcp/buildComputerMcp.test.ts`, through the public MCP tools + with REA's real helper, a fake `agent_env` package on `PYTHONPATH` and a fake + driver serving readiness, `computer_*` tools over streamable HTTP, + recording and `data/get`: start with readiness polling, the image pass-through, + argument pass-through and key translation, a driver refusal, a refused + argument, close and a later unknown id; a request bundle with REA's payload, + compiled by the fake runner; the missing payload and runner settings; a + runner failure with its output; removal by name and label; an unconfirmed teardown + retried after Docker recovers; cancellation of a start stopping the helper; + a long build's failure reported from the end of its output; a first + screenshot too large for the response that keeps `computer_id` for act and + close; the first screenshot's options. No result carries the control token. +- `tests/boundary/providers/buildComputerRecording.test.ts`: recording start, + end and one fetch per segment under REA's request id, the action log, a + long session's log kept within its allowance with every action counted and + the driver still getting each argument whole, a segment that differs from + its manifest, a recording over the allowance refused before any read, and an + answer outgrowing its segment's size. +- `src/application/desktop/DesktopBuildActionLog.test.ts`: the log names only + the arguments it cut, cuts text without splitting a character, and records + the outcome of an action dropped while it was pending. +- `tests/boundary/providers/buildComputerLifecycle.test.ts`: shutdown cancels + and joins a start in progress and refuses later starts; a failed start whose + teardown is unconfirmed stays held under the id its error names, keeps its + primary failure, and is released by a retried close and by shutdown. +- `src/process/ProviderProcessOutput.test.ts`: the supervisor's tail retention + keeps each stream's latest bytes, reports checkpoints over dropped bytes as + incomplete, and drops a UTF-8 character the cut would split. +- `tests/boundary/providers/buildComputerReal.test.ts`: opt-in with + `REA_BUILD_COMPUTER_REAL=1`, `REA_AGENT_ENV_COMMAND`, `REA_DOCKER_COMMAND` and + `REA_BUILD_COMPUTER_REA_ARCHIVE`; skipped here. + +## Checks against agentenv + +- `python -m agentenv_protocol.computer._declare` from agentenv-framework + 0.9.1431 declared both a Mousepad archive bundle and a `.deb` bundle with a + downloaded REA: packages, downloads per architecture, both run scripts and + the demand launch. +- `install-rea.sh` ran as root with the network off in a local Ubuntu 24.04 + container against a Node.js-shaped and an REA-shaped archive: it unpacked + both into `/opt/rea`, put `node` and `rea` on `PATH` and removed the + archives. + +## Real computer + +Run on 2026-11-05 on Ubuntu 24.04 amd64 with Docker, agentenv-framework +0.9.1431 and Node v24.22.1. REA's archive was packed from this tree with +`npm pack`, then `npm ci --omit=dev --omit=optional` in its `rea-prod` folder, +18.5 MB as `.tar.xz`. + +- `tests/boundary/providers/buildComputerReal.test.ts` passed in 25 s with the + image built; the first run, which builds it, took 109 s. +- Through `rea mcp` over stdio: `start_build_computer` from the Mousepad + request came up in 31 s with its first screenshot as a PNG. Mousepad opened + from Applications, Other, `mousepad`; a typed line was saved with `ctrl+s` + through the Save As dialog; actions took 6 to 60 ms beside the driver's own + typing and waits. `rea --version` answered 6.8.0 in the computer for its user + and root. `close_build_computer` took 0.5 s with `teardown: "agentenv"` and + left no container. The exported Inkscape bundle started cold in 59 s and + closed the same way; a start cancelled at 60 s left nothing up. +- With the helper killed, close removed the computer's container by its name, + `agent-`: agentenv labels only its VM and closed-network + containers. +- No result or log held the control token. diff --git a/package.json b/package.json index 6635d5ef71..a2849ebf63 100644 --- a/package.json +++ b/package.json @@ -70,6 +70,7 @@ "bridge/ghidra/ReaGhidraPrepareDol.java", "bridge/android/ReaJadxBridge.java", "bridge/mitmproxy/capture.py", + "bridge/agentenv/computer.py", "scripts/rea.mjs", "scripts/lib/cli-process-environment.mjs", "scripts/rea-android-workspace-guardian.mjs", diff --git a/src/application/desktop/DesktopBuildActionLog.test.ts b/src/application/desktop/DesktopBuildActionLog.test.ts new file mode 100644 index 0000000000..23934ba0ca --- /dev/null +++ b/src/application/desktop/DesktopBuildActionLog.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from "vitest"; + +import { + BuildComputerActionLog, + LOGGED_ACTIONS, + LOGGED_LIST_ITEMS, + LOGGED_TEXT_BYTES, +} from "./DesktopBuildActionLog.js"; + +describe("build computer action log", () => { + it("cuts each list and string argument, and names only what it cut", () => { + const log = new BuildComputerActionLog(); + const path = Array.from({ length: 3 * LOGGED_LIST_ITEMS }, (_, at) => [ + at, + at, + ]); + log.begin("drag", "computer_drag", { path, button: "left" }); + // Two-byte characters, so the cut must not split one. + log.begin("type", "computer_type", { text: "é".repeat(LOGGED_TEXT_BYTES) }); + log.begin("key", "computer_key", { keys: ["Control", "x".repeat(2000)] }); + log.begin("click", "computer_click", { x: 1, y: 2 }); + expect(log.entries).toEqual([ + expect.objectContaining({ + arguments: { path: path.slice(0, LOGGED_LIST_ITEMS), button: "left" }, + truncated_arguments: ["path"], + outcome: "pending", + }), + expect.objectContaining({ + arguments: { text: "é".repeat(LOGGED_TEXT_BYTES / 2) }, + truncated_arguments: ["text"], + }), + expect.objectContaining({ + arguments: { keys: ["Control", "x".repeat(LOGGED_TEXT_BYTES)] }, + truncated_arguments: ["keys"], + }), + expect.not.objectContaining({ truncated_arguments: expect.anything() }), + ]); + }); + + it("records the outcome of an action dropped while it was pending", () => { + const log = new BuildComputerActionLog(); + const slow = log.begin("key", "computer_key", { keys: ["Control", "s"] }); + for (let index = 0; index < LOGGED_ACTIONS; index += 1) + log.begin("wait", "computer_wait", { ms: 1 }).settle("applied"); + expect(log.dropped).toBe(1); + slow.settle("applied"); + expect(log.lastApplied).toEqual({ key: 0, wait: LOGGED_ACTIONS }); + expect(log.entries).toHaveLength(LOGGED_ACTIONS); + expect(log.entries.every((entry) => entry.action === "wait")).toBe(true); + }); +}); diff --git a/src/application/desktop/DesktopBuildActionLog.ts b/src/application/desktop/DesktopBuildActionLog.ts new file mode 100644 index 0000000000..aa5e112753 --- /dev/null +++ b/src/application/desktop/DesktopBuildActionLog.ts @@ -0,0 +1,155 @@ +import type { ComputerAction } from "../../domain/desktop/computerActions.js"; +import type { JsonValue } from "../../domain/jsonValue.js"; +import { boundedUtf8Text } from "../../process/boundedUtf8Text.js"; + +/** Bytes the log keeps of each string in an action's arguments, such as type's text. */ +export const LOGGED_TEXT_BYTES = 1024; +/** Items the log keeps of each list in an action's arguments, such as drag's path. */ +export const LOGGED_LIST_ITEMS = 64; +/** The latest actions the log keeps, and the JSON bytes they may take together. */ +export const LOGGED_ACTIONS = 256; +export const LOGGED_BYTES = 256 * 1024; + +type Outcome = "pending" | "applied" | "refused" | "failed"; + +/** One forwarded action, in the order REA sent it, as the witness reads it. */ +export interface BuildComputerAction { + readonly index: number; + readonly action: ComputerAction; + readonly tool: string; + /** The driver's arguments as sent, keys translated, each string and list cut to the log's allowance. */ + readonly arguments: Readonly>; + /** The arguments cut in this entry; the driver received them whole. */ + readonly truncated_arguments?: readonly string[]; + readonly started_at: string; + readonly ended_at: string | null; + readonly outcome: Outcome; +} + +/** A value within the log's allowance, and whether it was cut. */ +const bounded = ( + value: JsonValue, +): { readonly value: JsonValue; readonly cut: boolean } => { + if (typeof value === "string") { + const { text, truncated } = boundedUtf8Text(value, LOGGED_TEXT_BYTES); + return { value: text, cut: truncated }; + } + if (Array.isArray(value)) { + const items = value.slice(0, LOGGED_LIST_ITEMS).map(bounded); + return { + value: items.map((item) => item.value), + cut: value.length > items.length || items.some((item) => item.cut), + }; + } + if (value === null || typeof value !== "object") return { value, cut: false }; + const fields = Object.entries(value).map( + ([name, field]) => [name, bounded(field)] as const, + ); + return { + value: Object.fromEntries( + fields.map(([name, field]) => [name, field.value]), + ), + cut: fields.some(([, field]) => field.cut), + }; +}; + +const jsonBytes = (entry: BuildComputerAction): number => + Buffer.byteLength(JSON.stringify(entry)); + +/** + * A build computer's action log. It counts every action and keeps the index + * of the last applied action of each kind, so rules over a whole session hold + * however long it runs, and keeps the latest actions themselves within + * `LOGGED_ACTIONS` and `LOGGED_BYTES`, dropping the oldest first. + */ +export class BuildComputerActionLog { + readonly #kept: { entry: BuildComputerAction; bytes: number }[] = []; + readonly #lastApplied: Partial> = {}; + #total = 0; + #bytes = 0; + + /** Every action logged, kept or not; the next one's index. */ + get total(): number { + return this.#total; + } + + /** How many of the earliest actions are no longer kept. */ + get dropped(): number { + return this.#total - this.#kept.length; + } + + /** The kept actions, oldest first, indexed on from `dropped`. */ + get entries(): readonly BuildComputerAction[] { + return this.#kept.map(({ entry }) => entry); + } + + /** The index of the last applied action of each kind, kept or not. */ + get lastApplied(): Readonly>> { + return { ...this.#lastApplied }; + } + + /** Log an action as it is sent, pending, and how to record how it ended. */ + begin( + action: ComputerAction, + tool: string, + args: Readonly>, + ): { + readonly index: number; + readonly settle: (outcome: Exclude) => void; + } { + const index = this.#total; + const fields = Object.entries(args).map( + ([name, value]) => [name, bounded(value)] as const, + ); + const truncated = fields.flatMap(([name, field]) => + field.cut ? [name] : [], + ); + const entry: BuildComputerAction = { + index, + action, + tool, + arguments: Object.fromEntries( + fields.map(([name, field]) => [name, field.value]), + ), + ...(truncated.length === 0 ? {} : { truncated_arguments: truncated }), + started_at: new Date().toISOString(), + ended_at: null, + outcome: "pending", + }; + this.#total += 1; + this.#keep(this.#kept.length, entry); + return { + index, + settle: (outcome) => { + if (outcome === "applied") + this.#lastApplied[action] = Math.max( + index, + this.#lastApplied[action] ?? index, + ); + // An action dropped while pending leaves only its outcome above. + const slot = index - this.dropped; + if (this.#kept[slot] !== undefined) + this.#keep(slot, { + ...entry, + outcome, + ended_at: new Date().toISOString(), + }); + }, + }; + } + + /** Put an entry in its slot, then drop the oldest until the rest fit the allowance. */ + #keep(slot: number, entry: BuildComputerAction): void { + const bytes = jsonBytes(entry); + this.#bytes += bytes - (this.#kept[slot]?.bytes ?? 0); + this.#kept[slot] = { entry, bytes }; + while (this.#kept.length > LOGGED_ACTIONS || this.#bytes > LOGGED_BYTES) + this.#bytes -= this.#kept.shift()?.bytes ?? 0; + } +} + +/** What later stages read of a computer's action log. */ +export type ReadonlyActionLog = Pick< + BuildComputerActionLog, + "total" | "dropped" | "entries" | "lastApplied" +>; diff --git a/src/application/desktop/DesktopBuildBundle.ts b/src/application/desktop/DesktopBuildBundle.ts new file mode 100644 index 0000000000..014ad26e06 --- /dev/null +++ b/src/application/desktop/DesktopBuildBundle.ts @@ -0,0 +1,110 @@ +import { copyFile, mkdir, readFile, stat, writeFile } from "node:fs/promises"; +import { dirname, isAbsolute, join } from "node:path"; + +import { + renderBuildBundle, + type BuildRequest, + type ReaPayload, +} from "../../domain/desktop/appBuild.js"; +import { + AnalysisCapabilityUnavailableError, + AnalysisInputError, +} from "../../domain/analysisErrorCore.js"; +import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvComputerHelper.js"; +import { PrivateRuntimeRoot } from "../../process/PrivateRuntimeRoot.js"; + +const START = "start_build_computer"; +const MAX_KEY_BYTES = 1024 * 1024; + +/** The archive the build layer unpacks REA from, as the operator configured it. */ +const reaPayload = async ( + environment: Readonly, +): Promise => { + const path = environment.REA_BUILD_COMPUTER_REA_ARCHIVE; + const reason = + "A build computer from a request needs REA_BUILD_COMPUTER_REA_ARCHIVE, the absolute path of a tar archive of a runnable REA whose one top folder holds scripts/rea.mjs."; + if ( + path === undefined || + !isAbsolute(path) || + !(await stat(path).then( + (entry) => entry.isFile(), + () => false, + )) + ) + throw new AnalysisCapabilityUnavailableError( + AGENTENV_PROVIDER, + START, + reason, + { + userMessage: reason, + remediationAction: + "Pack a built REA with its production dependencies, set REA_BUILD_COMPUTER_REA_ARCHIVE to the archive and restart REA, or start from an exported bundle folder.", + }, + ); + return { kind: "file", path }; +}; + +/** An apt repository's key file, read whole once it is known to be a small regular file. */ +const readAptKey = async (path: string): Promise => { + const refuse = (reason: string, cause?: unknown) => + new AnalysisInputError(START, { + userMessage: `The apt key file ${path} ${reason}.`, + conformsToInputContract: true, + ...(cause === undefined ? {} : { cause }), + }); + const entry = await stat(path).catch((cause: unknown) => { + throw refuse("cannot be read", cause); + }); + if (!entry.isFile() || entry.size > MAX_KEY_BYTES) + throw refuse( + `is not a regular file of at most ${String(MAX_KEY_BYTES)} bytes, as a signing key is`, + ); + return readFile(path).catch((cause: unknown) => { + throw refuse("cannot be read", cause); + }); +}; + +/** Write the request's build bundle into a private folder, which the computer's owner removes on close. */ +const writeBuildBundle = async ( + request: BuildRequest, + payload: ReaPayload, +): Promise<{ + readonly workspace: PrivateRuntimeRoot; + readonly bundle: string; + readonly name: string; +}> => { + const key = + request.install.kind === "apt_repository" + ? await readAptKey(request.install.key_file) + : undefined; + const { name, files } = renderBuildBundle(request, payload, key); + const workspace = await PrivateRuntimeRoot.create({ + prefix: "rea-build-computer-", + }); + try { + const bundle = join(workspace.path, "bundle"); + for (const file of files) { + const target = join(bundle, file.path); + await mkdir(dirname(target), { recursive: true }); + if ( + typeof file.content === "string" || + file.content instanceof Uint8Array + ) + await writeFile(target, file.content); + else await copyFile(file.content.copyFrom, target); + } + return { workspace, bundle, name }; + } catch (cause: unknown) { + await workspace.close(); + throw cause; + } +}; + +/** + * Render a request's build bundle, with REA's payload from the operator's + * configuration, into a private folder the caller owns and removes. + */ +export const writeRequestBundle = async ( + request: BuildRequest, + environment: Readonly, +) => writeBuildBundle(request, await reaPayload(environment)); diff --git a/src/application/desktop/DesktopBuildComputers.ts b/src/application/desktop/DesktopBuildComputers.ts new file mode 100644 index 0000000000..a6b8577571 --- /dev/null +++ b/src/application/desktop/DesktopBuildComputers.ts @@ -0,0 +1,415 @@ +import { randomUUID } from "node:crypto"; + +import type { BuildSource } from "../../domain/desktop/appBuild.js"; +import { + driverCall, + type ComputerAction, + type ComputerActionInput, + type ScreenshotOptions, +} from "../../domain/desktop/computerActions.js"; +import { + AnalysisError, + type AnalysisCapturedOutput, +} from "../../domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisInputError, +} from "../../domain/analysisErrorCore.js"; +import type { JsonValue } from "../../domain/jsonValue.js"; +import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; +import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; +import { err, ok, type Result } from "../../domain/result.js"; +import { + AgentEnvComputer, + ComputerStartFailure, + type DriverResult, +} from "../../desktop/AgentEnvComputer.js"; +import { + AGENTENV_PROVIDER, + ComputerHelper, + type ComputerRelease, +} from "../../desktop/AgentEnvComputerHelper.js"; +import type { PrivateRuntimeRoot } from "../../process/PrivateRuntimeRoot.js"; +import { + BuildComputerActionLog, + type ReadonlyActionLog, +} from "./DesktopBuildActionLog.js"; +import { writeRequestBundle } from "./DesktopBuildBundle.js"; + +const START = "start_build_computer"; +const ACT = "act_on_build_computer"; +const CLOSE = "close_build_computer"; + +/** A result with the driver's content blocks, delivered after REA's own. */ +export interface ForwardedResult { + readonly value: Value; + readonly content: DriverResult["content"]; +} + +/** What later stages read of a live build computer: its action log and recording hooks. */ +export interface BuildComputerSession { + readonly sandboxId: string; + readonly actions: ReadonlyActionLog; + readonly startRecording: AgentEnvComputer["startRecording"]; + readonly endRecording: AgentEnvComputer["endRecording"]; + readonly recordingSegments: AgentEnvComputer["recordingSegments"]; +} + +/** A computer REA owns: a live one, or a failed start whose cleanup is still unconfirmed. */ +interface HeldComputer { + readonly computer: AgentEnvComputer | undefined; + /** Brings the computer down; absent when no helper was spawned. */ + readonly release: (() => Promise) | undefined; + readonly workspace: PrivateRuntimeRoot | undefined; + readonly actions: BuildComputerActionLog; + closing: boolean; +} + +/** A start still in progress, which shutdown cancels and joins. */ +interface PendingStart { + readonly settled: Promise; + readonly stop: AbortController; +} + +const unknownComputer = (operation: string) => + new AnalysisInputError(operation, { + userMessage: + "No build computer has this id in this REA server; start one with start_build_computer.", + conformsToInputContract: true, + }); + +const failure = ( + operation: string, + cause: unknown, + signal: AbortSignal, +): AnalysisError => { + if (cause instanceof AnalysisError) return cause; + if (signal.aborted) + return new AnalysisCancelledError(operation, { cause: signal.reason }); + return new ProviderAdapterError(AGENTENV_PROVIDER, operation, { + userMessage: cause instanceof Error ? cause.message : String(cause), + cause, + }); +}; + +/** + * Why a start failed, with the helper's output, and, when its teardown is + * unconfirmed, the id REA keeps that teardown under for close to retry. + */ +const startFailure = ( + cause: unknown, + signal: AbortSignal, + output: AnalysisCapturedOutput | undefined, + kept: { readonly id: string; readonly release: ComputerRelease }, +): AnalysisError => { + if (cause instanceof AnalysisError && kept.release.verified) return cause; + const options = { + cause, + ...(output === undefined ? {} : { capturedOutput: output }), + ...(kept.release.verified + ? {} + : { + cleanup: { + reason: kept.release.reason, + resources: kept.release.resources, + }, + diagnostics: { computer_id: kept.id }, + remediationAction: `REA keeps this computer's teardown as ${kept.id}; once the reported problem is fixed, call close_build_computer with that computer_id.`, + }), + }; + if (signal.aborted) + return new AnalysisCancelledError(START, { + ...options, + cause: signal.reason, + }); + return new ProviderAdapterError(AGENTENV_PROVIDER, START, { + ...(cause instanceof ComputerStartFailure && + cause.remediation !== undefined && + kept.release.verified + ? { remediationAction: cause.remediation } + : {}), + userMessage: cause instanceof Error ? cause.message : String(cause), + ...options, + }); +}; + +/** + * Owns the build computers one REA server keeps up: starts each from a + * request or a bundle, forwards actions to its driver with an action log, + * and brings each down on close or when the server closes. A start in + * progress is owned from its helper's spawn; one that fails without a + * confirmed teardown stays held so close and shutdown can retry it. + */ +export class DesktopBuildComputers { + readonly #computers = new Map(); + readonly #starts = new Set(); + #closed = false; + + constructor(private readonly environment: Readonly) {} + + async start( + input: { + readonly source: BuildSource; + readonly screenshot?: ScreenshotOptions | undefined; + }, + signal: AbortSignal, + ): Promise< + Result< + ForwardedResult<{ + readonly computer_id: string; + readonly sandbox_id: string; + readonly environment: string; + readonly screenshot: Readonly> | null; + }>, + AnalysisError + > + > { + if (this.#closed) + return err( + new AnalysisCancelledError(START, { + cause: new Error("REA is closing its build computers"), + }), + ); + const stop = new AbortController(); + const settled = this.#start(input, AbortSignal.any([signal, stop.signal])); + const pending = { settled, stop }; + this.#starts.add(pending); + try { + return await settled; + } finally { + this.#starts.delete(pending); + } + } + + async #start( + input: Parameters[0], + signal: AbortSignal, + ): ReturnType { + const { source } = input; + let target: { + readonly bundle: string; + readonly name: string; + readonly workspace?: PrivateRuntimeRoot; + }; + try { + target = + source.kind === "bundle" + ? { bundle: source.path, name: source.path } + : await writeRequestBundle(source, this.environment); + } catch (cause: unknown) { + return err(failure(START, cause, signal)); + } + const id = randomUUID(); + let helper: ComputerHelper | undefined; + let computer: AgentEnvComputer | undefined; + try { + helper = await ComputerHelper.spawn({ + bundle: target.bundle, + environment: this.environment, + signal, + operation: START, + }); + computer = await AgentEnvComputer.connect(helper, signal); + const shot = driverCall("screenshot", input.screenshot ?? {}); + if (!shot.ok) throw new TypeError("screenshot options are its own"); + const taken = await computer.callTool( + shot.value.tool, + shot.value.arguments, + signal, + ); + const live = computer; + this.#computers.set(id, { + computer: live, + release: () => live.close(), + workspace: target.workspace, + actions: new BuildComputerActionLog(), + closing: false, + }); + return ok({ + value: { + computer_id: id, + sandbox_id: live.sandboxId, + environment: target.name, + screenshot: taken.isError ? null : taken.structured, + }, + content: taken.content, + }); + } catch (cause: unknown) { + const owner = helper; + const connected = computer; + this.#computers.set(id, { + computer: undefined, + release: + connected !== undefined + ? () => connected.close() + : owner === undefined + ? undefined + : () => owner.release(), + workspace: target.workspace, + actions: new BuildComputerActionLog(), + closing: true, + }); + const release = await this.#retire(id); + return err(startFailure(cause, signal, owner?.output(), { id, release })); + } + } + + /** Forward one action to the computer's driver and log it; a driver refusal is an error carrying its message. */ + async act( + input: ComputerActionInput, + signal: AbortSignal, + ): Promise< + Result< + ForwardedResult<{ + readonly computer_id: string; + readonly action: ComputerAction; + readonly action_index: number; + readonly result: Readonly> | null; + }>, + AnalysisError + > + > { + const { computer_id: id, action, ...fields } = input; + const held = this.#computers.get(id); + const computer = held?.computer; + if (held === undefined || computer === undefined || held.closing) + return err(unknownComputer(ACT)); + const call = driverCall(action, fields); + if (!call.ok) + return err( + new AnalysisInputError( + ACT, + { + userMessage: `${action} does not take ${call.error.field}; it takes ${call.error.accepted.join(", ")}.`, + }, + [ + { + path: [call.error.field], + reason: "unknown_argument", + expected: [...call.error.accepted], + }, + ], + ), + ); + const logged = held.actions.begin( + action, + call.value.tool, + call.value.arguments, + ); + let result: DriverResult; + try { + result = await computer.callTool( + call.value.tool, + call.value.arguments, + signal, + ); + } catch (cause: unknown) { + logged.settle("failed"); + return err(failure(ACT, cause, signal)); + } + logged.settle(result.isError ? "refused" : "applied"); + if (result.isError) + return err( + new AnalysisInputError(ACT, { + userMessage: `The computer refused the ${action}: ${result.content + .flatMap((block) => + typeof block.text === "string" ? [block.text] : [], + ) + .join(" ")}`, + remediationAction: + "Follow the computer's message; actions on frame coordinates need a screenshot first.", + conformsToInputContract: true, + }), + ); + return ok({ + value: { + computer_id: id, + action, + action_index: logged.index, + result: result.structured, + }, + content: result.content, + }); + } + + /** Bring a computer down; one whose teardown is not confirmed stays held so close can be retried. */ + async close(id: string): Promise< + Result< + { + readonly computer_id: string; + readonly teardown: "agentenv" | "container_label"; + }, + AnalysisError + > + > { + if (!this.#computers.has(id)) return err(unknownComputer(CLOSE)); + const release = await this.#retire(id); + return release.verified + ? ok({ computer_id: id, teardown: release.teardown }) + : err( + new ProviderCleanupError( + AGENTENV_PROVIDER, + release.resources, + { reason: release.reason }, + { operation: CLOSE, remediationAction: release.reason }, + ), + ); + } + + /** Release a held computer and its bundle folder; it stays held while either is unconfirmed. */ + async #retire(id: string): Promise { + const held = this.#computers.get(id); + if (held === undefined) return { verified: true, teardown: "agentenv" }; + held.closing = true; + const release = (await held.release?.()) ?? { + verified: true, + teardown: "agentenv", + }; + if (!release.verified) return release; + try { + await held.workspace?.close(); + } catch (cause: unknown) { + return { + verified: false, + reason: `the build bundle folder could not be removed: ${cause instanceof Error ? cause.message : String(cause)}`, + resources: [held.workspace?.path ?? id], + }; + } + this.#computers.delete(id); + return release; + } + + /** Cancel and join the starts in progress, then bring every held computer down, as the server closes. */ + async closeAll(): Promise { + this.#closed = true; + const starts = [...this.#starts]; + for (const start of starts) + start.stop.abort(new Error("REA is closing its build computers")); + await Promise.allSettled(starts.map((start) => start.settled)); + const results = await Promise.all( + [...this.#computers.keys()].map((id) => this.close(id)), + ); + const failures = results.flatMap((result) => + result.ok ? [] : [result.error], + ); + if (failures.length > 0) + throw new AggregateError( + failures, + "Build computer teardown is incomplete", + ); + } + + /** A live computer's action log and recording hooks, for the witness. */ + session(id: string): BuildComputerSession | undefined { + const held = this.#computers.get(id); + const computer = held?.computer; + if (held === undefined || computer === undefined) return undefined; + return { + sandboxId: computer.sandboxId, + actions: held.actions, + startRecording: (...request) => computer.startRecording(...request), + endRecording: (...request) => computer.endRecording(...request), + recordingSegments: (...request) => computer.recordingSegments(...request), + }; + } +} diff --git a/src/composition/capabilityDeclarations.ts b/src/composition/capabilityDeclarations.ts index df2073a19d..e801eb65c1 100644 --- a/src/composition/capabilityDeclarations.ts +++ b/src/composition/capabilityDeclarations.ts @@ -227,6 +227,7 @@ export const CAPABILITY_DECLARATIONS = { }), ], }, + desktopBuildComputers: { providers: [] }, devicetree: { providers: [providerBinding(DEVICETREE_PROVIDER, ["inspect_devicetree"])], }, diff --git a/src/composition/capabilityServices.ts b/src/composition/capabilityServices.ts index 15fdc65471..0b238bf2d5 100644 --- a/src/composition/capabilityServices.ts +++ b/src/composition/capabilityServices.ts @@ -26,6 +26,7 @@ import { } from "./evm.js"; import { createSqliteDatabaseService } from "./sqlite.js"; import { createDesktopApplicationService } from "./desktop.js"; +import { DesktopBuildComputers } from "../application/desktop/DesktopBuildComputers.js"; import { createGoBinaryService } from "./go.js"; import { createRustBinaryService } from "./rust.js"; import { createBeamService } from "./erlang.js"; @@ -318,6 +319,10 @@ const modules = { }), ), desktopApplication: service(() => createDesktopApplicationService()), + desktopBuildComputers: capability( + ({ providerEnvironment }) => new DesktopBuildComputers(providerEnvironment), + (value) => value.closeAll(), + ), goBinary: service( ({ options }) => options.goBinary ?? createGoBinaryService(), ), diff --git a/src/contracts/desktopBuildToolContracts.ts b/src/contracts/desktopBuildToolContracts.ts new file mode 100644 index 0000000000..b50e3609d1 --- /dev/null +++ b/src/contracts/desktopBuildToolContracts.ts @@ -0,0 +1,114 @@ +import { z } from "zod"; + +import { buildSourceSchema } from "../domain/desktop/appBuild.js"; +import { + COMPUTER_ACTIONS, + computerActionInputSchema, + screenshotOptionsSchema, +} from "../domain/desktop/computerActions.js"; +import { jsonObjectSchema } from "../domain/jsonValue.js"; +import type { ToolContract } from "./toolContractTypes.js"; +import { toolContractMetadata } from "./toolEffects.js"; + +const computerId = z.uuid().describe("The id start_build_computer returned."); +const driverResult = jsonObjectSchema + .nullable() + .describe( + "The driver's structured result, such as frame_id, width and height; null when it returned none.", + ); +const EXAMPLE_ID = "00000000-0000-4000-8000-000000000001"; + +/** Session tools that keep an agentenv computer up and forward computer-use actions to it while an environment is built. */ +export const DESKTOP_BUILD_TOOL_CONTRACTS = [ + { + name: "start_build_computer", + ...toolContractMetadata("start_build_computer"), + kind: "application", + description: + "Start a local agentenv computer to build a desktop app environment on, and return its id with a first screenshot as an image. From a request, REA renders a bundle that installs the app from the Ubuntu archive, an apt repository or a .deb URL, opens it on demand from its desktop entry (it is not open at first: open it from the desktop's Applications menu, under Other, by the environment's name), and installs REA itself in a build-only layer from REA_BUILD_COMPUTER_REA_ARCHIVE; from a bundle folder, such as an exported one, it starts that bundle's smoke task as it is. Needs Docker and the agentenv runner named by REA_AGENT_ENV_COMMAND; the first start of a bundle builds its image, which downloads packages and can take minutes. screenshot sets the first screenshot's region, format and quality. The computer stays up, reachable on this host, until close_build_computer or the server closes; when a start fails before its teardown is confirmed, the error's computer_id names what close_build_computer retries.", + inputSchema: z.strictObject({ + source: buildSourceSchema, + screenshot: screenshotOptionsSchema.optional(), + }), + outputSchema: z.strictObject({ + computer_id: z.uuid(), + sandbox_id: z.string().describe("agentenv's sandbox id."), + environment: z + .string() + .describe("The rendered environment's name, or the bundle folder."), + screenshot: driverResult, + }), + examples: [ + { + title: "Build Mousepad from the Ubuntu archive", + input: { + source: { + kind: "request", + desktop_entry: "org.xfce.mousepad", + install: { kind: "archive", package: "mousepad" }, + }, + }, + }, + { + title: "Start an exported bundle", + input: { source: { kind: "bundle", path: "/tmp/mousepad-bundle" } }, + }, + ], + }, + { + name: "act_on_build_computer", + ...toolContractMetadata("act_on_build_computer"), + kind: "application", + description: `Drive the app on a build computer: one action, ${COMPUTER_ACTIONS.join(", ")}, sent to the computer's computer_ tool with the fields given, which are the driver's own arguments. Points are pixels of the latest frame, so take a screenshot first, or space norm on a 0..norm_scale grid; count 2 double-clicks; keys names one chord in W3C names, and spellings such as ctrl+s are translated. Returns the driver's content unchanged, its images included, and its structured result; an action's screenshot is taken at once, so after one that opens a menu or a window, take it with a wait instead; a refusal, such as a stale frame, is an error with the computer's message. Each action is logged on the computer.`, + inputSchema: computerActionInputSchema, + outputSchema: z.strictObject({ + computer_id: z.uuid(), + action: z.enum(COMPUTER_ACTIONS), + action_index: z + .int() + .nonnegative() + .describe("The action's place in the computer's action log."), + result: driverResult, + }), + examples: [ + { + title: "Look at the screen", + input: { computer_id: EXAMPLE_ID, action: "screenshot" }, + }, + { + title: "Double-click a point of the latest frame", + input: { + computer_id: EXAMPLE_ID, + action: "click", + x: 640, + y: 400, + count: 2, + }, + }, + { + title: "Save", + input: { computer_id: EXAMPLE_ID, action: "key", keys: ["ctrl+s"] }, + }, + ], + }, + { + name: "close_build_computer", + ...toolContractMetadata("close_build_computer"), + kind: "application", + description: + "Tear a build computer down and discard its state. agentenv's teardown is tried first; if it leaves the computer up, REA removes that exact sandbox's containers, the one named agent- and any labelled agentenv.sandbox=, with REA_DOCKER_COMMAND. When neither confirms it, the error names what is left and the computer stays listed, so closing again retries.", + inputSchema: z.strictObject({ computer_id: computerId }), + outputSchema: z.strictObject({ + computer_id: z.uuid(), + teardown: z + .enum(["agentenv", "container_label"]) + .describe("What confirmed the computer is gone."), + }), + examples: [ + { + title: "Close a build computer", + input: { computer_id: EXAMPLE_ID }, + }, + ], + }, +] as const satisfies readonly ToolContract[]; diff --git a/src/contracts/toolContracts.ts b/src/contracts/toolContracts.ts index 4d0ebcba60..00e213b942 100644 --- a/src/contracts/toolContracts.ts +++ b/src/contracts/toolContracts.ts @@ -6,6 +6,7 @@ import { WASM_TOOL_CONTRACTS } from "./wasm/wasmToolContracts.js"; import { APK_SIGNATURE_TOOL_CONTRACTS } from "./apkSignature/apkSignatureToolContracts.js"; import { EVM_TOOL_CONTRACTS } from "./evm/evmToolContracts.js"; import { DESKTOP_APPLICATION_TOOL_CONTRACTS } from "./desktopApplicationToolContracts.js"; +import { DESKTOP_BUILD_TOOL_CONTRACTS } from "./desktopBuildToolContracts.js"; import { GO_TOOL_CONTRACTS } from "./go/goToolContracts.js"; import { MODEL_TOOL_CONTRACTS } from "./model/modelToolContracts.js"; import { IL2CPP_TOOL_CONTRACTS } from "./il2cpp/il2cppToolContracts.js"; @@ -84,6 +85,7 @@ export const TOOL_CONTRACTS = [ ...WEB_RUNTIME_TOOL_CONTRACTS, ...WEB_NETWORK_CAPTURE_TOOL_CONTRACTS, ...ENVIRONMENT_TOOL_CONTRACTS, + ...DESKTOP_BUILD_TOOL_CONTRACTS, ...JAVASCRIPT_RECOVERY_TOOL_CONTRACTS, ...SESSION_TOOL_CONTRACTS, ...CUTTER_TOOL_CONTRACTS, diff --git a/src/contracts/toolEffects.test.ts b/src/contracts/toolEffects.test.ts index b138e74c0f..480044292b 100644 --- a/src/contracts/toolEffects.test.ts +++ b/src/contracts/toolEffects.test.ts @@ -176,6 +176,7 @@ describe("tool effects", () => { ({ effects }) => effects.executesForeignCode || effects.mutatesDevice, ); expect(destructive.map(({ name }) => name).sort()).toEqual([ + "act_on_build_computer", "capture_browser_scenario", "capture_electron_scenario", "capture_native_ui_scenario", @@ -195,6 +196,7 @@ describe("tool effects", () => { "rizin_debug_command", "start_adb_activity", "start_adb_app", + "start_build_computer", "start_rizin_debug_session", "stop_adb_app", "uninstall_adb_package", diff --git a/src/contracts/toolEffects.ts b/src/contracts/toolEffects.ts index 23cd952687..4076200999 100644 --- a/src/contracts/toolEffects.ts +++ b/src/contracts/toolEffects.ts @@ -735,6 +735,30 @@ const TOOL_EFFECT_AUDIT: Readonly> = { writesFilesystem: true, idempotent: false, }), + // Builds and boots a container that installs and runs the requested app. + start_build_computer: effects({ + mutatesSession: true, + writesFilesystem: true, + launchesProcess: true, + accessesNetwork: true, + executesForeignCode: true, + idempotent: false, + }), + // Drives the app through the computer's driver on a loopback port. + act_on_build_computer: effects({ + mutatesSession: true, + accessesNetwork: true, + changesUiState: true, + executesForeignCode: true, + idempotent: false, + }), + close_build_computer: effects({ + mutatesSession: true, + launchesProcess: true, + accessesNetwork: true, + mayDiscardData: true, + idempotent: false, + }), reconcile_javascript_runtime: evidence, recover_javascript_sources: effects({ mutatesSession: true, diff --git a/src/contracts/toolFamilies.ts b/src/contracts/toolFamilies.ts index b89851d2a0..034d0919c6 100644 --- a/src/contracts/toolFamilies.ts +++ b/src/contracts/toolFamilies.ts @@ -20,6 +20,7 @@ import { APK_SIGNATURE_TOOL_CONTRACTS } from "./apkSignature/apkSignatureToolCon import { ERLANG_BEAM_TOOL_CONTRACTS } from "./erlang/beamToolContracts.js"; import { PYTHON_PYC_TOOL_CONTRACTS } from "./python/pythonPycToolContracts.js"; import { DESKTOP_APPLICATION_TOOL_CONTRACTS } from "./desktopApplicationToolContracts.js"; +import { DESKTOP_BUILD_TOOL_CONTRACTS } from "./desktopBuildToolContracts.js"; import { IL2CPP_TOOL_CONTRACTS } from "./il2cpp/il2cppToolContracts.js"; import { MODEL_TOOL_CONTRACTS } from "./model/modelToolContracts.js"; import { MANAGED_TOOL_CONTRACTS } from "./managed/managedToolContracts.js"; @@ -65,6 +66,7 @@ export const TOOL_CONTRACT_GROUPS = { CAPTURE_TOOL_CONTRACTS, CUTTER_TOOL_CONTRACTS, DESKTOP_APPLICATION_TOOL_CONTRACTS, + DESKTOP_BUILD_TOOL_CONTRACTS, ELECTRON_TOOL_CONTRACTS, ENHANCED_TOOL_CONTRACTS, ENVIRONMENT_TOOL_CONTRACTS, @@ -232,6 +234,7 @@ export const TOOL_FAMILY_DEFINITIONS = [ ...TOOL_CONTRACT_GROUPS.APPLICATION_TOOL_CONTRACTS, ...TOOL_CONTRACT_GROUPS.ANALYSIS_SETTINGS_TOOL_CONTRACTS, ...TOOL_CONTRACT_GROUPS.ENVIRONMENT_TOOL_CONTRACTS, + ...TOOL_CONTRACT_GROUPS.DESKTOP_BUILD_TOOL_CONTRACTS, ...TOOL_CONTRACT_GROUPS.ANALYSIS_VIEW_TOOL_CONTRACTS, ...TOOL_CONTRACT_GROUPS.WEB_SCRIPT_TOOL_CONTRACTS, ...TOOL_CONTRACT_GROUPS.JAVASCRIPT_RECOVERY_TOOL_CONTRACTS, diff --git a/src/desktop/AgentEnvComputer.ts b/src/desktop/AgentEnvComputer.ts new file mode 100644 index 0000000000..4436273cd3 --- /dev/null +++ b/src/desktop/AgentEnvComputer.ts @@ -0,0 +1,365 @@ +import { createHash } from "node:crypto"; +import { setTimeout as delay } from "node:timers/promises"; +import { getHeapStatistics } from "node:v8"; + +import { StreamableHTTPClientTransport } from "@modelcontextprotocol/client"; +import { z } from "zod"; + +import { jsonObjectSchema, type JsonValue } from "../domain/jsonValue.js"; +import { StructuredContentMcpClient } from "../providers/StructuredContentMcpClient.js"; +import { validateMcpHttpResponse } from "../providers/Utf8McpResponse.js"; +import type { + ComputerHelper, + ComputerRelease, +} from "./AgentEnvComputerHelper.js"; + +const READY_TIMEOUT_MS = 180_000; +const REQUEST_TIMEOUT_MS = 120_000; +const RECORDINGS = ".agentenv-recordings"; +/** A control route's answer: readiness, or a recording's manifest, near 1.5 MB for a day of 15 s segments. */ +const CONTROL_RESPONSE_BYTES = 4 * 1024 * 1024; +/** What a data/get answer holds around one file's base64 bytes. */ +const ENVELOPE_BYTES = 64 * 1024; + +const readinessSchema = z.object({ + ready: z.boolean(), + checks: z + .array( + z.object({ + name: z.string(), + ok: z.boolean(), + detail: z.string().nullish(), + final: z.boolean().optional(), + }), + ) + .default([]), +}); +const recordingId = z.string().regex(/^rec-[0-9a-f]+$/u); +const recordingStartSchema = z.looseObject({ + recording_id: recordingId, + started_at: z.string(), +}); +const recordingManifestSchema = z.looseObject({ + recording_id: recordingId, + segments: z.array( + z.looseObject({ + index: z.int().nonnegative(), + path: z.string(), + bytes: z.int().nonnegative(), + sha256: z.string().regex(/^[0-9a-f]{64}$/u), + }), + ), +}); +/** An ended recording's manifest, as `computer-recording/v1` returns it. */ +export type RecordingManifest = z.infer; +const dataGetSchema = z.union([ + z.object({ + result: z.object({ + parts: z.array( + z.looseObject({ + file: z.looseObject({ bytes: z.string().optional() }).optional(), + metadata: z + .record(z.string(), z.looseObject({ path: z.string().optional() })) + .optional(), + }), + ), + }), + }), + z.object({ error: z.looseObject({ message: z.string() }) }), +]); + +/** A driver tool's answer as REA forwards it: its content blocks unchanged and its structured result. */ +export interface DriverResult { + readonly isError: boolean; + readonly content: readonly Readonly>[]; + readonly structured: Readonly> | null; +} + +/** A response body read only while it stays within `maxBytes`. */ +const readBounded = async ( + response: Response, + maxBytes: number, + what: string, +): Promise => { + const chunks: Uint8Array[] = []; + let bytes = 0; + if (response.body !== null) + for await (const chunk of response.body) { + bytes += chunk.byteLength; + if (bytes > maxBytes) + throw new Error(`${what} is more than ${String(maxBytes)} bytes.`); + chunks.push(chunk); + } + return Buffer.concat(chunks, bytes).toString("utf8"); +}; + +/** Why a computer did not come up, with what the caller can do about it. */ +export class ComputerStartFailure extends Error { + constructor( + message: string, + readonly remediation: string | undefined, + options?: ErrorOptions, + ) { + super(message, options); + this.name = "ComputerStartFailure"; + } +} + +const connectDriver = async ( + mcpUrl: string, + signal: AbortSignal, +): Promise => { + const client = new StructuredContentMcpClient({ + name: "rea-build-computer", + version: "1", + }); + const transport: StreamableHTTPClientTransport = + new StreamableHTTPClientTransport(new URL(mcpUrl), { + requestInit: { redirect: "error" }, + fetch: async (input, init) => + validateMcpHttpResponse(await fetch(input, init), () => { + void transport.close().catch(() => undefined); + }), + }); + await client.connect(transport, { timeout: REQUEST_TIMEOUT_MS, signal }); + return client; +}; + +/** + * One agentenv computer kept up for REA: the helper that holds it, one MCP + * session to its driver, and its control token, which never leaves this + * object. + */ +export class AgentEnvComputer { + readonly sandboxId: string; + readonly #baseUrl: string; + readonly #token: string | null; + readonly #helper: ComputerHelper; + readonly #client: StructuredContentMcpClient; + + private constructor(parts: { + readonly sandboxId: string; + readonly baseUrl: string; + readonly token: string | null; + readonly helper: ComputerHelper; + readonly client: StructuredContentMcpClient; + }) { + this.sandboxId = parts.sandboxId; + this.#baseUrl = parts.baseUrl; + this.#token = parts.token; + this.#helper = parts.helper; + this.#client = parts.client; + } + + /** + * Connect to the computer a helper keeps up: wait for its handshake and + * readiness and open an MCP session to its driver. A failure throws a + * `ComputerStartFailure`, or the signal's reason when cancelled; the + * helper's owner releases it either way. + */ + static async connect( + helper: ComputerHelper, + signal: AbortSignal, + ): Promise { + const handshake = await helper.handshake(signal); + if (handshake === undefined || "error" in handshake) + throw new ComputerStartFailure( + `agentenv did not bring the computer up: ${handshake?.error ?? "its helper ended without a handshake"}.`, + "Read the helper's output; `agent-env run --task smoke --no-emulate` reproduces the deploy.", + ); + let client: StructuredContentMcpClient | undefined; + try { + client = await connectDriver(handshake.mcp_url, signal); + const computer = new AgentEnvComputer({ + sandboxId: handshake.sandbox_id, + baseUrl: handshake.base_url.replace(/\/+$/u, ""), + token: handshake.control_token, + helper, + client, + }); + await computer.#waitReady(signal); + return computer; + } catch (cause: unknown) { + await client?.close().catch(() => undefined); + if (signal.aborted) throw signal.reason; + throw new ComputerStartFailure( + `The computer came up but REA could not use it: ${cause instanceof Error ? cause.message : String(cause)}`, + undefined, + { cause }, + ); + } + } + + /** Call one of the driver's `computer_*` tools on REA's MCP session. */ + async callTool( + name: string, + args: Readonly>, + signal: AbortSignal, + ): Promise { + const result = await this.#client.callTool( + { name, arguments: args }, + { timeout: REQUEST_TIMEOUT_MS, signal }, + ); + const structured = jsonObjectSchema.safeParse(result.structuredContent); + return { + isError: result.isError === true, + content: result.content.map((block) => jsonObjectSchema.parse(block)), + structured: structured.success ? structured.data : null, + }; + } + + /** Start a recording under the caller's request id; repeating the id answers the same recording. */ + async startRecording(requestId: string, signal?: AbortSignal) { + return recordingStartSchema.parse( + await this.#request( + "/agentenv/ext/computer-recording/start", + { request_id: requestId }, + signal, + ), + ); + } + + /** End a recording; its manifest lists the segments the computer keeps until teardown. */ + async endRecording( + id: string, + signal?: AbortSignal, + ): Promise { + return recordingManifestSchema.parse( + await this.#request( + "/agentenv/ext/computer-recording/end", + { recording_id: id }, + signal, + ), + ); + } + + /** + * Fetch an ended recording's segments through the driver's data plane, + * one request per segment, each read within its manifest size and checked + * against it. A recording larger than `maxBytes`, by default a quarter of + * the V8 heap as for retained Evidence, is refused before any read. + */ + async recordingSegments( + manifest: RecordingManifest, + signal?: AbortSignal, + maxBytes = Math.floor(getHeapStatistics().heap_size_limit / 4), + ): Promise { + const total = manifest.segments.reduce( + (sum, segment) => sum + segment.bytes, + 0, + ); + if (total > maxBytes) + throw new Error( + `Recording ${manifest.recording_id} holds ${String(total)} bytes, more than the ${String(maxBytes)} REA reads into memory for one recording; end recordings sooner so each stays smaller.`, + ); + const segments: { readonly path: string; readonly bytes: Buffer }[] = []; + for (const segment of manifest.segments) + segments.push({ + path: segment.path, + bytes: await this.#segment(segment, signal), + }); + return segments; + } + + async #segment( + segment: RecordingManifest["segments"][number], + signal: AbortSignal | undefined, + ): Promise { + const path = `${RECORDINGS}/${segment.path}`; + const answer = dataGetSchema.parse( + await this.#request( + "/agentenv", + { + jsonrpc: "2.0", + id: segment.index + 1, + method: "data/get", + params: { paths: [path] }, + }, + signal, + { + // Base64 plus the JSON-RPC envelope and the part's metadata. + bytes: Math.ceil(segment.bytes / 3) * 4 + ENVELOPE_BYTES, + what: `The computer's answer for recording segment ${segment.path}`, + }, + ), + ); + if ("error" in answer) + throw new Error( + `data/get of recording segment ${segment.path} failed: ${answer.error.message}`, + ); + const encoded = answer.result.parts.find( + (part) => Object.values(part.metadata ?? {})[0]?.path === path, + )?.file?.bytes; + const bytes = + encoded === undefined ? undefined : Buffer.from(encoded, "base64"); + if ( + bytes?.length !== segment.bytes || + createHash("sha256").update(bytes).digest("hex") !== segment.sha256 + ) + throw new Error( + `Recording segment ${segment.path} is missing or differs from its manifest.`, + ); + return bytes; + } + + /** End REA's MCP session and bring the computer down; a failed release can be retried. */ + async close(): Promise { + await this.#client.close().catch(() => undefined); + return this.#helper.release(); + } + + async #waitReady(signal: AbortSignal): Promise { + const deadline = Date.now() + READY_TIMEOUT_MS; + for (;;) { + const status = readinessSchema.parse( + await this.#request( + "/agentenv/ext/computer-ready/status", + undefined, + signal, + ), + ); + if (status.ready) return; + const waiting = status.checks.filter((check) => !check.ok); + const failed = waiting.find((check) => check.final === true); + if (failed !== undefined || Date.now() > deadline) + throw new Error( + failed === undefined + ? `The computer was not ready within ${String(READY_TIMEOUT_MS / 1000)} s; waiting on ${waiting.map((check) => check.name).join(", ")}.` + : `The computer's ${failed.name} check failed: ${failed.detail ?? "no detail"}.`, + ); + await delay(500, undefined, { signal }); + } + } + + /** A framework-only route of the computer, with its control token; its answer is read within `limit`. */ + async #request( + path: string, + body: unknown, + signal: AbortSignal | undefined, + limit = { + bytes: CONTROL_RESPONSE_BYTES, + what: `The computer's answer to ${path}`, + }, + ): Promise { + const timeout = AbortSignal.timeout(REQUEST_TIMEOUT_MS); + const response = await fetch(`${this.#baseUrl}${path}`, { + method: body === undefined ? "GET" : "POST", + headers: { + ...(this.#token === null + ? {} + : { authorization: `Bearer ${this.#token}` }), + ...(body === undefined ? {} : { "content-type": "application/json" }), + }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + redirect: "error", + signal: + signal === undefined ? timeout : AbortSignal.any([signal, timeout]), + }); + const text = await readBounded(response, limit.bytes, limit.what); + if (!response.ok) + throw new Error( + `The computer answered ${path} with HTTP ${String(response.status)}: ${text.slice(0, 512)}`, + ); + return JSON.parse(text) as unknown; + } +} diff --git a/src/desktop/AgentEnvComputerHelper.ts b/src/desktop/AgentEnvComputerHelper.ts new file mode 100644 index 0000000000..4548cdf871 --- /dev/null +++ b/src/desktop/AgentEnvComputerHelper.ts @@ -0,0 +1,351 @@ +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { constants } from "node:fs"; +import { access, open, realpath } from "node:fs/promises"; +import { basename, dirname, isAbsolute, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +import { z } from "zod"; + +import { AnalysisCapabilityUnavailableError } from "../domain/analysisErrorCore.js"; +import type { AnalysisCapturedOutput } from "../domain/analysisErrorBase.js"; +import { readBoundedControlLines } from "../process/BoundedControlLines.js"; +import { cleanupOwnedProcessGroup } from "../process/ProcessOwnership.js"; +import { + ProviderProcessSupervisor, + spawnOwnedProviderProcess, + type SpawnedOwnedProviderProcess, +} from "../process/ProviderProcess.js"; + +const execFileAsync = promisify(execFile); + +export const AGENTENV_PROVIDER = "agentenv"; +const HELPER = fileURLToPath( + new URL("../../bridge/agentenv/computer.py", import.meta.url), +); +const HANDSHAKE_BYTES = 64 * 1024; +const STDERR_BYTES = 16 * 1024; +const TEARDOWN_MS = 120_000; +const RUNNER_REMEDIATION = + "Set REA_AGENT_ENV_COMMAND to the absolute path of the agent-env program of an agentenv-framework install (a venv, uv tool or pipx), then restart REA."; + +const handshakeSchema = z.union([ + z.strictObject({ + base_url: z.url({ protocol: /^https?$/u }), + mcp_url: z.url({ protocol: /^https?$/u }), + control_token: z.string().min(1).nullable(), + sandbox_id: z.string().min(1), + }), + z.strictObject({ error: z.string() }), +]); +/** The helper's one JSON line: where the kept computer is, or why there is none. */ +export type Handshake = z.infer; + +/** How a computer's teardown ended: confirmed by agentenv or by removing its containers, or not confirmed. */ +export type ComputerRelease = + | { + readonly verified: true; + readonly teardown: "agentenv" | "container_label"; + } + | { + readonly verified: false; + readonly reason: string; + readonly resources: readonly string[]; + }; + +const unavailable = (operation: string, reason: string, cause?: unknown) => + new AnalysisCapabilityUnavailableError(AGENTENV_PROVIDER, operation, reason, { + userMessage: reason, + remediationAction: RUNNER_REMEDIATION, + ...(cause === undefined ? {} : { cause }), + }); + +const executable = async (path: string): Promise => + access(path, constants.X_OK).then( + () => true, + () => false, + ); + +const readHead = async (path: string): Promise => { + const file = await open(path, "r"); + try { + const { buffer, bytesRead } = await file.read(Buffer.alloc(512), 0, 512, 0); + return buffer.subarray(0, bytesRead).toString("utf8"); + } finally { + await file.close(); + } +}; + +/** + * The Python the agentenv runner runs with, so the helper can import + * `agent_env`: the interpreter its console script names on its `#!` line, + * or the `python3` beside the resolved script in its venv. + */ +export const agentEnvInterpreter = async ( + environment: Readonly, + operation: string, +): Promise => { + const command = environment.REA_AGENT_ENV_COMMAND; + if (command === undefined || !isAbsolute(command)) + throw unavailable( + operation, + "Build computers need REA_AGENT_ENV_COMMAND, the absolute path of agentenv's agent-env program; REA runs its helper with that runner's Python.", + ); + let program: string; + let head: string; + try { + program = await realpath(command); + head = await readHead(program); + } catch (cause: unknown) { + throw unavailable( + operation, + `REA_AGENT_ENV_COMMAND ${command} cannot be read.`, + cause, + ); + } + const named = /^#!(\/\S+)[ \t]*\r?\n/u.exec(head)?.[1]; + for (const candidate of [ + ...(named !== undefined && basename(named).startsWith("python") + ? [named] + : []), + join(dirname(program), "python3"), + ]) + if (await executable(candidate)) return candidate; + throw unavailable( + operation, + `REA_AGENT_ENV_COMMAND ${command} names no Python: its first line names no python interpreter and no python3 sits beside it.`, + ); +}; + +/** What a Docker container name may hold, so a sandbox id outside it names no container. */ +const CONTAINER_NAME = /^[A-Za-z0-9_.-]+$/u; + +/** + * The `docker ps` filters that find one sandbox's containers: agentenv's + * local provider runs a computer as `agent-` without a label, and + * labels only the containers and relays of its VM and closed-network modes. + */ +const sandboxFilters = (sandbox: string): readonly string[] => [ + ...(CONTAINER_NAME.test(sandbox) + ? [`name=^agent-${sandbox.replaceAll(".", "\\.")}$`] + : []), + `label=agentenv.sandbox=${sandbox}`, +]; + +/** Remove the containers of one sandbox REA's helper named, by their exact name and label; undefined once removed. */ +const removeSandbox = async ( + environment: Readonly, + sandbox: string, +): Promise => { + const filters = sandboxFilters(sandbox); + const manual = `Run docker rm -f ${filters.map((filter) => `$(docker ps -aq --filter '${filter}')`).join(" ")}.`; + const docker = environment.REA_DOCKER_COMMAND; + if (docker === undefined || !isAbsolute(docker)) + return `agentenv's teardown did not confirm sandbox ${sandbox} is gone, and REA_DOCKER_COMMAND is not an absolute path to remove its containers. ${manual}`; + const options = { env: environment, timeout: TEARDOWN_MS }; + try { + const containers = new Set(); + for (const filter of filters) { + const listed = await execFileAsync( + docker, + ["ps", "-aq", "--filter", filter], + options, + ); + for (const id of listed.stdout.split(/\s+/u).filter(Boolean)) + containers.add(id); + } + if (containers.size > 0) + await execFileAsync(docker, ["rm", "-f", ...containers], options); + return undefined; + } catch (cause: unknown) { + return `Removing sandbox ${sandbox}'s containers failed: ${cause instanceof Error ? cause.message : String(cause)}. ${manual}`; + } +}; + +const parseHandshake = (line: string): Handshake => { + try { + const parsed = handshakeSchema.safeParse(JSON.parse(line)); + if (parsed.success) return parsed.data; + } catch { + // Reported below as a line that is not a handshake. + } + return { error: "the helper wrote something other than its handshake" }; +}; + +/** + * REA's helper process holding one computer up: an owned, supervised child + * run with the agentenv runner's Python. It reads the first JSON line and + * keeps draining stdout, so a handshake arriving after a cancelled wait still + * names the sandbox teardown must reach. + */ +export class ComputerHelper { + readonly #supervisor: ProviderProcessSupervisor; + readonly #handshake: Promise; + #sandboxId: string | undefined; + /** The helper said no computer came up, and exits on its own after its teardown. */ + #failed = false; + #token: string | null = null; + #stopped: + | Promise<{ + readonly exit: number | null | undefined; + readonly stopped?: string; + }> + | undefined; + + private constructor( + private readonly spawned: SpawnedOwnedProviderProcess, + private readonly environment: Readonly, + ) { + this.#supervisor = new ProviderProcessSupervisor( + { + ...spawned, + ownsProcessLifetime: true, + cleanup: + spawned.cleanup ?? + (() => cleanupOwnedProcessGroup(spawned.ownership)), + }, + { + captureStdout: false, + // A failed build's error comes last, after its package installs. + maxDiagnosticBytes: { stdout: 0, stderr: STDERR_BYTES }, + diagnosticRetention: "tail", + }, + ); + // A helper that already exited must not crash REA when its stdin closes. + spawned.process.stdin?.on("error", () => undefined); + this.#handshake = new Promise((resolve) => { + const ended = () => resolve(undefined); + void this.#drain(resolve).then(ended, ended); + }); + } + + /** Spawn the helper for a bundle folder; it deploys the bundle's smoke task. */ + static async spawn(options: { + readonly bundle: string; + readonly environment: Readonly; + readonly signal: AbortSignal; + readonly operation: string; + }): Promise { + const { environment, signal } = options; + const interpreter = await agentEnvInterpreter( + environment, + options.operation, + ); + const spawned = await spawnOwnedProviderProcess({ + command: interpreter, + arguments: [HELPER, options.bundle, "smoke"], + runId: randomUUID(), + expectedCommand: null, + hostEnvironment: environment, + stdin: "pipe", + signal, + }); + return new ComputerHelper(spawned, environment); + } + + async #drain(resolve: (handshake: Handshake) => void): Promise { + const stdout = this.spawned.process.stdout; + if (stdout === null) return; + let first: Handshake | undefined; + for await (const line of readBoundedControlLines(stdout, HANDSHAKE_BYTES)) { + if (first !== undefined) continue; + first = line.ok + ? parseHandshake(line.value) + : { error: "the helper's handshake line is too long" }; + if ("sandbox_id" in first) { + this.#sandboxId = first.sandbox_id; + this.#token = first.control_token; + } else this.#failed = true; + resolve(first); + } + } + + /** The handshake, or undefined when the helper ended without one; cancellation rejects with its reason. */ + handshake(signal: AbortSignal): Promise { + signal.throwIfAborted(); + const aborted = new Promise((_, reject) => { + signal.addEventListener("abort", () => reject(signal.reason), { + once: true, + }); + }); + return Promise.race([this.#handshake, aborted]); + } + + /** The helper's stderr tail with the control token removed, and how it exited. */ + output(): AnalysisCapturedOutput { + const snapshot = this.#supervisor.snapshot(); + const token = this.#token; + return { + stdout: "", + stderr: + token === null + ? snapshot.stderr.text + : snapshot.stderr.text.replaceAll(token, "[control token]"), + truncated: + snapshot.diagnosticTruncated === true || + snapshot.stderr.observedBytes > snapshot.stderr.bytes, + exit_code: snapshot.exitCode ?? null, + signal: snapshot.signal ?? null, + }; + } + + /** + * Close stdin once the helper holds a computer; let one that reported its + * failure finish its own teardown; else SIGTERM. SIGTERM after a grace, + * then clear its process group. + */ + async #stop() { + const child = this.spawned.process; + const holding = this.#sandboxId !== undefined; + const waiting = holding || this.#failed; + if (holding) child.stdin?.end(); + else if (!this.#failed) child.kill("SIGTERM"); + if (!(await this.#supervisor.waitForExit(TEARDOWN_MS)) && waiting) { + child.kill("SIGTERM"); + await this.#supervisor.waitForExit(TEARDOWN_MS); + } + const exit = this.#supervisor.observedExit()?.code; + const result = await this.#supervisor.stop(); + return result.status === "incomplete" + ? { exit, stopped: result.reason } + : { exit }; + } + + /** + * Bring the computer down. The helper's exit 0 or 2 means agentenv tore + * down everything it kept; otherwise the containers of the sandbox it + * named are removed by their `agentenv.sandbox` label. A failed release + * can be retried. + */ + async release(): Promise { + const { exit, stopped } = await (this.#stopped ??= this.#stop()); + if (stopped !== undefined) { + this.#stopped = undefined; + return { + verified: false, + reason: `REA's agentenv helper did not stop: ${stopped}`, + resources: [ + `process group ${String(this.spawned.ownership.processGroupId)}`, + ], + }; + } + if (exit === 0 || exit === 2) + return { verified: true, teardown: "agentenv" }; + const sandbox = this.#sandboxId; + if (sandbox === undefined) + return { + verified: false, + reason: `REA's agentenv helper ended (exit ${String(exit)}) before naming its sandbox; look for its container with docker ps -a --filter name=agent- and docker ps -a --filter label=agentenv.sandbox.`, + resources: ["an unnamed agentenv sandbox"], + }; + const failure = await removeSandbox(this.environment, sandbox); + return failure === undefined + ? { verified: true, teardown: "container_label" } + : { + verified: false, + reason: failure, + resources: [`containers labelled agentenv.sandbox=${sandbox}`], + }; + } +} diff --git a/src/domain/desktop/appBuild.test.ts b/src/domain/desktop/appBuild.test.ts new file mode 100644 index 0000000000..2db90e7cd1 --- /dev/null +++ b/src/domain/desktop/appBuild.test.ts @@ -0,0 +1,189 @@ +import { describe, expect, it } from "vitest"; + +import { PACKAGE_METADATA } from "../../generatedPackageMetadata.js"; +import { + BUILD_NODE_VERSION, + buildSourceSchema, + renderBuildBundle, + type BuildBundleFile, + type BuildRequest, +} from "./appBuild.js"; + +const request = (install: BuildRequest["install"]): BuildRequest => ({ + kind: "request", + desktop_entry: "org.xfce.mousepad.desktop", + install, +}); +const localPayload = { kind: "file", path: "/builds/rea-prod.tar.xz" } as const; +const text = (files: readonly BuildBundleFile[], path: string): string => { + const content = files.find((file) => file.path === path)?.content; + if (typeof content !== "string") throw new Error(`no text file ${path}`); + return content; +}; + +describe("build bundle renderer", () => { + it("installs an archive package, opens it on demand and adds REA in a build-only layer", () => { + const bundle = renderBuildBundle( + request({ kind: "archive", package: "mousepad" }), + localPayload, + ); + expect(bundle.name).toBe("mousepad"); + expect(bundle.files.map((file) => file.path).sort()).toEqual([ + "envs/mousepad/env.toml", + "envs/mousepad/install-rea.sh", + "envs/mousepad/mousepad_app.py", + "envs/mousepad/rea-payload.tar", + "tasks/smoke.json", + ]); + expect( + bundle.files.find((file) => file.path.endsWith("rea-payload.tar")) + ?.content, + ).toEqual({ copyFrom: "/builds/rea-prod.tar.xz" }); + const module = text(bundle.files, "envs/mousepad/mousepad_app.py"); + expect(module).toContain( + 'packages=Packages(apt=["mousepad", "libgtk-3-bin"])', + ); + expect(module).toContain( + 'launch=Launch(at="demand", command=["gtk-launch", "org.xfce.mousepad"]),', + ); + expect(module).toContain('"arch": ["amd64", "arm64"]'); + expect(module).toContain( + `Download(url="https://nodejs.org/dist/v${BUILD_NODE_VERSION}/node-v${BUILD_NODE_VERSION}-linux-x64.tar.xz", sha256="6c172a9ab6718f7b801fb5a7bd486af2ee5e32758086a04b84fcce2aa77095e6", to="node-amd64.tar.xz", arch="amd64")`, + ); + expect(module).toContain( + 'sha256="dbabf7a1cbf947e6059ffad2c27b80041db97dea1e553b62afdee7f13fca336e", to="node-arm64.tar.xz", arch="arm64"', + ); + expect(module).toContain('run=[RunScript(script="install-rea.sh")]'); + // Building needs no seed and no data hooks. + expect(module).not.toMatch(/paths=|@add_data|@get_data|@reset_data/u); + expect(text(bundle.files, "envs/mousepad/env.toml")).toContain( + 'module = "mousepad_app.py:Mousepad"', + ); + expect(JSON.parse(text(bundle.files, "tasks/smoke.json"))).toEqual([ + { id: "env", type: "deploy_env", env_id: "mousepad" }, + ]); + const script = text(bundle.files, "envs/mousepad/install-rea.sh"); + expect(script).toContain("/opt/rea/payload/*/scripts/rea.mjs"); + expect(script).toContain( + `/opt/rea/node-v${BUILD_NODE_VERSION}-linux-*/bin/node`, + ); + }); + + it("pins the Node.js release REA itself pins", () => { + expect(BUILD_NODE_VERSION).toBe(PACKAGE_METADATA.pinnedNodeVersion); + }); +}); + +describe("build bundle install sources", () => { + it("declares a vendor apt repository with its key file in the environment folder", () => { + const key = Buffer.from( + "-----BEGIN PGP PUBLIC KEY BLOCK-----\n\nAAAA\n-----END PGP PUBLIC KEY BLOCK-----\n", + ); + const bundle = renderBuildBundle( + { + ...request({ + kind: "apt_repository", + package: "geany", + uri: "https://ppa.launchpadcontent.net/geany-dev/ppa/ubuntu", + suite: "noble", + components: ["main"], + key_file: "/keys/geany.asc", + }), + desktop_entry: "geany", + window_class: "Geany", + }, + localPayload, + key, + ); + const module = text(bundle.files, "envs/geany/geany_app.py"); + expect(module).toContain( + 'sources=[AptSource(apt="https://ppa.launchpadcontent.net/geany-dev/ppa/ubuntu", suite="noble", components=["main"], key="apt-keys/geany.asc")],', + ); + expect(module).toContain( + 'window={"class": "Geany"}, ready=Ready(window=True)', + ); + const copied = bundle.files.find( + (file) => file.path === "envs/geany/apt-keys/geany.asc", + )?.content; + expect(copied instanceof Uint8Array && key.equals(copied)).toBe(true); + }); + + it("downloads a .deb for its architecture and installs it after its dependencies", () => { + const sha256 = "a".repeat(64); + const bundle = renderBuildBundle( + request({ + kind: "package_file", + url: "https://downloads.example.com/varicad.deb", + sha256, + architecture: "amd64", + dependencies: ["libgl1"], + }), + { + kind: "download", + url: "https://releases.example.com/rea.tar.xz", + sha256: "b".repeat(64), + }, + ); + const module = text(bundle.files, "envs/mousepad/mousepad_app.py"); + expect(module).toContain( + 'packages=Packages(apt=["libgl1", "libgtk-3-bin"])', + ); + expect(module).toContain( + `Download(url="https://downloads.example.com/varicad.deb", sha256="${sha256}", to="vendor/package.deb", arch="amd64")`, + ); + expect(module).toContain( + `Download(url="https://releases.example.com/rea.tar.xz", sha256="${"b".repeat(64)}", to="rea-payload.tar")`, + ); + expect(module).toContain('"arch": ["amd64"]'); + expect(module).not.toContain('to="node-arm64.tar.xz"'); + expect(module).toContain( + 'run=[RunScript(script="install-package.sh"), RunScript(script="install-rea.sh")]', + ); + expect(text(bundle.files, "envs/mousepad/install-package.sh")).toContain( + "dpkg -i 'vendor/package.deb'", + ); + expect( + bundle.files.some((file) => file.path.endsWith("rea-payload.tar")), + ).toBe(false); + }); + + it("admits only what agentenv can install from", () => { + const parse = (value: unknown) => buildSourceSchema.safeParse(value); + expect( + parse({ + kind: "request", + desktop_entry: "../evil", + install: { kind: "archive", package: "mousepad" }, + }).success, + ).toBe(false); + expect( + parse({ + kind: "request", + desktop_entry: "mousepad", + install: { + kind: "package_file", + url: "https://user@example.com/a.deb", + sha256: "a".repeat(64), + architecture: "amd64", + }, + }).success, + ).toBe(false); + expect( + parse({ + kind: "request", + desktop_entry: "mousepad", + install: { + kind: "apt_repository", + package: "geany", + uri: "ftp://example.com/ubuntu", + suite: "noble", + key_file: "/keys/geany.gpg", + }, + }).success, + ).toBe(false); + expect(parse({ kind: "bundle", path: "relative/bundle" }).success).toBe( + false, + ); + expect(parse({ kind: "bundle", path: "/tmp/bundle" }).success).toBe(true); + }); +}); diff --git a/src/domain/desktop/appBuild.ts b/src/domain/desktop/appBuild.ts new file mode 100644 index 0000000000..b2b94f28a2 --- /dev/null +++ b/src/domain/desktop/appBuild.ts @@ -0,0 +1,321 @@ +import { createHash } from "node:crypto"; + +import { z } from "zod"; + +import { absoluteLocalPathStringSchema } from "../localPath.js"; +import { + appModuleFile, + aptSources, + className, + environmentName, + py, + pyList, + renderEnvToml, + renderInstallScript, + renderSmokeTask, +} from "./appExport.js"; +import { INSTALL_SCRIPT, type ExportPackageFile } from "./appExportInstall.js"; +import { isArmoredKey } from "./aptRepositories.js"; + +/** The Node.js release the build layer runs REA with: the one REA pins. */ +export const BUILD_NODE_VERSION = "24.22.1"; +/** SHA-256 of each architecture's Node.js tarball, from the release's SHASUMS256.txt. */ +const NODE_TARBALLS = [ + { + arch: "amd64", + platform: "linux-x64", + sha256: "6c172a9ab6718f7b801fb5a7bd486af2ee5e32758086a04b84fcce2aa77095e6", + }, + { + arch: "arm64", + platform: "linux-arm64", + sha256: "dbabf7a1cbf947e6059ffad2c27b80041db97dea1e553b62afdee7f13fca336e", + }, +] as const; +const COMPUTER_ARCHITECTURES = ["amd64", "arm64"] as const; +/** The package of `gtk-launch`, which opens the app from its desktop entry. */ +const LAUNCHER_PACKAGE = "libgtk-3-bin"; +const REA_INSTALL_SCRIPT = "install-rea.sh"; +const REA_PAYLOAD = "rea-payload.tar"; + +/** A URL agentenv downloads from: http or https in lowercase, without a user name or password. */ +const downloadUrlSchema = z + .url({ protocol: /^https?$/u }) + .regex(/^https?:\/\//u) + .refine( + (url) => { + const parsed = new URL(url); + return parsed.username === "" && parsed.password === ""; + }, + { message: "The URL must not hold a user name or password." }, + ); +const aptPackageSchema = z + .string() + .regex(/^[a-z0-9][a-z0-9+.-]+$/u) + .describe("A Debian package name."); +const sha256Schema = z.string().regex(/^[0-9a-f]{64}$/u); + +/** Where the computer installs the app from, chosen before anything is captured. */ +export const buildInstallSchema = z.discriminatedUnion("kind", [ + z.strictObject({ + kind: z.literal("archive"), + package: aptPackageSchema, + }), + z.strictObject({ + kind: z.literal("apt_repository"), + package: aptPackageSchema, + uri: downloadUrlSchema, + suite: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9._/-]*$/u), + components: z.array(z.string().regex(/^[A-Za-z0-9._-]+$/u)).default([]), + key_file: absoluteLocalPathStringSchema.describe( + "The repository's signing key on this host, armored or binary.", + ), + }), + z.strictObject({ + kind: z.literal("package_file"), + url: downloadUrlSchema, + sha256: sha256Schema, + architecture: z.enum(COMPUTER_ARCHITECTURES), + dependencies: z + .array(aptPackageSchema) + .default([]) + .describe( + "Packages apt installs first; dpkg installs the file offline, so its dependencies must be listed.", + ), + }), +]); + +/** A request for a build computer, or an existing bundle folder to keep up as it is. */ +export const buildSourceSchema = z.discriminatedUnion("kind", [ + z.strictObject({ + kind: z.literal("request"), + desktop_entry: z + .string() + .regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/u) + .max(255) + .describe( + "The desktop entry id the package installs, such as org.xfce.mousepad; the menu opens it with gtk-launch.", + ), + window_class: z + .string() + .min(1) + .max(255) + .optional() + .describe("The app window's WM_CLASS class, when known."), + install: buildInstallSchema, + }), + z.strictObject({ + kind: z.literal("bundle"), + path: absoluteLocalPathStringSchema.describe( + "An agentenv bundle folder whose tasks/smoke.json deploys one computer, such as an exported one.", + ), + }), +]); +export type BuildSource = z.output; +export type BuildRequest = Extract; + +/** Where the build layer gets REA from: a tar archive with one top folder holding `scripts/rea.mjs`. */ +export type ReaPayload = + | { readonly kind: "file"; readonly path: string } + | { + readonly kind: "download"; + readonly url: string; + readonly sha256: string; + }; + +/** One file of the bundle: its text or bytes, or a host file copied as it is. */ +export interface BuildBundleFile { + readonly path: string; + readonly content: string | Uint8Array | { readonly copyFrom: string }; +} + +const download = ( + url: string, + sha256: string, + to: string, + arch?: string, +): string => + `Download(url=${py(url)}, sha256=${py(sha256)}, to=${py(to)}${arch === undefined ? "" : `, arch=${py(arch)}`})`; + +/** The build-only step: unpack Node.js and REA into /opt/rea, as root and offline, and put both on PATH. */ +const renderReaInstallScript = (): string => `#!/bin/sh +# Build-only layer, run as root with the network off: unpack Node.js and REA into /opt/rea and put both on PATH. +set -eu +mkdir -p /opt/rea/payload +python3 - <<'PY' +import glob +import tarfile + +for name, into in [(path, "/opt/rea") for path in glob.glob("node-*.tar.xz")] + [("${REA_PAYLOAD}", "/opt/rea/payload")]: + with tarfile.open(name) as archive: + archive.extractall(into, filter="tar") +PY +set -- /opt/rea/payload/*/scripts/rea.mjs +if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then + echo "The REA archive must hold one top folder with scripts/rea.mjs." >&2 + exit 1 +fi +ln -sf /opt/rea/node-v${BUILD_NODE_VERSION}-linux-*/bin/node /usr/local/bin/node +printf '#!/bin/sh\\nexec /usr/local/bin/node %s "$@"\\n' "$1" > /usr/local/bin/rea +chmod 755 /usr/local/bin/rea +rm -f node-*.tar.xz ${REA_PAYLOAD} +`; + +/** What the computer installs for a request: packages, sources, downloads and run scripts. */ +const buildInstall = ( + request: BuildRequest, + name: string, + payload: ReaPayload, + aptKey: Uint8Array | undefined, +) => { + const { install } = request; + const architectures = + install.kind === "package_file" + ? [install.architecture] + : [...COMPUTER_ARCHITECTURES]; + if (install.kind === "apt_repository" && aptKey === undefined) + throw new TypeError("An apt repository needs its key file's bytes."); + const key = + install.kind === "apt_repository" && aptKey !== undefined + ? { + bytes: aptKey, + source: { + uri: install.uri, + suite: install.suite, + components: install.components, + keyFile: `apt-keys/${name}.${isArmoredKey(aptKey) ? "asc" : "gpg"}`, + keySha256: createHash("sha256").update(aptKey).digest("hex"), + }, + } + : undefined; + const packageFile: ExportPackageFile | null = + install.kind === "package_file" + ? { + url: install.url, + sha256: install.sha256, + path: "vendor/package.deb", + architecture: install.architecture, + dependencies: install.dependencies, + unreadDependencies: [], + } + : null; + const downloads = [ + ...NODE_TARBALLS.filter(({ arch }) => architectures.includes(arch)).map( + ({ arch, platform, sha256 }) => + download( + `https://nodejs.org/dist/v${BUILD_NODE_VERSION}/node-v${BUILD_NODE_VERSION}-${platform}.tar.xz`, + sha256, + `node-${arch}.tar.xz`, + arch, + ), + ), + ...(packageFile === null + ? [] + : [ + download( + packageFile.url, + packageFile.sha256, + packageFile.path, + packageFile.architecture, + ), + ]), + ...(payload.kind === "download" + ? [download(payload.url, payload.sha256, REA_PAYLOAD)] + : []), + ]; + return { + architectures, + packages: [ + ...(install.kind === "package_file" + ? install.dependencies + : [install.package]), + LAUNCHER_PACKAGE, + ], + key, + packageFile, + downloads, + scripts: [ + ...(packageFile === null ? [] : [INSTALL_SCRIPT]), + REA_INSTALL_SCRIPT, + ], + }; +}; + +/** The class module: the decorators that install and launch the app, and a class with no hooks. */ +const renderBuildModule = ( + request: BuildRequest, + entry: string, + plan: { readonly className: string }, + install: ReturnType, +): string => { + const window = + request.window_class === undefined + ? "" + : `, window={"class": ${py(request.window_class)}}, ready=Ready(window=True)`; + const sources = aptSources({ + aptSources: install.key === undefined ? [] : [install.key.source], + }); + return `"""${entry} as an agentenv computer app for building an environment with REA. + +Rendered by REA's start_build_computer: the app installed from the request, opened on demand, and REA with Node.js +${BUILD_NODE_VERSION} in /opt/rea. It is for building only; keep it local and never publish it. +""" + +from agentenv_protocol.computer import AptSource, ComputerAppEnvironment, Download, Launch, Packages, Ready, RunScript, computer_app, linux + + +@computer_app(supports=[{"os": {"type": "linux", "display": ["x11"]}, "arch": ${pyList(install.architectures)}}]) +@linux( + packages=Packages(apt=${pyList(install.packages)}),${sources} + downloads=[${install.downloads.join(", ")}], + run=[${install.scripts.map((script) => `RunScript(script=${py(script)})`).join(", ")}], + launch=Launch(at="demand", command=${pyList(["gtk-launch", entry])}${window}), +) +class ${plan.className}(ComputerAppEnvironment): + """${entry}, opened on demand for building: it keeps no seed and serves no data operations.""" +`; +}; + +/** + * The bundle a build computer starts from: the requested app installed and + * opened on demand, without a seed or data hooks, plus REA and Node.js in a + * build-only layer. Node.js is downloaded per architecture at put; REA is + * downloaded too, or shipped in the environment folder from a local archive. + * An apt repository's key file is copied into the environment folder. + */ +export const renderBuildBundle = ( + request: BuildRequest, + payload: ReaPayload, + aptKey?: Uint8Array, +): { readonly name: string; readonly files: readonly BuildBundleFile[] } => { + const entry = request.desktop_entry.replace(/\.desktop$/u, ""); + const name = environmentName(entry, (candidate) => candidate !== "app"); + const plan = { name, className: className(name) }; + const install = buildInstall(request, name, payload, aptKey); + const env = `envs/${name}`; + const file = (path: string, content: BuildBundleFile["content"]) => ({ + path: `${env}/${path}`, + content, + }); + return { + name, + files: [ + file("env.toml", renderEnvToml(plan)), + file( + appModuleFile(name), + renderBuildModule(request, entry, plan, install), + ), + file(REA_INSTALL_SCRIPT, renderReaInstallScript()), + ...(install.packageFile === null + ? [] + : [file(INSTALL_SCRIPT, renderInstallScript(install.packageFile))]), + ...(install.key === undefined + ? [] + : [file(install.key.source.keyFile, install.key.bytes)]), + ...(payload.kind === "file" + ? [file(REA_PAYLOAD, { copyFrom: payload.path })] + : []), + { path: "tasks/smoke.json", content: renderSmokeTask(plan) }, + ], + }; +}; diff --git a/src/domain/desktop/appExport.ts b/src/domain/desktop/appExport.ts index 22d9b66bfb..a58f5f8b62 100644 --- a/src/domain/desktop/appExport.ts +++ b/src/domain/desktop/appExport.ts @@ -276,12 +276,14 @@ export const launchCommand = ( }; /** A Python string literal; JSON escapes are valid Python escapes. */ -const py = (value: string): string => JSON.stringify(value); -const pyList = (values: readonly string[]): string => +export const py = (value: string): string => JSON.stringify(value); +export const pyList = (values: readonly string[]): string => `[${values.map(py).join(", ")}]`; /** The bundle's env.toml: a computer app deployed alone on the X11 base. */ -export const renderEnvToml = (plan: DesktopAppExportPlan): string => +export const renderEnvToml = ( + plan: Pick, +): string => [ `type = "computer_app"`, `module = ${py(`${appModuleFile(plan.name)}:${plan.className}`)}`, @@ -334,7 +336,10 @@ const aptPackages = (plan: DesktopAppExportPlan): readonly string[] => [ ]), ]; -const aptSources = (plan: DesktopAppExportPlan): string => +/** The decorator's `sources=` line for vendor repositories, or nothing for the archive. */ +export const aptSources = ( + plan: Pick, +): string => plan.aptSources.length === 0 ? "" : `\n sources=[${plan.aptSources @@ -515,5 +520,7 @@ The seed is the user's real data. Keep this bundle local; do not publish it. ${plan.dconf.length === 0 ? "" : "dconf keys are compiled into the user's dconf database, with `dconf-cli`, once when the app first starts; `data/reset` does not restore them.\n"}${taskSection}`; /** A smoke task that deploys the app. */ -export const renderSmokeTask = (plan: DesktopAppExportPlan): string => +export const renderSmokeTask = ( + plan: Pick, +): string => `${JSON.stringify([{ id: "env", type: "deploy_env", env_id: plan.name }], null, 2)}\n`; diff --git a/src/domain/desktop/computerActions.test.ts b/src/domain/desktop/computerActions.test.ts new file mode 100644 index 0000000000..3d298ee812 --- /dev/null +++ b/src/domain/desktop/computerActions.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest"; + +import { driverCall, w3cKeys } from "./computerActions.js"; + +describe("computer actions", () => { + it("translates model key spellings to W3C names and keeps W3C names and characters", () => { + expect(w3cKeys(["ctrl+s"])).toEqual(["Control", "s"]); + expect(w3cKeys(["cmd+Shift+z"])).toEqual(["Meta", "Shift", "z"]); + expect(w3cKeys(["Return", "esc", "pgdn", "f5", "space"])).toEqual([ + "Enter", + "Escape", + "PageDown", + "F5", + " ", + ]); + expect(w3cKeys(["Control", "ArrowUp", "+", "ctrl++"])).toEqual([ + "Control", + "ArrowUp", + "+", + "Control", + "+", + ]); + // An unknown name passes through for the driver to refuse with its hint. + expect(w3cKeys(["Hyper"])).toEqual(["Hyper"]); + }); + + it("passes the driver's arguments through and translates key fields", () => { + expect( + driverCall("click", { x: 500, y: 250, count: 2, space: "norm" }), + ).toEqual({ + ok: true, + value: { + tool: "computer_click", + arguments: { x: 500, y: 250, count: 2, space: "norm" }, + }, + }); + expect(driverCall("key", { keys: ["ctrl+s"] })).toEqual({ + ok: true, + value: { tool: "computer_key", arguments: { keys: ["Control", "s"] } }, + }); + expect( + driverCall("drag", { + path: [ + [1, 2], + [3, 4], + ], + modifiers: ["shift"], + }), + ).toEqual({ + ok: true, + value: { + tool: "computer_drag", + arguments: { + path: [ + [1, 2], + [3, 4], + ], + modifiers: ["Shift"], + }, + }, + }); + }); + + it("refuses a field the action does not take", () => { + expect(driverCall("type", { text: "hi", x: 3 })).toEqual({ + ok: false, + error: { field: "x", accepted: ["text", "screenshot"] }, + }); + }); +}); diff --git a/src/domain/desktop/computerActions.ts b/src/domain/desktop/computerActions.ts new file mode 100644 index 0000000000..04a7a80e16 --- /dev/null +++ b/src/domain/desktop/computerActions.ts @@ -0,0 +1,197 @@ +import { z } from "zod"; + +import { fixedTupleSchema } from "../fixedTupleSchema.js"; +import type { JsonValue } from "../jsonValue.js"; +import { err, ok, type Result } from "../result.js"; + +/** Actions REA forwards to an agentenv computer's driver, each as its `computer_` tool. */ +export const COMPUTER_ACTIONS = [ + "screenshot", + "click", + "drag", + "scroll", + "type", + "key", + "wait", +] as const; +export type ComputerAction = (typeof COMPUTER_ACTIONS)[number]; + +const POINTER = ["frame_id", "space", "norm_scale"] as const; + +/** The driver's argument names each action takes, from its `computer_*` tool schemas. */ +const ACTION_FIELDS: Readonly> = { + screenshot: ["region", "format", "quality", ...POINTER], + click: ["x", "y", "button", "count", "modifiers", "screenshot", ...POINTER], + drag: [ + "path", + "button", + "duration_ms", + "modifiers", + "screenshot", + ...POINTER, + ], + scroll: [ + "direction", + "x", + "y", + "amount", + "modifiers", + "screenshot", + ...POINTER, + ], + type: ["text", "screenshot"], + key: ["keys", "repeat", "screenshot"], + wait: ["ms", "screenshot"], +}; + +const coordinate = z + .number() + .describe("Frame pixels, or 0..norm_scale with space norm."); + +/** The driver's screenshot arguments that size its image. */ +const SCREENSHOT_FIELDS = { + region: fixedTupleSchema([z.number(), z.number(), z.number(), z.number()]) + .optional() + .describe("screenshot: [x0, y0, x1, y1] to crop to."), + format: z.enum(["png", "jpeg"]).optional(), + quality: z.int().min(1).max(100).optional().describe("jpeg quality."), +}; + +/** How a build computer's first screenshot is taken; smaller images cost fewer tokens. */ +export const screenshotOptionsSchema = z.strictObject(SCREENSHOT_FIELDS); +export type ScreenshotOptions = z.output; + +/** One action on a build computer: the driver's own argument names, passed through as given. */ +export const computerActionInputSchema = z.strictObject({ + computer_id: z.uuid().describe("The id start_build_computer returned."), + action: z + .enum(COMPUTER_ACTIONS) + .describe("The driver's computer_ tool to call."), + x: coordinate.optional(), + y: coordinate.optional(), + frame_id: z + .string() + .min(1) + .optional() + .describe("The frame the point is on; the latest by default."), + space: z + .enum(["frame", "norm"]) + .optional() + .describe( + "frame: the frame's pixels, the default; norm: a 0..norm_scale grid over it.", + ), + norm_scale: z.number().positive().optional(), + button: z + .string() + .min(1) + .optional() + .describe("left, the default, right or middle."), + count: z.int().positive().optional().describe("click: 2 double-clicks."), + modifiers: z + .array(z.string().min(1)) + .optional() + .describe("Keys held during a click, drag or scroll."), + path: z + .array(fixedTupleSchema([z.number(), z.number()])) + .min(2) + .optional() + .describe("drag: [x, y] points from press to release."), + duration_ms: z.int().nonnegative().optional(), + direction: z.enum(["up", "down", "left", "right"]).optional(), + amount: z.int().positive().optional().describe("scroll: notches."), + text: z.string().optional().describe("type: \\n presses Enter, \\t Tab."), + keys: z + .array(z.string().min(1)) + .min(1) + .optional() + .describe( + 'key: one chord pressed in order, as ["Control", "s"] or ["ctrl+s"].', + ), + repeat: z.int().positive().optional(), + ms: z.int().nonnegative().optional().describe("wait: milliseconds."), + ...SCREENSHOT_FIELDS, + screenshot: z + .boolean() + .optional() + .describe("Return a screenshot taken after the action."), +}); +export type ComputerActionInput = z.output; + +/** Lowercase model and X11 spellings of keys the driver names in W3C terms. */ +const KEY_NAMES: ReadonlyMap = new Map([ + ["ctrl", "Control"], + ["control", "Control"], + ["alt", "Alt"], + ["option", "Alt"], + ["shift", "Shift"], + ["cmd", "Meta"], + ["command", "Meta"], + ["meta", "Meta"], + ["super", "Meta"], + ["win", "Meta"], + ["enter", "Enter"], + ["return", "Enter"], + ["esc", "Escape"], + ["escape", "Escape"], + ["tab", "Tab"], + ["space", " "], + ["backspace", "Backspace"], + ["delete", "Delete"], + ["del", "Delete"], + ["insert", "Insert"], + ["home", "Home"], + ["end", "End"], + ["pageup", "PageUp"], + ["page_up", "PageUp"], + ["pgup", "PageUp"], + ["pagedown", "PageDown"], + ["page_down", "PageDown"], + ["pgdn", "PageDown"], + ["up", "ArrowUp"], + ["down", "ArrowDown"], + ["left", "ArrowLeft"], + ["right", "ArrowRight"], +]); + +const ONE_CHARACTER = /^.$/su; + +/** One key's W3C name: a character as given, a known spelling translated, any other name left for the driver to judge. */ +const w3cKey = (spelling: string): string => { + if (ONE_CHARACTER.test(spelling)) return spelling; + const lower = spelling.toLowerCase(); + return ( + KEY_NAMES.get(lower) ?? + (/^f(?:[1-9]|1\d|2[0-4])$/u.test(lower) ? lower.toUpperCase() : spelling) + ); +}; + +/** W3C key names from W3C names or model spellings such as `ctrl+s`, which name a chord in order. */ +export const w3cKeys = (keys: readonly string[]): string[] => + keys.flatMap((key) => + ONE_CHARACTER.test(key) ? [key] : key.split(/\+(?!$)/u).map(w3cKey), + ); + +/** The driver tool and arguments for an action, or the first field the action does not take. */ +export const driverCall = ( + action: ComputerAction, + fields: Readonly>, +): Result< + { readonly tool: string; readonly arguments: Record }, + { readonly field: string; readonly accepted: readonly string[] } +> => { + const accepted = ACTION_FIELDS[action]; + const given = Object.entries(fields).flatMap(([name, value]) => + value === undefined ? [] : [[name, value] as const], + ); + const refused = given.find(([name]) => !accepted.includes(name))?.[0]; + if (refused !== undefined) return err({ field: refused, accepted }); + const translated = Object.fromEntries( + given.map(([name, value]) => [ + name, + (name === "keys" || name === "modifiers") && Array.isArray(value) + ? w3cKeys(value.filter((key) => typeof key === "string")) + : value, + ]), + ); + return ok({ tool: `computer_${action}`, arguments: translated }); +}; diff --git a/src/process/ProviderProcess.ts b/src/process/ProviderProcess.ts index 83946af8f2..10ce701278 100644 --- a/src/process/ProviderProcess.ts +++ b/src/process/ProviderProcess.ts @@ -206,6 +206,11 @@ export interface ProviderProcessSupervisorOptions { readonly captureStdout?: boolean; /** Retained output budget; overflow is drained and reported, not retained. */ readonly maxDiagnosticBytes?: ProcessDiagnosticBudget; + /** + * Which bytes the budget keeps: the first ones (the default), or each + * stream's latest, where a failing command's last words are. + */ + readonly diagnosticRetention?: "head" | "tail"; /** * Single-line token ending at a non-ASCII-word byte or EOF, located before retention * applies, so its latest occurrence survives an exhausted budget. @@ -833,13 +838,24 @@ export class ProviderProcessSupervisor { streamLimit === undefined ? bytes.byteLength : Math.max(0, streamLimit - capture.bytes); - const retained = Math.min( - bytes.byteLength, - aggregateRemaining, - streamRemaining, - ); - if (retained < bytes.byteLength) this.#diagnosticTruncated = true; - if (retained > 0) capture.append(bytes.subarray(0, retained)); + let truncated: boolean; + if (this.#options.diagnosticRetention === "tail") { + // The stream may give up its own earlier bytes to keep these. + const limit = Math.min( + capture.bytes + aggregateRemaining, + streamLimit ?? Number.MAX_SAFE_INTEGER, + ); + truncated = capture.appendTail(bytes, limit) > 0; + } else { + const retained = Math.min( + bytes.byteLength, + aggregateRemaining, + streamRemaining, + ); + if (retained > 0) capture.append(bytes.subarray(0, retained)); + truncated = retained < bytes.byteLength; + } + if (truncated) this.#diagnosticTruncated = true; this.#options.onDiagnostic?.({ type: "output", stream, @@ -848,7 +864,7 @@ export class ProviderProcessSupervisor { stream === "stdout" ? this.#stdoutObservedBytes : this.#stderrObservedBytes, - ...(retained < bytes.byteLength ? { truncated: true } : {}), + ...(truncated ? { truncated: true } : {}), }); } @@ -868,6 +884,8 @@ export class ProviderProcessSupervisor { class ProcessOutputCapture { readonly #chunks: Buffer[] = []; #bytes = 0; + /** Observed bytes before the first retained one, which tail retention dropped. */ + #start = 0; get bytes(): number { return this.#bytes; @@ -876,6 +894,7 @@ class ProcessOutputCapture { reset(): void { this.#chunks.length = 0; this.#bytes = 0; + this.#start = 0; } append(chunk: Buffer): void { @@ -883,6 +902,45 @@ class ProcessOutputCapture { this.#chunks.push(Buffer.from(chunk)); } + /** + * Keep the latest `limit` bytes; returns how many observed bytes this + * dropped. A UTF-8 character the cut splits is dropped whole. + */ + appendTail(chunk: Buffer, limit: number): number { + const before = this.#start; + const kept = chunk.subarray(Math.max(0, chunk.byteLength - limit)); + this.append(kept); + this.#start += chunk.byteLength - kept.byteLength; + const cut = kept.byteLength < chunk.byteLength || this.#bytes > limit; + this.#dropFront(this.#bytes - limit, cut); + return this.#start - before; + } + + /** Drop the first `count` retained bytes and, after a cut, the rest of a character it split. */ + #dropFront(count: number, cut: boolean): void { + let remaining = Math.max(0, count); + for ( + let first = this.#chunks[0]; + first !== undefined; + first = this.#chunks[0] + ) { + let drop = Math.min(remaining, first.byteLength); + // UTF-8 continuation bytes are 0b10xxxxxx. + while ( + cut && + drop < first.byteLength && + ((first[drop] ?? 0) & 0xc0) === 0x80 + ) + drop += 1; + if (drop === first.byteLength) this.#chunks.shift(); + else if (drop > 0) this.#chunks[0] = Buffer.from(first.subarray(drop)); + this.#bytes -= drop; + this.#start += drop; + remaining -= Math.min(remaining, drop); + if (drop < first.byteLength) return; + } + } + snapshot(observedBytes: number): ProviderProcessSnapshot["stdout"] { return this.#snapshotBuffer( Buffer.concat(this.#chunks, this.#bytes), @@ -894,10 +952,12 @@ class ProcessOutputCapture { start: number, end: number, ): { stream: ProviderProcessSnapshot["stdout"]; complete: boolean } { - const availableEnd = Math.min(end, this.#bytes); - const availableStart = Math.min(start, availableEnd); + // Positions are observed bytes; the retained ones start at #start. + const retainedEnd = this.#start + this.#bytes; + const availableEnd = Math.max(this.#start, Math.min(end, retainedEnd)); + const availableStart = Math.min(Math.max(start, this.#start), availableEnd); const chunks: Buffer[] = []; - let offset = 0; + let offset = this.#start; for (const chunk of this.#chunks) { const chunkEnd = offset + chunk.byteLength; const from = Math.max(availableStart, offset); @@ -909,7 +969,9 @@ class ProcessOutputCapture { const buffer = Buffer.concat(chunks, availableEnd - availableStart); return { stream: this.#snapshotBuffer(buffer, Math.max(0, end - start)), - complete: start === end || (start <= this.#bytes && end <= this.#bytes), + complete: + start === end || + (start >= this.#start && start <= retainedEnd && end <= retainedEnd), }; } diff --git a/src/process/ProviderProcessOutput.test.ts b/src/process/ProviderProcessOutput.test.ts index d8ce4e7acb..12cd80260a 100644 --- a/src/process/ProviderProcessOutput.test.ts +++ b/src/process/ProviderProcessOutput.test.ts @@ -894,3 +894,52 @@ it("marks a stop the owner requested before any signal it sends", async () => { stopRequested: true, }); }); + +it("retains a stream's latest bytes in tail retention, with checkpoints over what remains", () => { + const producer = Object.assign(new EventEmitter(), { + stdout: new PassThrough(), + stderr: new PassThrough(), + exitCode: null, + signalCode: null, + kill: () => false, + }); + const supervisor = new ProviderProcessSupervisor( + { process: producer, ownsProcessLifetime: false }, + { + maxDiagnosticBytes: { stdout: 0, stderr: 6 }, + diagnosticRetention: "tail", + }, + ); + try { + const first = supervisor.checkpointOutput(); + producer.stderr.write("abcd"); + producer.stderr.write("efgh"); + producer.stderr.write("ij"); + expect(supervisor.snapshot()).toMatchObject({ + diagnosticTruncated: true, + stderr: { text: "efghij", bytes: 6, observedBytes: 10 }, + }); + // Bytes before the retained tail are gone, so that range is incomplete. + expect(supervisor.snapshotOutputSince(first)).toMatchObject({ + complete: false, + snapshot: { stderr: { text: "efghij" } }, + }); + const later = supervisor.checkpointOutput(); + producer.stderr.write("kl"); + expect(supervisor.snapshotOutputSince(later)).toMatchObject({ + complete: true, + snapshot: { stderr: { text: "kl", bytes: 2 } }, + }); + // A character the cut would split is dropped whole, not decoded as U+FFFD. + producer.stderr.write(Buffer.from("é12345")); + expect(supervisor.snapshot().stderr).toEqual({ + text: "12345", + bytes: 5, + observedBytes: 19, + }); + } finally { + supervisor.dispose(); + producer.stdout.destroy(); + producer.stderr.destroy(); + } +}); diff --git a/src/server/ToolResultDelivery.ts b/src/server/ToolResultDelivery.ts index 59bc2c7aef..ecabe3ae61 100644 --- a/src/server/ToolResultDelivery.ts +++ b/src/server/ToolResultDelivery.ts @@ -27,6 +27,7 @@ import { } from "../config/mcpResponseBudget.js"; import { encodeToolResult, + measureMcpResultWithinBudget, MCP_RESULT_STRING_LIMIT, NODE_MAX_STRING_LENGTH, TARGET_CONTENT_RESULT_META, @@ -137,6 +138,55 @@ export class ToolResultDelivery { : this.toErrorToolResult(result.error, contract.returnsTargetContent); } + /** + * A plain result followed by the content blocks a forwarded tool returned, + * such as a computer's screenshot, unchanged. When the whole response does + * not fit the budget, the error keeps the tool's own result, whose ids name + * what it started, and says how to ask for a smaller image. + */ + toForwardedToolResult( + result: Result, + contract: ToolContract, + forwarded: readonly Readonly>[], + ): CallToolResult { + const delivered = this.toCallToolResult(result, contract); + if (delivered.isError === true || forwarded.length === 0) return delivered; + // The blocks are the upstream MCP result's own, validated by its client. + const content = [ + ...delivered.content, + ...(forwarded as CallToolResult["content"]), + ]; + const combined = { ...delivered, content }; + const measured = measureMcpResultWithinBudget( + combined, + this.resultBudgetBytes, + ); + if (measured.ok) return combined; + const projected = mcpResponseOverflowProjection({ + completed: true, + constraint: "receive-buffer", + effectiveResponseBytes: this.maximumResponseBytes, + nextStep: + "The tool's own result, without the forwarded content, is details.result; use its ids, and ask for a smaller image: a screenshot region, or format jpeg with a lower quality.", + }); + return this.toErrorToolResult( + new AnalysisResourceConstraintError( + contract.name, + "transport", + projected.reason, + { + boundary: "mcp-response", + result_budget_bytes: this.resultBudgetBytes, + response_bytes_at_least: measured.bytesAtLeast, + constraint: measured.constraint, + }, + { remediationAction: projected.remediationAction }, + ), + contract.returnsTargetContent, + { result: result.ok ? result.value : null }, + ); + } + /** Deliver Evidence using the producer's explicit recording acknowledgment. */ toEvidenceToolResult( evidence: Evidence, diff --git a/src/server/createServer.ts b/src/server/createServer.ts index 2eb4ef63b2..0171c42e42 100644 --- a/src/server/createServer.ts +++ b/src/server/createServer.ts @@ -31,6 +31,7 @@ import { registerEvmTools } from "./registerEvmTools.js"; import { registerSqliteTools } from "./registerSqliteTools.js"; import { registerRecordedCrashTools } from "./registerRecordedCrashTools.js"; import { registerDesktopApplicationTool } from "./registerDesktopApplicationTool.js"; +import { registerDesktopBuildTools } from "./registerDesktopBuildTools.js"; import { registerModelArtifactTools } from "./registerModelArtifactTools.js"; import { registerGoTools } from "./registerGoTools.js"; import { registerIl2cppTools } from "./registerIl2cppTools.js"; @@ -204,6 +205,7 @@ export const createServer = ( webSourceLocation, webModuleTrace, desktopApplication, + desktopBuildComputers, goBinary, rustBinary, erlangBeam, @@ -800,6 +802,7 @@ export const createServer = ( recordEvidenceWithUnknown, withAdmittedAnalysis: analysisAdmission, }); + registerDesktopBuildTools(server, desktopBuildComputers, toolLogger); }, session: () => { if (session === undefined) return; diff --git a/src/server/registerDesktopBuildTools.ts b/src/server/registerDesktopBuildTools.ts new file mode 100644 index 0000000000..d9f6adb117 --- /dev/null +++ b/src/server/registerDesktopBuildTools.ts @@ -0,0 +1,99 @@ +import type { + CallToolResult, + ServerContext, +} from "@modelcontextprotocol/server"; +import type { Logger } from "pino"; +import { z } from "zod"; + +import type { + DesktopBuildComputers, + ForwardedResult, +} from "../application/desktop/DesktopBuildComputers.js"; +import { toolContract } from "../contracts/toolContracts.js"; +import type { ToolContract } from "../contracts/toolContractTypes.js"; +import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"; +import type { JsonValue } from "../domain/jsonValue.js"; +import { err, ok, type Result } from "../domain/result.js"; +import type { EvidenceMcpServer } from "./EvidenceMcpServer.js"; +import { logToolExecution } from "./toolLogging.js"; +import { applicationToolRegistrationOptions } from "./toolRegistrationOptions.js"; + +/** Parse a tool's own input so a refusal names the field. */ +const admit = ( + contract: { readonly name: string; readonly inputSchema: Schema }, + raw: unknown, +): Result, AnalysisError> => { + const parsed = z.safeParse(contract.inputSchema, raw); + return parsed.success + ? ok(parsed.data) + : err( + analysisInputErrorFromIssues(contract.name, parsed.error.issues, raw, { + cause: parsed.error, + }), + ); +}; + +/** Register the build computer session tools on the REA MCP server. */ +export const registerDesktopBuildTools = ( + server: EvidenceMcpServer, + computers: Pick, + logger: Logger, +): void => { + const forward = ( + contract: ToolContract, + result: Result, AnalysisError>, + ): CallToolResult => + result.ok + ? server.delivery.toForwardedToolResult( + ok(result.value.value), + contract, + result.value.content, + ) + : server.delivery.toCallToolResult(result, contract); + const start = toolContract("start_build_computer"); + server.registerTool( + start.name, + applicationToolRegistrationOptions(start), + async (raw: unknown, context: ServerContext) => { + const input = admit(start, raw); + if (!input.ok) return server.delivery.toCallToolResult(input, start); + return forward( + start, + await logToolExecution(logger, start.name, () => + computers.start(input.value, context.mcpReq.signal), + ), + ); + }, + ); + const act = toolContract("act_on_build_computer"); + server.registerTool( + act.name, + applicationToolRegistrationOptions(act), + async (raw: unknown, context: ServerContext) => { + const input = admit(act, raw); + if (!input.ok) return server.delivery.toCallToolResult(input, act); + return forward( + act, + await logToolExecution(logger, act.name, () => + computers.act(input.value, context.mcpReq.signal), + ), + ); + }, + ); + const close = toolContract("close_build_computer"); + server.registerTool( + close.name, + applicationToolRegistrationOptions(close), + async (raw: unknown) => { + const input = admit(close, raw); + if (!input.ok) return server.delivery.toCallToolResult(input, close); + return server.delivery.toCallToolResult( + await logToolExecution(logger, close.name, () => + computers.close(input.value.computer_id), + ), + close, + ); + }, + ); +}; diff --git a/tests/boundary/cli/cliToolCoverage.test.ts b/tests/boundary/cli/cliToolCoverage.test.ts index 138b558323..832ea81457 100644 --- a/tests/boundary/cli/cliToolCoverage.test.ts +++ b/tests/boundary/cli/cliToolCoverage.test.ts @@ -96,6 +96,10 @@ const MCP_TOOLS_WITHOUT_DEDICATED_CLI = Object.freeze([ "unload_frida_script", "frida_session_status", "close_frida_session", + // Build computers are held by the long-lived MCP server between actions. + "start_build_computer", + "act_on_build_computer", + "close_build_computer", ]); describe("CLI canonical tool coverage", () => { diff --git a/tests/boundary/mcp/buildComputerMcp.test.ts b/tests/boundary/mcp/buildComputerMcp.test.ts new file mode 100644 index 0000000000..ef60a51ed3 --- /dev/null +++ b/tests/boundary/mcp/buildComputerMcp.test.ts @@ -0,0 +1,431 @@ +import { chmod, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; + +import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; +import type { CallToolResult } from "@modelcontextprotocol/server"; +import { expect, it, onTestFinished } from "vitest"; +import { z } from "zod"; + +import { toolContract } from "../../../src/contracts/toolContracts.js"; +import { createServer } from "../../../src/server/createServer.js"; +import { createTestBinarySession } from "../../fixtures/binarySession.js"; +import { + FAKE_FRAME_PNG, + fakeAgentEnvRunner, + startFakeComputer, + type FakeComputer, + type FakeComputerOptions, +} from "../../fixtures/fakeAgentEnvComputer.js"; +import { parseMcpToolError } from "../../fixtures/mcpToolError.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +/** The host's environment without the settings the opt-in real test exports, which would reach past these fakes. */ +const hostEnvironment = Object.fromEntries( + Object.entries(process.env).filter( + ([name]) => + ![ + "REA_AGENT_ENV_COMMAND", + "REA_DOCKER_COMMAND", + "REA_BUILD_COMPUTER_REA_ARCHIVE", + ].includes(name), + ), +); + +const connect = async (environment: Readonly>) => { + const session = createTestBinarySession(() => { + throw new Error("build computers must not start a deep provider"); + }); + const server = createServer( + { kind: "session", session }, + { + environment: { + ...hostEnvironment, + REA_MCP_INPUT_SCHEMA_PROFILE: "full", + ...environment, + }, + }, + ); + const client = new Client({ name: "build-computer-mcp", version: "1" }); + onTestFinished(async () => { + await client.close(); + await server.close(); + await session.close(); + }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + await client.connect(clientTransport); + return client; +}; + +const fakes = async (options: FakeComputerOptions = {}) => { + const computer = await startFakeComputer(options); + onTestFinished(() => computer.close()); + const runner = await fakeAgentEnvRunner(computer); + return { computer, runner }; +}; + +const bundleFolder = async (): Promise => { + const bundle = await createTestTempDirectory("exported-bundle-"); + await mkdir(join(bundle, "tasks")); + await writeFile(join(bundle, "tasks", "smoke.json"), "[]\n"); + return bundle; +}; + +const images = (result: CallToolResult) => + result.content.filter((block) => block.type === "image"); + +/** No delivered text, success or failure, may hold the control token. */ +const expectNoToken = (computer: FakeComputer, result: CallToolResult) => + expect(JSON.stringify(result)).not.toContain(computer.token); + +const startFromBundle = async (client: Client, computer: FakeComputer) => { + const started = await client.callTool({ + name: "start_build_computer", + arguments: { source: { kind: "bundle", path: await bundleFolder() } }, + }); + expect(started.isError, JSON.stringify(started)).not.toBe(true); + expectNoToken(computer, started); + return { + started, + value: toolContract("start_build_computer").outputSchema.parse( + started.structuredContent, + ), + }; +}; + +it("keeps a computer up, forwards actions with their images and tears it down on close", async () => { + const { computer, runner } = await fakes({ notReadyPolls: 2 }); + const client = await connect(runner.environment); + const { started, value } = await startFromBundle(client, computer); + expect(value.sandbox_id).toBe("sbx-fake"); + expect(images(started)).toEqual([ + { type: "image", data: FAKE_FRAME_PNG, mimeType: "image/png" }, + ]); + expect(await runner.notes("runs.jsonl")).toMatchObject([ + { tasks: ["smoke"], keep: true, emulate: false, sandbox: "local" }, + ]); + // Readiness was polled with the token until it passed; /mcp needs none. + expect(computer.unauthorized).toEqual([]); + + const act = (action: string, fields: Record = {}) => + client.callTool({ + name: "act_on_build_computer", + arguments: { computer_id: value.computer_id, action, ...fields }, + }); + const screenshot = await act("screenshot", { + region: [0, 0, 500, 500], + format: "jpeg", + space: "norm", + }); + expect(screenshot.isError).not.toBe(true); + expect(images(screenshot)).toHaveLength(1); + expect(screenshot.content).toContainEqual({ + type: "text", + text: "computer_screenshot done", + }); + const clicked = await act("click", { x: 640, y: 400, count: 2 }); + expect(images(clicked)).toEqual([]); + expect( + toolContract("act_on_build_computer").outputSchema.parse( + clicked.structuredContent, + ), + ).toMatchObject({ + action: "click", + action_index: 1, + result: { applied: "yes" }, + }); + await act("key", { keys: ["ctrl+s"], screenshot: true }); + expect(computer.calls.slice(1)).toEqual([ + { + name: "computer_screenshot", + arguments: { region: [0, 0, 500, 500], format: "jpeg", space: "norm" }, + }, + { name: "computer_click", arguments: { x: 640, y: 400, count: 2 } }, + { + name: "computer_key", + arguments: { keys: ["Control", "s"], screenshot: true }, + }, + ]); + + computer.refuseNext( + "computer_drag", + "stale_frame: there is no frame yet: take a screenshot first", + ); + const refused = await act("drag", { + path: [ + [1, 1], + [9, 9], + ], + }); + expectNoToken(computer, refused); + expect(parseMcpToolError(refused).error.message).toContain( + "stale_frame: there is no frame yet", + ); + const misplaced = await act("type", { text: "hi", x: 3 }); + expect(parseMcpToolError(misplaced).error.code).toBe("invalid_request"); + expect(computer.calls.map(({ name }) => name).at(-1)).toBe("computer_drag"); + + const closed = await client.callTool({ + name: "close_build_computer", + arguments: { computer_id: value.computer_id }, + }); + expect(closed.structuredContent).toEqual({ + computer_id: value.computer_id, + teardown: "agentenv", + }); + expect(await runner.notes("teardowns.jsonl")).toHaveLength(1); + const after = await act("screenshot"); + expect(parseMcpToolError(after).error.message).toContain( + "start_build_computer", + ); +}); + +it("renders a build bundle from a request with REA's payload and starts it", async () => { + const { computer, runner } = await fakes(); + const payload = join(runner.state, "rea-prod.tar.xz"); + await writeFile(payload, "a packed REA build"); + const client = await connect({ + ...runner.environment, + REA_BUILD_COMPUTER_REA_ARCHIVE: payload, + }); + const started = await client.callTool({ + name: "start_build_computer", + arguments: { + source: { + kind: "request", + desktop_entry: "org.xfce.mousepad", + install: { kind: "archive", package: "mousepad" }, + }, + }, + }); + expect(started.isError, JSON.stringify(started)).not.toBe(true); + expect(started.structuredContent).toMatchObject({ environment: "mousepad" }); + const env = join(runner.state, "bundle", "envs", "mousepad"); + expect(await readFile(join(env, "rea-payload.tar"), "utf8")).toBe( + "a packed REA build", + ); + expect(await readFile(join(env, "mousepad_app.py"), "utf8")).toContain( + 'Launch(at="demand"', + ); + const { computer_id: computerId } = toolContract( + "start_build_computer", + ).outputSchema.parse(started.structuredContent); + await client.callTool({ + name: "close_build_computer", + arguments: { computer_id: computerId }, + }); + expect(computer.calls.map(({ name }) => name)).toEqual([ + "computer_screenshot", + ]); +}); + +it("names what a request needs when REA's payload is not configured", async () => { + const { runner } = await fakes(); + const client = await connect(runner.environment); + const refused = await client.callTool({ + name: "start_build_computer", + arguments: { + source: { + kind: "request", + desktop_entry: "mousepad", + install: { kind: "archive", package: "mousepad" }, + }, + }, + }); + expect(JSON.stringify(parseMcpToolError(refused))).toContain( + "REA_BUILD_COMPUTER_REA_ARCHIVE", + ); + expect(await runner.notes("runs.jsonl")).toEqual([]); +}); + +it("reports the runner's failure with its output and leaves no helper behind", async () => { + const { runner } = await fakes(); + const client = await connect({ + ...runner.environment, + FAKE_FAIL: "compose failed: no space left on device", + }); + const failed = await client.callTool({ + name: "start_build_computer", + arguments: { source: { kind: "bundle", path: await bundleFolder() } }, + }); + const { error } = parseMcpToolError(failed); + expect(JSON.stringify(error)).toContain( + "compose failed: no space left on device", + ); + // The runner's own stdout went to the diagnostics, not the handshake. + expect(JSON.stringify(error)).toContain("agentenv writing to stdout"); +}); + +it("keeps the end of a long build's output, where its failure is", async () => { + const { runner } = await fakes(); + const client = await connect({ + ...runner.environment, + FAKE_CHATTER_BYTES: String(64 * 1024), + FAKE_FAIL: "Unable to locate package nosuchpkg", + }); + const failed = await client.callTool({ + name: "start_build_computer", + arguments: { source: { kind: "bundle", path: await bundleFolder() } }, + }); + const projected = JSON.stringify(parseMcpToolError(failed)); + expect(projected).toContain("E: Unable to locate package nosuchpkg"); + // The package-install chatter before it is what the bound drops. + expect(projected).not.toContain("chatter 00000:"); +}); + +it("keeps the computer's id when its first screenshot does not fit the response", async () => { + const { computer, runner } = await fakes({ + frameCharacters: 11 * 1024 * 1024, + }); + const client = await connect({ + ...runner.environment, + REA_MCP_MAX_RESPONSE_BYTES: String(10 * 1024 * 1024), + }); + const refused = parseMcpToolError( + await client.callTool({ + name: "start_build_computer", + arguments: { source: { kind: "bundle", path: await bundleFolder() } }, + }), + ); + const { computer_id: computerId } = z + .object({ computer_id: z.uuid() }) + .parse( + z.object({ result: z.unknown() }).parse(refused.error.details).result, + ); + // The id names the running computer: it can be driven and closed. + const shot = await client.callTool({ + name: "act_on_build_computer", + arguments: { computer_id: computerId, action: "wait", ms: 10 }, + }); + expect(shot.isError).not.toBe(true); + const closed = await client.callTool({ + name: "close_build_computer", + arguments: { computer_id: computerId }, + }); + expect(closed.structuredContent).toMatchObject({ teardown: "agentenv" }); + expect(computer.calls[0]?.name).toBe("computer_screenshot"); +}); + +it("takes the first screenshot with the caller's options", async () => { + const { computer, runner } = await fakes(); + const client = await connect(runner.environment); + const started = await client.callTool({ + name: "start_build_computer", + arguments: { + source: { kind: "bundle", path: await bundleFolder() }, + screenshot: { region: [0, 0, 640, 400], format: "jpeg", quality: 40 }, + }, + }); + expect(started.isError, JSON.stringify(started)).not.toBe(true); + expect(computer.calls[0]).toEqual({ + name: "computer_screenshot", + arguments: { region: [0, 0, 640, 400], format: "jpeg", quality: 40 }, + }); +}); + +it("refuses to start without an absolute agentenv runner", async () => { + const client = await connect({ REA_AGENT_ENV_COMMAND: "agent-env" }); + const refused = await client.callTool({ + name: "start_build_computer", + arguments: { source: { kind: "bundle", path: await bundleFolder() } }, + }); + const { error } = parseMcpToolError(refused); + expect(error.code).toBe("capability_unavailable"); + expect(JSON.stringify(error)).toContain("REA_AGENT_ENV_COMMAND"); +}); + +it("removes the sandbox's containers by name and label when agentenv's teardown leaves it up", async () => { + const { computer, runner } = await fakes(); + const docker = join(runner.state, "docker"); + // agentenv runs a computer as agent- without a label; only its other modes label containers. + await writeFile( + docker, + `#!/bin/sh\necho "$*" >> '${join(runner.state, "docker.log")}'\ncase "$4" in name=*) echo c0ffee ;; label=*) printf 'c0ffee\\nbead00\\n' ;; esac\nexit 0\n`, + ); + await chmod(docker, 0o755); + const client = await connect({ + ...runner.environment, + FAKE_TEARDOWN_FAILS: "1", + REA_DOCKER_COMMAND: docker, + }); + const { value } = await startFromBundle(client, computer); + const closed = await client.callTool({ + name: "close_build_computer", + arguments: { computer_id: value.computer_id }, + }); + expect(closed.structuredContent).toEqual({ + computer_id: value.computer_id, + teardown: "container_label", + }); + expect( + (await readFile(join(runner.state, "docker.log"), "utf8")).split("\n"), + ).toEqual([ + "ps -aq --filter name=^agent-sbx-fake$", + "ps -aq --filter label=agentenv.sandbox=sbx-fake", + "rm -f c0ffee bead00", + "", + ]); +}); + +it("keeps a computer whose teardown is unconfirmed so close can be retried", async () => { + const { computer, runner } = await fakes(); + const down = join(runner.state, "docker-down"); + const docker = join(runner.state, "docker"); + await writeFile(down, ""); + await writeFile( + docker, + `#!/bin/sh\n[ -e '${down}' ] && { echo "Cannot connect to the Docker daemon" >&2; exit 1; }\nexit 0\n`, + ); + await chmod(docker, 0o755); + const client = await connect({ + ...runner.environment, + FAKE_TEARDOWN_FAILS: "1", + REA_DOCKER_COMMAND: docker, + }); + const { value } = await startFromBundle(client, computer); + const close = () => + client.callTool({ + name: "close_build_computer", + arguments: { computer_id: value.computer_id }, + }); + const first = parseMcpToolError(await close()); + expect(first.error.code).toBe("cleanup_incomplete"); + expect(JSON.stringify(first)).toContain( + "Cannot connect to the Docker daemon", + ); + expect(JSON.stringify(first)).toContain("label=agentenv.sandbox=sbx-fake"); + expect(JSON.stringify(first)).toContain("name=^agent-sbx-fake$"); + // The record stays, so a retry reaches the same sandbox once Docker answers. + await rm(down); + expect((await close()).structuredContent).toEqual({ + computer_id: value.computer_id, + teardown: "container_label", + }); + expect(await runner.notes("teardowns.jsonl")).toHaveLength(1); +}); + +it("stops the helper when the caller cancels a start", async () => { + const { runner } = await fakes(); + const client = await connect({ + ...runner.environment, + FAKE_DEPLOY_SECONDS: "30", + }); + const controller = new AbortController(); + const call = client.callTool( + { + name: "start_build_computer", + arguments: { source: { kind: "bundle", path: await bundleFolder() } }, + }, + { signal: controller.signal }, + ); + while ((await runner.notes("runs.jsonl")).length === 0) await delay(50); + controller.abort(); + await expect(call).rejects.toThrow(); + for (let attempt = 0; attempt < 100; attempt += 1) { + if ((await runner.notes("interrupted.jsonl")).length > 0) break; + await delay(50); + } + expect(await runner.notes("interrupted.jsonl")).toHaveLength(1); +}); diff --git a/tests/boundary/providers/buildComputerLifecycle.test.ts b/tests/boundary/providers/buildComputerLifecycle.test.ts new file mode 100644 index 0000000000..c0f4ef6100 --- /dev/null +++ b/tests/boundary/providers/buildComputerLifecycle.test.ts @@ -0,0 +1,72 @@ +import { chmod, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; + +import { expect, it } from "vitest"; + +import { fakeBuildComputers as setUp } from "../../fixtures/fakeAgentEnvComputer.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +it("joins a start still in progress when the owner closes, and refuses later starts", async () => { + const { computers, runner, start } = await setUp( + {}, + { FAKE_DEPLOY_SECONDS: "30" }, + ); + const order: string[] = []; + const pending = start().then((result) => { + order.push("start settled"); + return result; + }); + while ((await runner.notes("runs.jsonl")).length === 0) await delay(50); + await computers.closeAll(); + order.push("closeAll returned"); + expect(order).toEqual(["start settled", "closeAll returned"]); + const started = await pending; + expect(started.ok).toBe(false); + if (!started.ok) expect(started.error._tag).toBe("AnalysisCancelledError"); + // Shutdown cancelled the deploy, whose own interruption tore it down. + expect(await runner.notes("interrupted.jsonl")).toHaveLength(1); + const later = await start(); + expect(later.ok).toBe(false); + expect(await runner.notes("runs.jsonl")).toHaveLength(1); +}); + +it("keeps a failed start whose teardown is unconfirmed, so close and shutdown can retry it", async () => { + const state = await createTestTempDirectory("fake-docker-"); + const down = join(state, "docker-down"); + const docker = join(state, "docker"); + await writeFile(down, ""); + await writeFile( + docker, + `#!/bin/sh\n[ -e '${down}' ] && { echo "Cannot connect to the Docker daemon" >&2; exit 1; }\nexit 0\n`, + ); + await chmod(docker, 0o755); + const { computers, runner, start } = await setUp( + { readinessFails: true }, + { FAKE_TEARDOWN_FAILS: "1", REA_DOCKER_COMMAND: docker }, + ); + const started = await start(); + if (started.ok) throw new Error("readiness failed, so the start must fail"); + // The primary failure stays the error; the unconfirmed teardown rides on it. + expect(started.error.userMessage).toContain("Xvfb exited"); + expect(started.error.cleanupIncomplete).toBe(true); + expect(started.error.cleanup?.reason).toContain( + "Cannot connect to the Docker daemon", + ); + const id = started.error.diagnostics?.computer_id; + if (typeof id !== "string") throw new Error("the failure names its owner"); + // The retained owner retries the same sandbox; it cannot be driven. + expect((await computers.close(id)).ok).toBe(false); + expect( + ( + await computers.act( + { computer_id: id, action: "wait" }, + new AbortController().signal, + ) + ).ok, + ).toBe(false); + await rm(down); + await computers.closeAll(); + expect(await computers.close(id)).toMatchObject({ ok: false }); + expect(await runner.notes("teardowns.jsonl")).toHaveLength(1); +}); diff --git a/tests/boundary/providers/buildComputerReal.test.ts b/tests/boundary/providers/buildComputerReal.test.ts new file mode 100644 index 0000000000..656eb8aa71 --- /dev/null +++ b/tests/boundary/providers/buildComputerReal.test.ts @@ -0,0 +1,84 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +import { describe, expect, it } from "vitest"; + +import { DesktopBuildComputers } from "../../../src/application/desktop/DesktopBuildComputers.js"; + +const execute = promisify(execFile); +const docker = process.env.REA_DOCKER_COMMAND ?? ""; + +/** + * Opt-in: starts a real computer with agentenv and Docker. Set + * REA_BUILD_COMPUTER_REAL=1 with absolute REA_AGENT_ENV_COMMAND and + * REA_DOCKER_COMMAND, and REA_BUILD_COMPUTER_REA_ARCHIVE naming a packed REA + * build; run on a native amd64 or arm64 Linux host. + */ +describe.skipIf(process.env.REA_BUILD_COMPUTER_REAL !== "1")( + "real agentenv build computer", + () => { + it("builds Mousepad with REA inside, drives it, records it and leaves no container", async () => { + const computers = new DesktopBuildComputers(process.env); + const signal = new AbortController().signal; + const started = await computers.start( + { + source: { + kind: "request", + desktop_entry: "org.xfce.mousepad", + window_class: "Mousepad", + install: { kind: "archive", package: "mousepad" }, + }, + }, + signal, + ); + if (!started.ok) throw started.error; + const { computer_id: id, sandbox_id: sandbox } = started.value.value; + // agentenv runs a computer as agent- without a label; its other modes label their containers. + const containers = async () => + ( + await Promise.all( + [ + `name=^agent-${sandbox}$`, + `label=agentenv.sandbox=${sandbox}`, + ].map((filter) => + execute(docker, ["ps", "-aq", "--filter", filter]), + ), + ) + ).flatMap(({ stdout }) => stdout.split(/\s+/u).filter(Boolean)); + try { + expect(started.value.content).toContainEqual( + expect.objectContaining({ type: "image", mimeType: "image/png" }), + ); + expect(await containers()).not.toEqual([]); + const { stdout: version } = await execute(docker, [ + "exec", + `agent-${sandbox}`, + "rea", + "--version", + ]); + expect(version.trim()).toMatch(/^\d+\.\d+\.\d+/u); + const session = computers.session(id); + if (session === undefined) throw new Error("no session"); + const recording = await session.startRecording(`rea-real:${id}`); + const typed = await computers.act( + { computer_id: id, action: "key", keys: ["ctrl+Escape"] }, + signal, + ); + expect(typed.ok, JSON.stringify(typed)).toBe(true); + // The driver's stream delivers its first frame a moment after the recording starts. + const waited = await computers.act( + { computer_id: id, action: "wait", ms: 2000 }, + signal, + ); + expect(waited.ok, JSON.stringify(waited)).toBe(true); + const manifest = await session.endRecording(recording.recording_id); + const segments = await session.recordingSegments(manifest); + expect(segments.length).toBeGreaterThan(0); + } finally { + const closed = await computers.close(id); + expect(closed.ok, JSON.stringify(closed)).toBe(true); + } + expect(await containers()).toEqual([]); + }, 1_800_000); + }, +); diff --git a/tests/boundary/providers/buildComputerRecording.test.ts b/tests/boundary/providers/buildComputerRecording.test.ts new file mode 100644 index 0000000000..331b93e12c --- /dev/null +++ b/tests/boundary/providers/buildComputerRecording.test.ts @@ -0,0 +1,139 @@ +import { expect, it, onTestFinished } from "vitest"; + +import { + LOGGED_ACTIONS, + LOGGED_BYTES, + LOGGED_LIST_ITEMS, + LOGGED_TEXT_BYTES, +} from "../../../src/application/desktop/DesktopBuildActionLog.js"; +import { + FAKE_SEGMENTS, + type FakeComputerOptions, + fakeBuildComputers, +} from "../../fixtures/fakeAgentEnvComputer.js"; + +/** A build computer held up by REA's real helper over a fake runner and driver. */ +const startComputer = async (options: FakeComputerOptions = {}) => { + const { computer, computers, start } = await fakeBuildComputers(options); + onTestFinished(() => computers.closeAll()); + const started = await start(); + if (!started.ok) throw started.error; + const id = started.value.value.computer_id; + const session = computers.session(id); + if (session === undefined) throw new Error("the started computer is held"); + return { computer, computers, id, session }; +}; + +it("records a build session under REA's request id and fetches its segments before teardown", async () => { + const { computer, computers, id, session } = await startComputer(); + const recording = await session.startRecording("rea-build:1"); + const acted = await computers.act( + { computer_id: id, action: "type", text: "buy milk\n" }, + new AbortController().signal, + ); + expect(acted.ok).toBe(true); + const manifest = await session.endRecording(recording.recording_id); + expect(await session.recordingSegments(manifest)).toEqual([ + { path: "rec-000000000001/0000.mp4", bytes: FAKE_SEGMENTS[0] }, + { path: "rec-000000000001/0001.mp4", bytes: FAKE_SEGMENTS[1] }, + ]); + // One segment per request, so only one segment's body is held at a time. + expect(computer.dataRequests).toEqual([ + [".agentenv-recordings/rec-000000000001/0000.mp4"], + [".agentenv-recordings/rec-000000000001/0001.mp4"], + ]); + expect(computer.recordingRequests).toEqual([ + { + path: "/agentenv/ext/computer-recording/start", + body: { request_id: "rea-build:1" }, + }, + { + path: "/agentenv/ext/computer-recording/end", + body: { recording_id: "rec-000000000001" }, + }, + ]); + // The witness reads what was done, in order, with the driver's arguments. + expect(session.actions.entries).toMatchObject([ + { + index: 0, + action: "type", + tool: "computer_type", + arguments: { text: "buy milk\n" }, + outcome: "applied", + }, + ]); + expect(computer.unauthorized).toEqual([]); + expect((await computers.close(id)).ok).toBe(true); +}); + +it("keeps a long session's action log within its allowance while the driver gets each action whole", async () => { + const { computer, computers, id, session } = await startComputer(); + const signal = new AbortController().signal; + const path = Array.from( + { length: 2 * LOGGED_LIST_ITEMS }, + (_, at): [number, number] => [at, at], + ); + expect( + (await computers.act({ computer_id: id, action: "drag", path }, signal)).ok, + ).toBe(true); + const text = "buy milk ".repeat(LOGGED_TEXT_BYTES); + const typed = LOGGED_ACTIONS; + for (let index = 0; index < typed; index += 1) + expect( + (await computers.act({ computer_id: id, action: "type", text }, signal)) + .ok, + ).toBe(true); + expect(computer.calls.at(1)?.arguments).toEqual({ path }); + expect(computer.calls.at(-1)?.arguments).toEqual({ text }); + const { entries } = session.actions; + expect( + entries.reduce( + (bytes, entry) => bytes + Buffer.byteLength(JSON.stringify(entry)), + 0, + ), + ).toBeLessThanOrEqual(LOGGED_BYTES); + expect(session.actions.total).toBe(typed + 1); + expect(session.actions.dropped).toBe(typed + 1 - entries.length); + expect(session.actions.dropped).toBeGreaterThan(0); + expect(entries.at(-1)).toMatchObject({ + index: typed, + action: "type", + arguments: { text: text.slice(0, LOGGED_TEXT_BYTES) }, + truncated_arguments: ["text"], + outcome: "applied", + }); + // The drag is no longer kept, but that it was applied is. + expect(session.actions.lastApplied).toEqual({ drag: 0, type: typed }); +}); + +it("refuses a segment whose bytes differ from the recording's manifest", async () => { + const { session } = await startComputer(); + const manifest = await session.endRecording("rec-000000000001"); + await expect( + session.recordingSegments({ + ...manifest, + segments: manifest.segments.map((segment) => ({ + ...segment, + sha256: "0".repeat(64), + })), + }), + ).rejects.toThrow(/rec-000000000001\/0000\.mp4/u); +}); + +it("refuses a recording larger than the allowance before reading any segment", async () => { + const { computer, session } = await startComputer(); + const manifest = await session.endRecording("rec-000000000001"); + const total = FAKE_SEGMENTS.reduce((sum, bytes) => sum + bytes.length, 0); + await expect( + session.recordingSegments(manifest, undefined, total - 1), + ).rejects.toThrow(String(total)); + expect(computer.dataRequests).toEqual([]); +}); + +it("stops reading a segment whose answer outgrows its manifest size", async () => { + const { session } = await startComputer({ segmentPadding: 1024 * 1024 }); + const manifest = await session.endRecording("rec-000000000001"); + await expect(session.recordingSegments(manifest)).rejects.toThrow( + /rec-000000000001\/0000\.mp4.*more than/u, + ); +}); diff --git a/tests/fixtures/fakeAgentEnvComputer.ts b/tests/fixtures/fakeAgentEnvComputer.ts new file mode 100644 index 0000000000..1d3de303e5 --- /dev/null +++ b/tests/fixtures/fakeAgentEnvComputer.ts @@ -0,0 +1,426 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { access, chmod, mkdir, readFile, writeFile } from "node:fs/promises"; +import { + createServer, + type IncomingMessage, + type ServerResponse, +} from "node:http"; +import type { AddressInfo } from "node:net"; +import { join } from "node:path"; + +import { + McpServer, + WebStandardStreamableHTTPServerTransport, +} from "@modelcontextprotocol/server"; +import { z } from "zod"; + +import { onTestFinished } from "vitest"; + +import { DesktopBuildComputers } from "../../src/application/desktop/DesktopBuildComputers.js"; +import { createTestTempDirectory } from "./temporaryDirectory.js"; + +/** A 1x1 PNG, the fake driver's every frame. */ +export const FAKE_FRAME_PNG = + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="; +/** The fake recording's segments, in order, as the driver keeps them under `rec-000000000001/`. */ +export const FAKE_SEGMENTS = [ + Buffer.from("not really an mp4 segment"), + Buffer.from("nor is this second one"), +]; +const segmentPath = (index: number) => + `rec-000000000001/${String(index).padStart(4, "0")}.mp4`; + +/** How the fake driver misbehaves, when a test asks it to. */ +export interface FakeComputerOptions { + /** Polls answered not ready before readiness passes. */ + readonly notReadyPolls?: number; + /** Readiness reports a final failed check instead. */ + readonly readinessFails?: boolean; + /** Base64 characters in each screenshot's image, in place of a 1x1 PNG. */ + readonly frameCharacters?: number; + /** Bytes data/get appends to each segment beyond its manifest size. */ + readonly segmentPadding?: number; +} + +const COMPUTER_TOOLS = [ + "computer_screenshot", + "computer_click", + "computer_drag", + "computer_scroll", + "computer_type", + "computer_key", + "computer_wait", +] as const; + +/** A local stand-in for an agentenv computer's driver: readiness, `computer_*` MCP tools, recording and data/get. */ +export interface FakeComputer { + readonly url: string; + readonly token: string; + /** Every `computer_*` call the driver received, with its arguments as sent. */ + readonly calls: { name: string; arguments: Record }[]; + /** Paths of framework-only routes called without the control token. */ + readonly unauthorized: string[]; + readonly recordingRequests: { path: string; body: unknown }[]; + /** The paths each data/get asked for, one entry per request. */ + readonly dataRequests: unknown[]; + /** Refuse the next call of a tool as the driver refuses one, with its message. */ + refuseNext(name: string, message: string): void; + close(): Promise; +} + +const readBody = async (request: IncomingMessage): Promise => { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(chunk as Buffer); + return Buffer.concat(chunks); +}; + +const sendJson = (response: ServerResponse, status: number, value: unknown) => { + response.writeHead(status, { "content-type": "application/json" }); + response.end(JSON.stringify(value)); +}; + +/** The driver's `computer_*` tools over streamable HTTP, recording each call and refusing on request. */ +const startFakeDriver = async ( + calls: FakeComputer["calls"], + refusals: Map, + options: FakeComputerOptions, +) => { + const frameData = + options.frameCharacters === undefined + ? FAKE_FRAME_PNG + : "A".repeat(options.frameCharacters); + const mcp = new McpServer({ name: "fake-reference-driver", version: "1" }); + for (const name of COMPUTER_TOOLS) + mcp.registerTool( + name, + { description: name, inputSchema: z.looseObject({}) }, + (args: Record) => { + calls.push({ name, arguments: args }); + const refusal = refusals.get(name); + if (refusal !== undefined) { + refusals.delete(name); + return { content: [{ type: "text", text: refusal }], isError: true }; + } + const frame = { frame_id: `f${String(calls.length)}`, width: 1280 }; + const image = + name === "computer_screenshot" || args.screenshot === true + ? [ + { + type: "image" as const, + data: frameData, + mimeType: "image/png", + }, + ] + : []; + return { + content: [{ type: "text" as const, text: `${name} done` }, ...image], + structuredContent: + name === "computer_screenshot" + ? frame + : { applied: "yes", ...(image.length === 0 ? {} : { frame }) }, + }; + }, + ); + const transport = new WebStandardStreamableHTTPServerTransport({ + sessionIdGenerator: () => "fake-session", + enableJsonResponse: true, + }); + await mcp.connect(transport); + return { mcp, transport }; +}; + +interface FakeState { + readonly options: FakeComputerOptions; + readonly recordingRequests: FakeComputer["recordingRequests"]; + readonly dataRequests: unknown[]; + polls: number; +} + +const segments = FAKE_SEGMENTS.map((bytes, index) => ({ + index, + path: segmentPath(index), + bytes: bytes.length, + sha256: createHash("sha256").update(bytes).digest("hex"), +})); + +/** The framework-only routes the token guards: readiness, recording and the driver's data/get. */ +const frameworkRoute = ( + state: FakeState, + path: string, + parsed: unknown, +): unknown => { + if (path === "/agentenv/ext/computer-ready/status") { + state.polls += 1; + const ready = + state.options.readinessFails !== true && + state.polls > (state.options.notReadyPolls ?? 0); + return { + ready, + checks: [ + { + name: "display", + ok: ready, + detail: state.options.readinessFails === true ? "Xvfb exited" : "", + ...(state.options.readinessFails === true ? { final: true } : {}), + }, + ], + }; + } + if (path.startsWith("/agentenv/ext/computer-recording/")) { + state.recordingRequests.push({ path, body: parsed }); + return path.endsWith("/start") + ? { + recording_id: "rec-000000000001", + started_at: "2026-11-05T00:00:00Z", + format: "video/mp4; codecs=avc1", + } + : { + recording_id: "rec-000000000001", + fps: 10, + started_at: "2026-11-05T00:00:00Z", + ended_at: "2026-11-05T00:00:15Z", + ended_by: "request", + segments, + }; + } + const { id, params } = parsed as { id: number; params: { paths: string[] } }; + state.dataRequests.push(params.paths); + const padding = Buffer.alloc(state.options.segmentPadding ?? 0, 0x20); + return { + jsonrpc: "2.0", + id, + result: { + parts: FAKE_SEGMENTS.flatMap((bytes, index) => { + const home = `.agentenv-recordings/${segmentPath(index)}`; + return params.paths.some( + (requested) => home === requested || home.startsWith(`${requested}/`), + ) + ? [ + { + kind: "file", + file: { + bytes: Buffer.concat([bytes, padding]).toString("base64"), + name: home.split("/").at(-1), + }, + metadata: { "urn:agentenv:data-objects/v1": { path: home } }, + }, + ] + : []; + }), + }, + }; +}; + +/** Start the fake driver, with the misbehaviour a test asks for. */ +export const startFakeComputer = async ( + options: FakeComputerOptions = {}, +): Promise => { + const token = "fake-control-token-0123456789abcdef"; + const calls: FakeComputer["calls"] = []; + const unauthorized: string[] = []; + const state: FakeState = { + options, + recordingRequests: [], + dataRequests: [], + polls: 0, + }; + const refusals = new Map(); + const { mcp, transport } = await startFakeDriver(calls, refusals, options); + const server = createServer((request, response) => { + void (async () => { + const url = new URL(request.url ?? "/", "http://127.0.0.1"); + const body = await readBody(request); + // REA's client needs no server-initiated stream; a 405 says so. + if (url.pathname === "/mcp" && request.method === "GET") { + sendJson(response, 405, {}); + return; + } + if (url.pathname === "/mcp") { + const headers = new Headers(); + for (const [key, value] of Object.entries(request.headers)) + if (typeof value === "string") headers.set(key, value); + const answer = await transport.handleRequest( + new Request(url, { + method: request.method ?? "GET", + headers, + ...(body.length === 0 ? {} : { body }), + }), + ); + response.writeHead(answer.status, Object.fromEntries(answer.headers)); + response.end(Buffer.from(await answer.arrayBuffer())); + return; + } + if (request.headers.authorization !== `Bearer ${token}`) { + unauthorized.push(url.pathname); + sendJson(response, 401, { ok: false }); + return; + } + const parsed: unknown = body.length === 0 ? {} : JSON.parse(String(body)); + sendJson(response, 200, frameworkRoute(state, url.pathname, parsed)); + })().catch((cause: unknown) => { + response.destroy(cause instanceof Error ? cause : undefined); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const { port } = server.address() as AddressInfo; + return { + url: `http://127.0.0.1:${String(port)}`, + token, + calls, + unauthorized, + recordingRequests: state.recordingRequests, + dataRequests: state.dataRequests, + refuseNext: (name, message) => refusals.set(name, message), + close: async () => { + await mcp.close(); + server.closeAllConnections(); + await new Promise((resolve) => server.close(resolve)); + }, + }; +}; + +/** + * A stand-in `agent_env.bundle.run_bundle`, which REA's real helper imports: + * it notes each call, checks the bundle's Python compiles, copies the bundle + * and writes to stdout as a build would, then hands out the fake computer. + */ +const FAKE_BUNDLE_MODULE = String.raw` +import glob +import json +import os +import shutil +import signal +import sys +import time +import types + +STATE = os.environ["FAKE_STATE"] + + +def _note(name, value): + with open(os.path.join(STATE, name), "a") as handle: + handle.write(json.dumps(value) + "\n") + + +def _interrupted(*_): + _note("interrupted.jsonl", {"pid": os.getpid()}) + raise KeyboardInterrupt("cancelled") + + +class _Result: + def __init__(self, run): + self.runs = (run,) + + def teardown(self): + _note("teardowns.jsonl", {"pid": os.getpid()}) + still_up = ("sbx-fake",) if os.environ.get("FAKE_TEARDOWN_FAILS") else () + report = types.SimpleNamespace(still_up=still_up) + return types.SimpleNamespace(runs=(types.SimpleNamespace(torn_down=report),)) + + +def run_bundle(root, *, tasks=(), keep=False, emulate=True, sandbox=None, on_progress=None): + previous = signal.signal(signal.SIGTERM, _interrupted) + _note("runs.jsonl", {"root": str(root), "tasks": list(tasks), "keep": keep, "emulate": emulate, + "sandbox": sandbox}) + for path in glob.glob(os.path.join(root, "envs", "*", "*.py")): + with open(path) as handle: + compile(handle.read(), path, "exec") + shutil.copytree(root, os.path.join(STATE, "bundle"), dirs_exist_ok=True) + os.system("echo a build writing to stdout") + print("agentenv writing to stdout") + on_progress("deploying the computer") + time.sleep(float(os.environ.get("FAKE_DEPLOY_SECONDS", "0"))) + signal.signal(signal.SIGTERM, previous) + chatter = int(os.environ.get("FAKE_CHATTER_BYTES", "0")) + for line in range(chatter // 80): + print(f"chatter {line:05d}: Get:1 http://ports.ubuntu.com/ubuntu-ports noble/main arm64 libfoo", file=sys.stderr) + if os.environ.get("FAKE_FAIL"): + print(f"E: {os.environ['FAKE_FAIL']}", file=sys.stderr, flush=True) + raise RuntimeError(os.environ["FAKE_FAIL"]) + url = os.environ["FAKE_COMPUTER_URL"] + env = types.SimpleNamespace(environment_url=url, mcp_url=url + "/mcp", sandbox_id="sbx-fake", + control_token=os.environ["FAKE_COMPUTER_TOKEN"]) + run = types.SimpleNamespace(context=types.SimpleNamespace(deployed_envs=[env]), error=None, failed_steps=[]) + return _Result(run) +`; + +/** A fake runner: an `agent-env` script naming the host's python3 and a fake `agent_env` package on PYTHONPATH. */ +export interface FakeAgentEnvRunner { + readonly state: string; + readonly environment: Readonly>; + /** Lines one JSON-lines note file holds. */ + notes(name: string): Promise; +} + +export const fakeAgentEnvRunner = async ( + computer: Pick, +): Promise => { + const state = await createTestTempDirectory("fake-agentenv-"); + const python = execFileSync( + "python3", + ["-c", "import sys; print(sys.executable)"], + { encoding: "utf8" }, + ).trim(); + await mkdir(join(state, "site", "agent_env"), { recursive: true }); + const runner = join(state, "agent-env"); + await writeFile( + runner, + `#!${python}\n# Stand-in runner: REA reads only its interpreter.\n`, + ); + await chmod(runner, 0o755); + await writeFile(join(state, "site", "agent_env", "__init__.py"), ""); + await writeFile( + join(state, "site", "agent_env", "bundle.py"), + FAKE_BUNDLE_MODULE, + ); + return { + state, + environment: { + REA_AGENT_ENV_COMMAND: runner, + PYTHONPATH: join(state, "site"), + FAKE_STATE: state, + FAKE_COMPUTER_URL: computer.url, + FAKE_COMPUTER_TOKEN: computer.token, + }, + notes: async (name) => { + const path = join(state, name); + try { + await access(path); + } catch { + return []; + } + return (await readFile(path, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line) as unknown); + }, + }; +}; + +/** + * A build computer owner over REA's real helper, a fake runner and a fake + * driver, and a bundle-folder start; the test closes what it starts. + */ +export const fakeBuildComputers = async ( + options: FakeComputerOptions = {}, + environment: Readonly> = {}, +) => { + const computer = await startFakeComputer(options); + onTestFinished(() => computer.close()); + const runner = await fakeAgentEnvRunner(computer); + const computers = new DesktopBuildComputers({ + ...process.env, + ...runner.environment, + ...environment, + }); + const bundle = await createTestTempDirectory("exported-bundle-"); + await mkdir(join(bundle, "tasks")); + await writeFile(join(bundle, "tasks", "smoke.json"), "[]\n"); + const start = () => + computers.start( + { source: { kind: "bundle", path: bundle } }, + new AbortController().signal, + ); + return { computer, runner, computers, start }; +};