morluto/rea
1
0 Fork 0
mirror of https://github.com/morluto/rea.git synced 2026-11-07 05:56:19 -10:00
Code Issues Packages Projects Releases 1 Wiki Activity

feat(desktop): capture, draft, export and witness a task inside a build computer (#9045)

Browse Source
This commit is contained in:
Edgar Arakelyan authored and GitHub committed 2026-11-07 17:54:59 +08:00
1 parent 8eff651270
commit c5301e6145
80 files changed
+8541 -479
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

bridge/agentenv/capture.py
+262
View File
@@ -0,0 +1,262 @@
"""Run REA inside an agentenv build computer for REA on the host: capture the app, draft its task and export it.
REA sends this file's source through the computer's ``computer-exec/v1`` route as
``python3 -I -c <source> <command> <JSON arguments>``; each command prints one JSON line and exits. ``start``
leaves the capture running on its own: its stdin is a FIFO it holds open for reading and writing, so it never
sees the end of its input, and ``mark`` and ``stop`` write their requests there and wait for the capture's own
acknowledgement on its progress log. A small shell around the capture writes its exit status to a file, the
mark a later command reads. REA runs as the session user in the home, with no ``AGENTENV_*`` variable, so
neither REA nor the app it launches holds the computer's control token, and with ``APT_CONFIG`` naming the
package lists ``start`` downloads as that user, so the capture finds the package's sources.
"""
from __future__ import annotations
import hashlib
import json
import os
import re
import shutil
import subprocess
import sys
import time
POLL_S = 0.2
TAIL_BYTES = 4096
def _say(value: dict) -> int:
print(json.dumps(value), flush=True)
return 0
def _tail(path: str) -> str:
try:
with open(path, "rb") as handle:
handle.seek(max(0, os.path.getsize(path) - TAIL_BYTES))
return handle.read().decode("utf-8", "replace")
except OSError:
return ""
def _environment(work: str) -> dict:
environment = {name: value for name, value in os.environ.items() if not name.startswith("AGENTENV_")}
environment["APT_CONFIG"] = os.path.join(work, "apt", "apt.conf")
return environment
def _apt(work: str, timeout: float) -> dict:
"""Download the package lists into the work folder as the session user, until one update succeeds."""
root = os.path.join(work, "apt")
lists = os.path.join(root, "lists")
updated = os.path.join(root, "updated")
os.makedirs(os.path.join(lists, "partial"), exist_ok=True)
os.makedirs(os.path.join(root, "cache", "archives", "partial"), exist_ok=True)
with open(os.path.join(root, "apt.conf"), "w", encoding="utf-8") as conf:
conf.write(
f'Dir::State::Lists "{lists}";\n'
f'Dir::Cache "{root}/cache";\n'
'Debug::NoLocking "true";\n'
'APT::Update::Error-Mode "any";\n'
)
# apt keeps lists compressed as it fetched them, such as *_Packages.lz4 on Ubuntu.
count = lambda: len([name for name in os.listdir(lists) if re.search(r"_Packages(\.\w+)?$", name)]) # noqa: E731
# A marker left by an update that fetched nothing is not a success; download again.
if os.path.exists(updated) and count() > 0:
return {"lists": count(), "updated": False}
log = os.path.join(root, "update.log")
try:
with open(log, "wb") as output:
code = subprocess.run(["apt-get", "update"], env=_environment(work), stdin=subprocess.DEVNULL,
stdout=output, stderr=subprocess.STDOUT, timeout=timeout).returncode
except (OSError, subprocess.TimeoutExpired) as error:
return {"lists": count(), "updated": True, "error": f"{type(error).__name__}: {error}"}
lists_fetched = count()
if code == 0 and lists_fetched > 0:
# Only a whole update that fetched lists is kept, so a failed or empty one runs again.
open(updated, "w").close()
return {"lists": lists_fetched, "updated": True, "exit_code": code}
if code == 0:
return {"lists": lists_fetched, "updated": True, "exit_code": code,
"error": "apt-get update exited 0 without a package list"}
return {"lists": lists_fetched, "updated": True, "exit_code": code,
"error": f"apt-get update exited {code}: {_tail(log)[-1500:].strip()}"}
def _exit_code(capture: str):
try:
with open(os.path.join(capture, "exit"), encoding="utf-8") as handle:
return int(handle.read().strip() or "-1")
except (OSError, ValueError):
return None
def _event_line(path: str, offset: int, kinds: set, name):
"""The first progress line after ``offset`` whose event is of one of ``kinds``, for ``name`` when given."""
try:
with open(path, "rb") as handle:
handle.seek(offset)
lines = handle.read().decode("utf-8", "replace").splitlines()
except OSError:
return None
for line in lines:
try:
event = json.loads(line)["rea_progress"]["event"]
except (ValueError, KeyError, TypeError):
continue
if isinstance(event, dict) and event.get("kind") in kinds and (name is None or event.get("name") == name):
return line
return None
def _wait(capture: str, offset: int, kinds: set, name, timeout: float) -> dict:
"""Wait for the capture's event, or for it to exit, or for time to run out."""
log = os.path.join(capture, "progress.log")
deadline = time.monotonic() + timeout
while True:
line = _event_line(log, offset, kinds, name)
if line is not None:
return {"line": line}
code = _exit_code(capture)
if code is not None:
return {"exited": code, "progress": _tail(log), "output": _tail(os.path.join(capture, "capture.out"))}
if time.monotonic() > deadline:
return {"timed_out": True, "progress": _tail(log)}
time.sleep(POLL_S)
def start(arguments: dict) -> int:
work, capture = os.path.abspath(arguments["work"]), os.path.abspath(arguments["capture"])
rea = shutil.which("rea")
if rea is None:
return _say({"error": "REA is not installed in this computer; start the computer from a request, which installs it."})
if os.path.exists(capture):
return _say({"error": f"the capture folder {capture} is already used"})
apt = _apt(work, arguments.get("apt_timeout_s", 600))
try:
os.makedirs(capture)
except FileExistsError:
return _say({"error": f"a stop claimed {capture} before the capture launched", "apt": apt})
control = os.path.join(capture, "control")
os.mkfifo(control, 0o600)
argv = [rea, "capture-desktop-application", "--desktop-entry", arguments["desktop_entry"],
"--output", os.path.join(capture, "capture"), "--exclude", work, "--json"]
for value in arguments.get("arguments", []):
argv += ["--argument", value]
if arguments.get("package_file"):
argv += ["--package-file", arguments["package_file"]]
# Reading and writing, so the capture never reads the end of its input between requests.
reader = os.open(control, os.O_RDWR)
try:
with open(os.path.join(capture, "capture.out"), "wb") as out, \
open(os.path.join(capture, "progress.log"), "wb") as progress:
process = subprocess.Popen(
["sh", "-c", 'done=$1; shift; "$@"; code=$?; printf "%s" "$code" > "$done.tmp" && mv "$done.tmp" "$done"',
"sh", os.path.join(capture, "exit"), *argv],
cwd=os.path.expanduser("~"), env=_environment(work), stdin=reader, stdout=out, stderr=progress,
start_new_session=True)
finally:
os.close(reader)
waited = _wait(capture, 0, {"window_open"}, None, arguments.get("timeout_s", 120))
return _say({"pid": process.pid, "apt": apt, **waited})
def _request(capture: str, line: str) -> bool:
"""Write one request to the running capture; False when nothing reads its FIFO any more."""
try:
fd = os.open(os.path.join(capture, "control"), os.O_WRONLY | os.O_NONBLOCK)
except OSError:
return False
try:
os.write(fd, (line + "\n").encode("utf-8"))
finally:
os.close(fd)
return True
def mark(arguments: dict) -> int:
capture, name = os.path.abspath(arguments["capture"]), arguments["name"]
code = _exit_code(capture)
log = os.path.join(capture, "progress.log")
if code is not None:
return _say({"exited": code, "progress": _tail(log)})
# A checkpoint already taken, whose answer was lost, is answered from the capture's own log.
taken = _event_line(log, 0, {"checkpoint"}, name)
if taken is not None:
return _say({"line": taken})
offset = os.path.getsize(log)
if not _request(capture, f"checkpoint {name}"):
return _say({"error": "the capture is not reading its requests"})
return _say(_wait(capture, offset, {"checkpoint", "checkpoint_refused"}, name, arguments.get("timeout_s", 600)))
def stop(arguments: dict) -> int:
capture = os.path.abspath(arguments["capture"])
if _exit_code(capture) is None and not os.path.exists(os.path.join(capture, "control")):
# Never launched: claim the folder, so a start still on its way finds it used and launches nothing.
os.makedirs(capture, exist_ok=True)
return _say({"absent": True})
if _exit_code(capture) is None:
_request(capture, "stop")
deadline = time.monotonic() + arguments.get("timeout_s", 600)
while (code := _exit_code(capture)) is None and time.monotonic() < deadline:
time.sleep(POLL_S)
log = os.path.join(capture, "progress.log")
if code is None:
return _say({"timed_out": True, "progress": _tail(log)})
output = os.path.join(capture, "capture.out")
return _say({"exit_code": code, "progress": _tail(log), "output_bytes": os.path.getsize(output)})
def run(arguments: dict) -> int:
"""Run one REA command with its output in a file, after writing the files it reads."""
rea = shutil.which("rea")
if rea is None:
return _say({"error": "REA is not installed in this computer"})
for path, text in arguments.get("files", {}).items():
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as handle:
handle.write(text)
with open(arguments["stdout"], "wb") as out:
try:
done = subprocess.run([rea, *arguments["argv"]], cwd=os.path.expanduser("~"),
env=_environment(os.path.abspath(arguments["work"])), stdin=subprocess.DEVNULL, stdout=out,
stderr=subprocess.PIPE, timeout=arguments.get("timeout_s", 1800))
except subprocess.TimeoutExpired:
return _say({"timed_out": True})
return _say({"exit_code": done.returncode, "stdout_bytes": os.path.getsize(arguments["stdout"]),
"stderr": done.stderr[-TAIL_BYTES:].decode("utf-8", "replace")})
def listing(arguments: dict) -> int:
"""Every regular file under a folder, by its path relative to the home, with its size and SHA-256.
The list goes to the file ``output`` names, outside the folder, and the answer gives its size: a large
bundle's list outgrows the 1 MiB of stdout a command answers with, so REA reads it through ``data/get``.
"""
home = os.path.expanduser("~")
files, skipped = [], []
for directory, folders, names in os.walk(os.path.abspath(arguments["root"])):
folders.sort()
for name in sorted(names):
path = os.path.join(directory, name)
relative = os.path.relpath(path, home)
if os.path.islink(path) or not os.path.isfile(path):
skipped.append(relative)
continue
digest = hashlib.sha256()
with open(path, "rb") as handle:
for chunk in iter(lambda: handle.read(1 << 20), b""):
digest.update(chunk)
files.append({"path": relative, "size": os.path.getsize(path), "sha256": digest.hexdigest()})
output = os.path.abspath(arguments["output"])
os.makedirs(os.path.dirname(output), exist_ok=True)
with open(output, "w", encoding="utf-8") as handle:
json.dump({"files": files, "skipped": skipped}, handle)
return _say({"listing_bytes": os.path.getsize(output)})
COMMANDS = {"start": start, "mark": mark, "stop": stop, "run": run, "list": listing}
if __name__ == "__main__":
sys.exit(COMMANDS[sys.argv[1]](json.loads(sys.argv[2])))
docs/.vitepress/config.ts
+4
View File
@@ -82,6 +82,10 @@ export default defineConfig({
text: "Desktop build computer",
link: "/desktop-build-computer",
},
{
text: "Building and witnessing",
link: "/desktop-build-witness",
},
{ text: ".NET assemblies", link: "/managed-code-analysis" },
{
text: "Python packaged applications",
docs/desktop-application-capture.md
+12 -2
View File
@@ -84,8 +84,18 @@ gone, the capture looks for any process still carrying the run token, so a
worker whose launcher exited before it was sampled keeps the session open. A
sample whose window list could not be read is not taken as an empty one; the
limitations say how many there were. One queued line is handled between
samples, and reading stdin pauses while 16 lines wait. Progress lines on stderr say when
the window has opened and when each checkpoint is taken.
samples, and reading stdin pauses while 16 lines wait.
Progress lines on stderr are JSON, `{"rea_progress": {"phase":
"desktop_capture", "message": ...}}`. Those a driver waits on also carry an
`event`: `{"kind": "window_open"}` once the window opens; `{"kind":
"checkpoint", "name", "trigger", "at_ms", "entries", "changes",
"retained_files", "retained_bytes"}` once a checkpoint has been taken, the
baseline, `stop` and `exit` included, with the paths walked, the paths
changed since the checkpoint before and the files retained; and `{"kind":
"checkpoint_refused", "name", "reason"}` for a requested name it refused. A
driver that sends `checkpoint <name>` waits for the event naming it rather
than for a fixed time.
When the session ends the capture sends SIGTERM to the launched process, its
process group and every process that carries the run token, waits up to five
docs/desktop-application-export.md
+10 -6
View File
@@ -118,11 +118,12 @@ The class declares:
taskbar or says it is a splash, and an app whose splash shows first would
otherwise count as ready before its own window opens. Arguments that are the captured home or under it, and
`<option>=<path>` values under it, are respelled under the computer user's
`$HOME`. agentenv starts the app in its own folder, so when an argument, or
the program's own path such as `./app`, is relative the command first
changes to the captured working directory
(`"$HOME"` when that was the home), and still starts if that folder is
missing. The export refuses a capture that saw no application window.
`$HOME`. agentenv starts the app in its own folder, where an open or save
dialog would show the framework's files, so the command first changes to the
captured working directory (`"$HOME"` when that was the home, as for a
desktop entry without `Path`), which a relative argument or program path
such as `./app` also needs, and still starts if that folder is missing. The
export refuses a capture that saw no application window.
- For an Electron application (the capture found its fuse sentinel or a
`resources/app.asar`), `--no-sandbox` right after the program unless the
command already has it: Chromium's sandbox needs user namespaces the
@@ -154,7 +155,10 @@ retain is not seeded: one in the state view without retained content at that
checkpoint, or one the capture omitted (for example too large or past the
content budget). The result lists them in `unseeded_files` and the README
names them. When the application held files open in folders whose starting
content the capture did not retain, a limitation names those folders.
content the capture did not retain, a limitation names those folders. The
desktop Trash, `~/.local/share/Trash`, is never the application's, so a file
the user moved there is neither owned nor seeded, even when a window title
names it; a limitation lists the files the Trash held at the seed.
The application's dconf keys at the seed, if any, are written once when the
app first starts, also for an application that owns no files, and the class adds
docs/desktop-application-state.md
+4 -2
View File
@@ -29,14 +29,16 @@ changed later. Each path records the evidence the capture holds about it:
| `selected_path` | It lies under a `--state-path` or absolute `--argument` of the capture |
| `dconf_application_keys` | It is a dconf database holding groups whose path names the application |
| `desktop_service` | It lies where desktop services and toolkits keep shared files, or caches |
| `desktop_trash` | It lies in the desktop Trash, `~/.local/share/Trash` |
A folder beside a state directory whose name differs from it only in letter
case counts as that state directory, so a profile the application first
creates during the capture, such as GIMP's `~/.config/GIMP` beside the
candidate `~/.config/gimp`, is its state too.
The role follows from the reasons, in this order: `application_state` for
application keys in dconf, or a state directory outside shared desktop
The role follows from the reasons, in this order: `shared_desktop` for a file
in the desktop Trash, which the user deleted, whatever else names it;
`application_state` for application keys in dconf, or a state directory outside shared desktop
locations; `document` for a file named in a window title or selected by the
caller, even if the application holds it open; `application_state` for another
open file outside shared locations; `shared_desktop` for other files in shared
docs/desktop-application-tasks.md
+7 -3
View File
@@ -301,9 +301,13 @@ capture host's name and user removed, and at the end checkpoint, the
captured end state. Those are the capture's first and last checkpoints unless
`--seed-checkpoint` and `--end-checkpoint` name others.
It refuses a task whose every check passes on the seed, and a check that fails
on the end state, with that check's reason. A capture yields one task, so the
option is taken once. The result's `task` names the task, its number of checks
and those that fail on the seed. A refused task writes no bundle.
on the end state, with that check's reason. Only a failure on what the seed's
view shows counts: a task whose checks fail on the seed only because a view
could not be read or may have left out what they look for, such as a line past
the 5,000 a text view keeps, is refused, naming each failure. A capture yields
one task, so the option is taken once. The result's `task` names the task, its
number of checks and those that fail on what the seed shows. A refused task
writes no bundle.
The bundle then holds, besides the app and `tasks/smoke.json`:
docs/desktop-build-computer.md
+47 -13
View File
@@ -12,7 +12,9 @@ agent's computer-use actions to it through three MCP tools:
| `close_build_computer` | Tears the computer down, by agentenv's teardown or by removing its sandbox's containers |
The tools are MCP-only, like the debugger sessions: the computer lives as long
as the REA MCP server that started it, or until it is closed.
as the REA MCP server that started it, or until it is closed. Capturing the
app, exporting its bundle and witnessing the task on such a computer are in
[Building and witnessing an environment](desktop-build-witness.md).
## Requirements
@@ -26,7 +28,11 @@ as the REA MCP server that started it, or until it is closed.
command to run instead.
- For a request, `REA_BUILD_COMPUTER_REA_ARCHIVE`: the absolute path of a tar
archive of a runnable REA, with its production dependencies, whose one top
folder holds `scripts/rea.mjs`. The computer runs REA from it.
folder holds `scripts/rea.mjs`. The computer runs REA from it. From a
checkout, `npm run pack:build-computer-runtime -- /path/to/rea-prod.tar.xz`
writes one: `npm pack`, its package folder as
`rea-prod` with the lockfile, `npm ci --omit=dev --omit=optional`, then
`rea-prod` as `.tar.xz`.
Set these where the MCP server starts; a shell export does not reach a server
that is already running.
@@ -35,8 +41,9 @@ that is already running.
From a request, REA renders a bundle in a private temporary folder: a computer
app on the `ubuntu-24.04` base with the `reference` driver that installs the
app, opens it on demand with `gtk-launch <desktop entry>` and keeps no seed and
no data hooks. The app is not open when the computer starts: agentenv lists it
app, opens it on demand with `gtk-launch <desktop entry>` from the computer
user's home, so its open and save dialogs start there rather than in the
framework's folder for the app, and keeps no seed and no data hooks. The app is not open when the computer starts: agentenv lists it
in the desktop's Applications menu under Other by the environment's name, such
as `mousepad`, and that entry launches it through the app's server.
@@ -80,14 +87,25 @@ starts that bundle's `tasks/smoke.json` as it is:
{ "source": { "kind": "bundle", "path": "/home/ada/mousepad-bundle" } }
```
REA runs its helper with `PYTHONDONTWRITEBYTECODE=1`, which agentenv's own
import of the bundle's app class inherits, so a start leaves no `__pycache__`
in the folder and its files stay as they were shared.
REA runs the deploy with `run_bundle(keep=True, emulate=False)` on the local
sandbox provider, so only the host's own architecture is built. The first start
of a bundle builds its image and can take minutes; later starts reuse it. REA
then waits until the computer's readiness checks pass, opens one MCP session
to its driver and takes a screenshot, with the `region`, `format` and `quality`
given in `screenshot`. The result holds `computer_id`, agentenv's `sandbox_id`,
the environment name or bundle folder, and the screenshot's frame as structured
content beside its image.
of a bundle builds its image and can take minutes; later starts reuse it.
While it waits, REA sends an MCP progress notification every five seconds with
agentenv's latest line, so a client that resets its request timeout on
progress keeps waiting. REA then waits until the computer's readiness checks
pass and opens one MCP session to its driver. Readiness can pass before the
desktop has drawn, so REA takes a whole-screen PNG screenshot every 500 ms
while it is one colour throughout. The retries share 10 s, each given only
what is left of it, and when that runs out REA returns the latest frame;
progress notifications continue meanwhile. Given `screenshot` with a
`region`, `format` or `quality`, REA then takes one with them, a `region`
cropping that frame. The result holds
`computer_id`, agentenv's `sandbox_id`, the environment name or bundle folder,
and the screenshot's frame as structured content beside its image.
A failed start returns the last 16 KiB of the helper's stderr, where a failing
build prints its error, and its exit status; a cancelled one stops the helper,
@@ -110,6 +128,15 @@ arguments for `computer_<action>`, which REA passes through:
| `type` | `text`, where `\n` presses Enter and `\t` Tab |
| `key` | `keys`, one chord pressed in order, and `repeat` |
| `wait` | `ms` |
| `reset` | `screenshot` |
`reset` is not a driver tool: REA calls the app's `data/reset`, which restores
the bundle's seed and relaunches the app, then waits until the computer's
readiness checks pass again, sending an MCP progress notification every five
seconds while it waits. With `screenshot: true` it then takes a screenshot; if
that fails, the reset still counts as applied and its result has no `frame`
but a `screenshot_error` with the computer's message. A computer started from
a request has no data operations, so it refuses a reset.
Pointer actions also take `frame_id`, `space` and `norm_scale`, and every
action but `screenshot` takes `screenshot: true` for an image afterwards. That
@@ -129,6 +156,11 @@ larger than the MCP budget is refused with how to ask for a smaller image: a
`computer_id` included, as `details.result`, since the action or start has
happened.
Actions, the capture steps and the task checks on one computer run one at a
time, in the order REA receives them; one that has to wait for the call
before it sends an MCP progress notification every five seconds meanwhile.
Different computers run side by side.
Every action is logged on the computer in order with the arguments sent, its
start and end and whether it was applied, refused or failed. The log keeps the
latest 256 actions within 256 KiB of JSON, dropping the oldest first. Each
@@ -140,8 +172,8 @@ the last applied action of each kind for rules over the whole session. The log
and the computer's recording hooks (`computer-recording/v1` start and end under
REA's own request id, and the segments fetched through the driver's `data/get`
one at a time, each read within its manifest size and checked against it) are
kept for later build stages; they are not tools. A recording larger than a
quarter of the V8 heap is refused before any segment is read.
what the witness records; they are not tools of their own. A recording larger
than a quarter of the V8 heap is refused before any segment is read.
## Close
@@ -153,7 +185,9 @@ containers of that exact sandbox: agentenv runs a computer as
`agentenv.sandbox=<sandbox id>`, so REA lists both with `docker ps -aq
--filter`, removes them with `docker rm -f`, and says
`teardown: "container_label"`. When neither confirms it, the error names what
is left and the computer stays listed, so closing it again retries. Closing the
is left and the computer stays listed, so closing it again retries. A close
cancels the call in progress on the computer, refuses the calls waiting for
it, and tears down once that call has ended. Closing the
MCP server closes every computer the same way, and a helper whose REA server
exits sees its stdin close and tears down.
docs/desktop-build-witness.md
+175
View File
@@ -0,0 +1,175 @@
# Building and witnessing an environment
An agent such as Claude Code builds a desktop app environment from a request
on a [build computer](desktop-build-computer.md): it makes the seed through
the app, makes the change the task asks for, exports the bundle with the task
it writes, and then witnesses the task in a fresh computer of that bundle.
REA runs its own capture, task draft and export inside the build computer and
grades the witness with the same checks the exported verifier runs.
| Tool | What it does |
| -------------------------- | ------------------------------------------------------------------------------------------- |
| `start_build_capture` | Downloads the package lists as the computer's user and launches the app under REA's capture |
| `mark_build_checkpoint` | Takes a named checkpoint and returns the capture's acknowledgement with its counts |
| `stop_build_capture` | Stops the capture and returns its checkpoints and the task draft between `seed` and `end` |
| `export_build_environment` | Exports the bundle with the agent's task inside the computer and copies it to a folder here |
| `check_build_task` | Grades a task against the app's live state; with a `stage`, files the check in the witness |
Like the computer tools, these are MCP-only and act on a `computer_id` from
`start_build_computer`.
## Build
1. `start_build_computer` from a request. REA is installed in the computer.
2. `start_build_capture`. REA writes an apt configuration under
`~/.rea-build/apt` and runs `apt-get update` as the computer's user, so the
capture finds the package's sources (`package_lists` counts the lists,
and `package_lists_error` gives apt's exit status and last lines when the
download failed; a later start runs the update again until one succeeds). It then starts `rea capture-desktop-application
--desktop-entry <entry>`
in the computer, leaving `~/.rea-build` out of every checkpoint, and returns
once the app's window is open, with a screenshot. For an app installed
from a `.deb` file, the build layer keeps that file at
`/opt/rea/package.deb` and the capture records it with `--package-file`. Do not open the app from
the menu first: the capture refuses an app whose window is already open.
3. Make the seed through the app with `act_on_build_computer`, saving it
where the user keeps such files, such as `~/Documents`, then
`mark_build_checkpoint` `seed`.
4. Make the change the task will ask for, save it, and
`mark_build_checkpoint` `end`.
5. `stop_build_capture`. The app is closed and the capture finishes. The
result lists the checkpoints with their counts, the package and the
sources the capture found, its limitations and the draft: each file that
changed between `seed` and `end` with its net change, and suggested
checks with a placeholder prompt.
6. Write the task: its name, the prompt in the user's own words, and checks
that hold after the change and not on the seed.
7. `export_build_environment` with the task and a new or empty `directory`.
The export runs inside the computer with `--seed-checkpoint seed
--end-checkpoint end`, so it seeds what was saved at `seed`, scrubs the
computer's host name and user from it, and refuses a task whose checks
all pass on the seed. A task larger than 96 KiB as REA passes it to the
computer is refused before anything runs. A file moved to the desktop Trash is not seeded,
and the export's limitations name what the Trash held at `seed`. For a
`.deb` app that no apt source the capture found offers, the export takes
the request's URL with `--package-url`; one apt offers installs from apt. REA lists the bundle's files with their sizes
and SHA-256 in the computer, into a file beside the bundle that it reads
through `data/get`, since a large bundle's list outgrows a command's
output; it then brings the files back through the driver's
`data/get` in answers of at most 8 MiB, and checks and writes each answer
into the folder before asking for the next, so one answer at a time is
held; a failure leaves no partial folder.
8. `close_build_computer`.
A checkpoint name is lowercase letters, digits, dot, dash and underscore; the
capture's own `baseline`, `stop`, `time-limit` and `exit` are refused, and so
is a name already taken. The acknowledgement comes from the capture after it
has walked the home and retained what changed: `entries` walked, `changes`
since the checkpoint before, `retained_files` and `retained_bytes`.
Each step can be asked again after its answer was lost:
- A start whose answer is lost, or whose window never opened, stops the
capture it may have left running. Until the computer confirms that stop,
REA keeps the capture and the next `start_build_capture` retries the stop
before starting another.
- A checkpoint the capture took whose acknowledgement was lost comes back
from the capture's own log when the same name is marked again.
- A stop whose summary or draft was not read finishes when stop is asked
again, and a finished stop repeats its result. The draft and the export
trust the checkpoints the capture's own Evidence records.
- A step that saved something, such as an export's bundle or the witness's
record, returns it even when its call is cancelled or the computer closes
as the save finishes. `start_build_capture` saves nothing, so a close or
cancellation during its screenshot of the opened app cancels it.
The capture, the draft and the export run as the computer's user in its home,
without any `AGENTENV_*` variable, so neither REA nor the app holds the
computer's control token. REA sends each command through agentenv's
`computer-exec/v1` route with that token, which stays in REA, and reads larger
outputs, such as the draft, the export's Evidence and the bundle's file
list, from files under `~/.rea-build` through `data/get`, since a command's
output is capped at 1 MiB.
Calls on one computer run one at a time, in the order REA receives them:
an action, a capture step, a check or a close waits until the call before
it on that computer has answered, and sends progress notifications while
it waits. So a check reads the app with no action half done, and a witness
stage is admitted, read and filed before anything else on that computer
runs. Calls on different computers run side by side. Closing a computer
cancels the call in progress on it and refuses the calls still waiting.
## Witness
Start a fresh computer from the exported folder with `start_build_computer`
and `{"kind": "bundle", "path": ...}`; agentenv boots the app with the seed.
Then, with the task exactly as exported:
| `stage` | Before it | Its checks must |
| ------------- | ----------------------------------------------------------------- | ---------------------------- |
| `seed` | nothing; give `directory`, a new or empty folder for the record | fail at least once |
| `after_edit` | the task done through the app from its prompt alone | all pass |
| `near_miss` | a change that is almost right, such as a misspelled line | fail at least once |
| `after_reset` | `act_on_build_computer` with `action: "reset"` as the last change | each come out as on the seed |
`check_build_task` reads the app's `data/get`, takes the views it returns
beside the files and evaluates each check with REA's evaluator, the one the
export validates tasks with and the exported verifier mirrors, one result per
check. Without `stage` it only grades. The `seed` stage begins a recording
of the display under REA's own request id, one per computer; a seed check
that fails after it, such as on a failed read of the app's state or a lost
reply to the recording's start, can be asked again, and asks for the
recording under the same id, which the computer answers with the recording
it already began. The stages come in order. The edit and the near miss each
need an applied click, drag, scroll, type or key since the stage before, with
no reset after the last of them, since a reset puts the seed back rather than
changing the app; `after_reset` needs the reset as the last change, and a
reset that leaves any check unlike on the seed fails the witness, its
stage's `remediation` naming those checks. On a computer started from a
bundle folder, the seed refuses a task whose `tasks/<name>.json` or verifier
in the bundle differs from what the export writes for it. A reset
relaunches the app, and REA waits until the computer is ready again before
answering, sending progress notifications while it waits.
Only a check that fails on what the app's view shows counts as a failure. A
check that fails because its file or table could not be read, such as an
SQLite file locked past the reader's wait, or because what it looks for may
lie in what its view cut, is `undecided` in its result: it shows nothing
about the task. That covers a line `lines_present` misses in a text view cut
at 5,000 lines, a row `row_present` misses in a table whose view keeps only
its first rows, a `value_equals` pointer that names nothing in a truncated
view or past a cut zip listing, and the absence checks and counts on a cut
view. A check that passes on what the view kept stays decided, and so does a
value found there that differs from the one expected. The result's `passed`
and description are the exported verifier's, which has no undecided results.
A stage that only undecided checks would settle, such as a near miss whose
one failing check could not read its file, files nothing and is refused;
check it again once the files read whole.
A stage whose checks do not show what it must ends the witness, and so does
`after_reset`. REA then ends the recording, fetches its segments through
`data/get`, each checked against the recording's manifest, and writes into
`directory`:
- `recording/NNNN.mp4`, the recording's segments;
- `witness.json`, REA Evidence (`rea.desktop-build-witness`) holding the
verdict, the stage that failed if one did, the task's name and SHA-256,
each stage's results with how many logged actions came before it, the
computer's [action log](desktop-build-computer.md#act) as it keeps it, each
action with its arguments, timing and outcome, with `actions_dropped`
counting the earliest actions it no longer keeps, and the recording's
segments with their sizes and SHA-256. The record's limitations say when
the log dropped actions or cut arguments, which each entry names in
`truncated_arguments`.
The stages' rules read the log's count of actions and the last applied
action of each kind, which it keeps for the whole session, so they hold
however many entries it dropped. The result returns the same record. When writing it fails, such as for a
folder something else wrote into, the witness keeps its stages: check the
stage that ended it again, with the same task, and REA writes the record
without checking again. Once it is saved, the same check returns the saved
record again, so a record saved as its call was cancelled is not lost. A
computer is witnessed once; witness again from a fresh computer.
See the [specification](specs/desktop-app-environments/build-witness/spec.md)
for scope and verification.
docs/index.md
+1
View File
@@ -64,6 +64,7 @@ For runtime instrumentation, see the [Frida guide](./frida-instrumentation.md).
| Export a captured application as an environment | [Desktop application export](desktop-application-export.md) |
| Write and validate a task for an exported app | [Desktop application tasks](desktop-application-tasks.md) |
| Drive a computer while building an environment | [Desktop build computer](desktop-build-computer.md) |
| Capture, export and witness a built environment | [Building and witnessing](desktop-build-witness.md) |
| Use CLI/MCP and retain evidence | [CLI](cli.md), [MCP contracts](mcp-contracts.md), [agent prompts](mcp-prompts.md), [obligation ledgers](reconstruction-obligation-ledgers.md) |
For development, use [testing](testing.md), [client verification](client-verification.md),
docs/specs/desktop-app-environments/README.md
+5 -1
View File
@@ -42,7 +42,11 @@ The work lands as one PR per step:
12. [Build computer](build-computer/spec.md): an agentenv computer REA keeps
up from a request or a bundle, with REA installed inside for building,
driven through REA's MCP tools while an agent builds the environment.
13. [XML text checks](xml-text-checks/spec.md): XML whose DOCTYPE only names
13. [Build and witness](build-witness/spec.md): REA's capture, task draft
and export run inside the build computer with acknowledged checkpoints,
the bundle comes back to the host, and the task is witnessed live in a
fresh computer of it: seed, edit, near miss and reset, recorded.
14. [XML text checks](xml-text-checks/spec.md): XML whose DOCTYPE only names
its DTD read as XML, starting with AbiWord, and checks of an element's
whole text across the runs an app splits a paragraph into.
docs/specs/desktop-app-environments/build-witness/plan.md
+32
View File
@@ -0,0 +1,32 @@
# Plan
1. Capture acknowledgement: `desktopCaptureEventSchema` and
`desktopCaptureProgressSchema` in the capture's domain schema; the session
reports `window_open` and `checkpoint_refused`, the capture a
`checkpoint` event after each checkpoint is taken; the CLI writes the event
in its progress line.
2. Controller: `bridge/agentenv/capture.py`, with `start`, `mark`, `stop`,
`run` and `list`, each answering one JSON line.
3. Adapter: `AgentEnvComputer` gains `exec`, `homeFiles`, `appData` and
`waitUntilReady`, and reports start progress;
`src/desktop/BuildComputerController.ts` runs the controller and fetches a
home folder in bounded, checked batches.
4. Domain: `buildCapture.ts` (checkpoint names, `seed` and `end`) and
`buildWitness.ts` (stages, their order and expectations, the app's state
views); `computerActions.ts` gains `reset`.
5. Application: `DesktopBuildCapture` (start, mark, stop and draft) with
`DesktopBuildExport` and the shared `DesktopBuildCommands`, and
`DesktopBuildWitness` with `DesktopBuildWitnessRecord`, held per live
computer by `DesktopBuildComputers`, which also resets;
`DesktopBuildSteps` serves the step tools by `computer_id`. The output
tree writer moves to `DesktopOutputTree.ts` for the export, the build
export and the witness record.
6. MCP: contracts, effects, CLI-parity list, registration with heartbeat
progress.
7. Launch from the home in `launchCommand` and the build bundle; the packing
script.
8. Docs: the guide, the capture, export and build computer guides, this spec.
9. Tests first: the witness rules, the reset action, the launch commands, the
capture's acknowledgements on Linux, and boundary tests through the MCP
tools with a fake computer that runs the real controller against a REA
stand-in printing the real CLI's outputs; then the amd64 VM.
docs/specs/desktop-app-environments/build-witness/spec.md
+113
View File
@@ -0,0 +1,113 @@
# Build and witness
## Outcome
An agent that builds a desktop app environment on a build computer captures
the app with REA inside the computer, marks the seed and the end and gets an
acknowledgement for each, drafts the task, exports the bundle with the task
it wrote and brings the bundle back to the host, all through REA's MCP tools.
It then witnesses the task live in a fresh computer of that bundle: the seed
fails a check, the edit made from the prompt passes every check, a near miss
fails one, and after `data/reset` one fails again. REA keeps the action log
and the recording and writes the record as Evidence.
## Requirements
- MCP-only session tools in the `application` family, on a build computer's
`computer_id`: `start_build_capture`, `mark_build_checkpoint`,
`stop_build_capture`, `export_build_environment` and `check_build_task`, and
a `reset` action of `act_on_build_computer`.
- Commands in the computer go through agentenv's `computer-exec/v1` route
with the control token, which stays in REA and is removed from any output
REA reports. A controller, `bridge/agentenv/capture.py`, travels with each
command as `python3 -I -c`; it runs REA as the session user in the home
without `AGENTENV_*` variables and with `APT_CONFIG` naming apt lists it
downloads into `~/.rea-build/apt` before the first capture.
- The capture runs detached, its stdin a FIFO it holds open for reading and
writing; a shell around it records its exit status. `~/.rea-build` is left
out of every walk. A start returns once the capture reports the window
open; a refused or early-ended start stops the capture and reports its
output.
- Acknowledgement: `capture-desktop-application` reports, beside each progress
message, an `event`: `window_open`; `checkpoint` with its name, trigger,
time, entries walked, changes and retained files and bytes, once it is
taken, the baseline, `stop` and `exit` included; and `checkpoint_refused`
with the reason. `mark_build_checkpoint` writes `checkpoint <name>` and
waits for the event naming it, returning its counts. Names follow a narrow
pattern, without the capture's own names; a name already taken is refused.
- `stop_build_capture` sends `stop`, waits for the exit status, reads the
capture's Evidence and, when `seed` and `end` were acknowledged, runs
`describe-desktop-application-task --seed-checkpoint seed --end-checkpoint
end` and returns the changed files, their net change and suggested checks.
- `export_build_environment` writes the agent's task under `~/.rea-build`,
runs `export-desktop-application` with it and the same checkpoints into a
new folder there, reports a refused task with the export's own reason,
lists the bundle with sizes and SHA-256, fetches it through the driver's
`data/get` in answers of at most 8 MiB, checks each file and writes the
caller's new or empty folder through the safe output tree.
- `check_build_task` reads the one app's `data/get` through the card's
child URL, takes the state views beside its files and evaluates the task's
checks with `evaluateTaskChecks`, one result per check. With `stage`, the
checks file a witness stage: `seed` (with `directory`) starts a recording
and must fail a check; `after_edit` must pass all; `near_miss` must fail
one; `after_reset` must give each check its seed result. Stages come in order, for one task; the
edit and the near miss each need an applied input action since the stage
before, `after_reset` a reset as the last change. A stage that does not hold,
or the last, ends the witness: the recording ends, its segments are fetched
and checked, and `directory` gets the segments and `witness.json`, Evidence
of the verdict, the stages, the action log and the recording.
- The `reset` action calls the app's `data/reset`, then waits for the
computer's readiness checks, since the reset relaunches the app.
- Gaps of the build computer's first real run: a packing script for
`REA_BUILD_COMPUTER_REA_ARCHIVE`; build and exported apps start in the home
(the captured working directory), not in agentenv's folder for the app; and
MCP progress notifications every five seconds while a computer starts and
while the capture, draft, export and witness tools work.
- Retries keep ownership: a start whose answer is lost stops its capture,
and REA keeps a capture whose stop is unconfirmed, retrying that stop
before another start; a lost checkpoint acknowledgement comes back from
the capture's log; a stop resumes reading its result or repeats it; the
draft and export read the checkpoints from the capture's Evidence; a seed
check that failed after opening is asked again with its task and folder;
a record whose save failed is written again on a repeat of its stage.
- The edit and the near miss count only click, drag, scroll, type and key,
and a reset after the last of them refuses the stage.
- A `.deb` app's file is kept in the build layer for `--package-file`, and
its URL reaches the export's `--package-url` only when the capture's
package facts say apt does not install it, as the export itself decides.
A failed `apt-get update` reports its status and last lines, and the next
start runs it again. The bundle is
written as it arrives.
- Gaps of a blind validation run: the desktop Trash is never the app's state
or a document, so a file the builder trashed is neither owned nor seeded,
and the export's limitations name what the Trash held at the seed; a
start from a bundle folder writes no bytecode into it; the first
screenshot waits, briefly and bounded, until the frame is not one colour.
- Gaps of the second review: calls on one computer (actions, capture
steps, checks and close) run one at a time in arrival order, so a stage's
admission, read and filing see no other call, while computers run side by
side; a close cancels the call in progress and refuses those waiting. The
witness's recording has one request id per computer, so a seed retried
after a lost reply asks for the recording it began. A check that fails for
want of a readable or whole view is `undecided` and proves nothing; a
stage only such checks would settle files nothing and can be checked
again. Undecided includes a failure to find a line, row or value that may
lie in what a cut view left out; a check that passes on what the view kept
stays decided. The export likewise keeps a task only when a check fails on
what the seed's view shows. The controller writes a folder's file list to a
file that REA reads through `data/get`, so a large bundle's list is not cut
at the 1 MiB of a command's output.
- The witness reads the build computer's bounded action log: its stage rules
use the log's count and the last applied action of each kind, and the
record holds the entries the log keeps, `actions_dropped`, and a
limitation when the log dropped actions or cut arguments.
## Non-goals
- Running the exported `verify.py` in the witness; REA's evaluator is the
one it mirrors, and the Python parity tests cover the port.
- A witness by a separate solving agent, remote sandbox providers, emulated
architectures, and choosing the near miss for the agent.
- Recording the build itself; only the witness is recorded.
docs/specs/desktop-app-environments/build-witness/tasks.md
+13
View File
@@ -0,0 +1,13 @@
# Tasks
- [x] Record the outcome, requirements and non-goals.
- [x] Acknowledge checkpoints from the capture with their counts.
- [x] Run capture, draft and export in the computer and bring the bundle back.
- [x] Check a task against a live app and witness it with a recording.
- [x] Reset the app's data and wait for it to be ready.
- [x] Pack REA's runtime, start apps in the home, report start progress.
- [x] Document the tools.
- [x] Add the tests and run the build and witness on an amd64 host.
- [x] Fix the blind validation run's gaps and the export's install source for a `.deb`.
- [x] Fix the second review's gaps: one call at a time per computer, the recording's request id, undecided checks and the bundle's file list.
- [x] Leave a failure that a cut view may have caused undecided: a missing line, row or value.
docs/specs/desktop-app-environments/build-witness/verification.md
+252
View File
@@ -0,0 +1,252 @@
# Verification
Run on 2026-11-05: gates and source tests on macOS 26.5 with Node v22.23.1
and Python 3.11; Linux tests and the real run on Ubuntu 24.04 amd64 with
Docker, agentenv-framework 0.9.1431 and Node v24.22.1.
## Tests
Each new test fails on the code before this change, run from that code with
only the new tests and fixtures added: on macOS, and on Linux for the
capture's acknowledgements.
- `src/application/desktop/DesktopApplicationCapture.test.ts` (Linux): the
capture reports the baseline, `window_open`, `seed` with the counts its
checkpoint recorded, a refused second `seed`, then `stop` and `exit`.
Before, no event reached the driver and the test ran out of time.
- `src/domain/desktop/buildWitness.test.ts`: each stage's expectation, the
order of stages with the actions each needs, and the app's state views.
- `src/domain/desktop/computerActions.test.ts`: `reset` maps to
`data/reset` and takes only `screenshot`.
- `src/domain/desktop/appExport.test.ts`, `appBuild.test.ts`: exported and
build apps start in the captured working directory, `"$HOME"` for the home.
- `tests/boundary/mcp/buildCaptureMcp.test.ts`: through the MCP tools, a
request computer whose `computer-exec` runs REA's real controller on this
host, against a REA stand-in that acknowledges requests as the capture
does and prints what the real CLI wrote for a seed-and-end capture: package
lists under `APT_CONFIG` (compressed, as Ubuntu keeps them), the start with
a screenshot, `seed` with its counts, a repeated and a reserved name
refused, `end`, the draft, and the export with the task, whose files arrive
byte for byte. REA ran without `AGENTENV_*`, in the home, with
`--exclude ~/.rea-build` and both checkpoints. Steps out of order and a
capture on a bundle's computer are refused. No result holds the token.
- `tests/boundary/mcp/buildWitnessMcp.test.ts`: a plain check; the witness
with a fake app whose state the test changes; the edit refused before an
action, `after_reset` before a reset; the reset relaunch; the record with
four held stages, the action log, the recording segment and
`witness.json` read back as Evidence; a seed that already passes ends the
witness as failed and later stages are refused; `directory` missing or not
empty is refused.
- PR 2's boundary tests now share `tests/fixtures/buildComputerMcp.ts`.
## Gates
`typecheck`, `format:check`, `build:cached`, `knip`, `jscpd`, `docs:check`,
`docs:build`, `verify:test-discovery`, `verify:architecture-guards`,
`verify:module-boundaries` and `mcp-catalog:check` passed. `lint` and
`lint:typed` fail only on main's `tests/acceptance/webSourceLocationCli.test.ts`.
jscpd's one clone in a touched file is the existing pair of cancellation
tests in `DesktopApplicationCapture.test.ts`. The desktop, contract, schema,
CLI-parity and build boundary tests passed on macOS (Linux-only cases
skipped) and, on Linux, the desktop and build tests ran 467 passed.
## Real computer
REA's archive was packed from this tree with `npm run
pack:build-computer-runtime`, 16.8 MiB in 54 s. Through `rea mcp` over stdio,
driven as the building agent:
- `start_build_computer` from the Mousepad request built its image and came
up in 80 s, with an MCP progress notification every 5 s naming agentenv's
stage. A later start took 22.5 s.
- `start_build_capture` returned in 7.3 s with Mousepad open under capture.
A typed two-line list saved through Save As, which opened on the home, as
`~/Documents/todo.txt`; `seed` was acknowledged in 0.45 s (54 entries, 7
changes, 4 files retained) and a second `seed` refused; a third line saved,
`end` in 0.45 s.
- `stop_build_capture` took 1.7 s: checkpoints `baseline`, `seed`, `end`,
`stop`, `exit`; `mousepad` 0.6.1-1build2 from `noble/universe`; the draft's
one file, `Documents/todo.txt`, with the line added and a suggested
`lines_present`. This run counted 0 package lists, because apt keeps them
as `*_Packages.lz4`; the count was fixed and a later start reported 15.
- `export_build_environment` refused a `line_count` without `exact` or
`contains` with the export's reason, then exported the task in 1.3 s: 9
files, 50,846 bytes, `failing_on_seed` `dentist` and `one-dentist`, seed
`Documents/todo.txt`, nothing left unscrubbed, and the app launched with
`cd "$HOME"`. `close_build_computer` took 0.5 s.
- A fresh computer of the bundle came up in 72 s. The witness held: seed
failed two checks; the line added through Open and Save passed all three;
`call the dentists` failed `dentist`; the reset, ready in 0.8 s, relaunched
Mousepad and its check failed two again. The record has 15 actions and 5
recording segments, 68.6 s and 1.2 MB, written with `witness.json`.
- Every computer closed with `teardown: "agentenv"`; Docker held only the
local registry it had before. The MCP server's log never names a token.
## Review fixes
Run on macOS after the first review; the VM was not used. Each new test
failed on the reviewed head, with only the new tests and fixtures added, and
passes now. The fake computer now runs a command and then loses its reply
when a test asks, fails an app's `data/get` and keeps a relaunched app
unready for some polls.
- `buildWitness.test.ts`: a reset is neither the edit nor the near miss, and
a reset after the last input refuses them; a near miss made after a reset
is admitted.
- `appBuild.test.ts`: the build layer copies a `.deb` to
`/opt/rea/package.deb` before the app's own install removes it.
- `BuildComputerController.test.ts`: each checked answer is handed over
before the next is fetched.
- `buildCaptureRetryMcp.test.ts`: a start whose reply was lost stops its
capture, and while that stop's reply is lost too REA refuses another start
naming the capture, then retries the stop first; a lost checkpoint reply
comes back on a repeat; a stop whose draft reply was lost finishes on a
repeat and repeats its result, and the export follows; apt's exit 100 and
message reach `package_lists_error`; a `.deb` request captures with
`--package-file` and exports with `--package-url`.
- `buildWitnessMcp.test.ts`: a seed whose read failed is checked again with
the one recording; a record whose save was refused is saved on a repeat of
`after_reset` without reading the app again; a slow reset sends progress.
## Validation-run fixes
Run on macOS after a blind validation run of the build flow and a review
finding; the VM was not used. Each new test failed on be30603ec with only it
and its fixture changes added, unless noted, and passes now.
- `stateRoles.test.ts`: a path in `~/.local/share/Trash` has the reason
`desktop_trash` and the role `shared_desktop`, whatever else names it,
a window title included; `~/.local/share/Trashcan` is not the Trash.
- `DesktopApplicationExport.test.ts`: a file named in the window title and
moved to the Trash before the seed checkpoint is neither owned nor seeded,
and a limitation names both of the Trash's files. Before,
`.local/share/Trash/files` was owned and the file seeded, as the run saw
with `poster.svg`.
- `buildComputerMcp.test.ts`: the fake runner loads each of the bundle's
Python files in a child Python as agentenv's `_declare` does, and a start
from a bundle folder leaves its file list unchanged. Before,
`envs/notes/__pycache__/notes_app.cpython-311.pyc` appeared. In
agentenv-framework 0.9.1444 that child is the only Python that imports the
bundle's files, and it inherits the helper's environment.
- `buildComputerMcp.test.ts`: after two black frames, the start returns the
third, drawn one; a `region`, `format` or `quality` applies to a screenshot
taken after it. The fake's frame is now a 2x1 PNG of two colours.
`PngVisualDiff.test.ts` covers the new one-colour check.
- `buildComputerMcp.test.ts`, after review: with every frame black and the
third screenshot's reply a minute late, the start returns the second frame
at the 10 s bound and reports `waiting for the desktop to draw` as
progress meanwhile. Before, the retry kept its 120 s request timeout and
progress stopped once readiness passed, so the test ran out of time.
- `buildCaptureRetryMcp.test.ts`: a `.deb` request whose captured package
apt also offers exports without `--package-url`; one that no apt source
offers exports with it, from the real CLI's output for that capture (this
case passed before too). `appExportInstall.test.ts`: the new check agrees
with the export's own install choice.
`typecheck`, `format:check`, `build:cached`, `knip`, `jscpd`, `docs:check`,
`docs:build`, `verify:test-discovery`, `verify:architecture-guards`,
`verify:module-boundaries` and `mcp-catalog:check` passed; `lint` and
`lint:typed` fail only on main's `tests/acceptance/webSourceLocationCli.test.ts`.
The desktop source tests ran 453 passed (Linux-only cases skipped), and the
build and desktop boundary tests 67 passed with the real test skipped.
Not verified on a real computer: that the first frame not of one colour
shows the panel on the computer's desktop.
## Second review fixes
Run on macOS after the second review; the VM was not used. Each new
boundary test failed on the reviewed head, with only the new tests and
fixtures added, and passes now. The fake computer now cuts a command's
stdout at 1 MiB as the computer does, loses a recording start's reply when
a test asks, and a bundle folder can name the fake computer its start
hands out, so one server can hold two.
- `buildWitnessMcp.test.ts`: two `after_edit` checks sent together file
one stage, the other is refused as out of order, and `after_reset` stays
refused until a near miss. Before, both were filed, and after a reset the
record held with the stages `seed`, `after_edit`, `after_edit`,
`after_reset`. An `after_reset` check sent while a reset relaunches the
app waits for it and is admitted; before, it ran during the reset and was
refused. A seed retried after its recording's start reply was lost asks
for the same request id; before, a new one. A seed and a near miss whose
only failing check could not read its file are refused and filed once
the file reads; before, the read error was filed as the failure.
- `buildComputerMcp.test.ts`: a screenshot sent behind a slow reset on one
computer answers after it, while a screenshot on another computer answers
first, and the waiting action sends progress; a close during a reset
cancels it and refuses an action and a capture step waiting behind it,
and tears the computer down. Before, the screenshot answered first and
the close left the reset running.
- `buildComputerLifecycle.test.ts`: the owner's shutdown cancels a call in
progress and the one waiting behind it.
- `buildCaptureRetryMcp.test.ts`: an export whose bundle's file list is
about 1.1 MB brings every file back. Before, the list was cut at 1 MiB and
the export failed with the controller not answering.
- `DesktopBuildTurns.test.ts`: calls run one at a time in arrival order, a
call that throws frees its turn, a call cancelled while it waits keeps its
place until the one before it ends, and a close runs once the call it
cancelled has ended.
- `buildWitness.test.ts` and `appTaskChecks.test.ts`: the three-way stage
verdict, and which failures are `undecided`.
## Bounded action log
Run on macOS after merging the build computer's bounded action log; the VM
was not used. The stage rules now read the log's count and the last applied
action of each kind, and the record its kept entries.
- `buildWitness.test.ts`: every earlier admission case, through the log a
list of actions makes, and the rules over a log of 1,000 actions whose
earliest are dropped.
- `buildWitnessMcp.test.ts`: a witness of 263 actions, one typing 2 KiB,
is admitted at every stage and held; its record keeps 256 actions from
index 7 with `actions_dropped` 7, names `text` in the typed action's
`truncated_arguments`, and its Evidence states both limitations.
## Failures on cut views
Run on macOS; the VM was not used. The first three tests failed on the
reviewed head, with only the tests and fixtures added, and pass now.
- `appTaskChecks.test.ts`: `lines_present` missing line 5,500 of a
6,000-line file, `row_present` missing row 501 of a table cut at 500, and
`value_equals` whose pointer names nothing in a cut text, SQLite, XML or
zip listing view are undecided; a line, row or value the view kept stays
decided. Before, each was a decided failure.
- `buildWitnessMcp.test.ts`: a seed and a near miss whose only failing check
misses `eggs` in a text view cut at 5,000 lines are refused and filed once
the view is whole. Before, the seed was filed as held.
- `DesktopTaskExport.test.ts`: a task whose only seed failure misses `milk`
in a 6,000-line note is refused, naming the failure, and with a row check
that fails on the seed too it is kept with only that check in
`failing_on_seed`. Before, the export kept the first task.
- `desktopTaskPython.test.ts`: the exported verifier fails the same cut-view
cases with the same descriptions, as it did before.
## Cancellation after a step
Run on macOS; the VM was not used. A step returns what it saved even when
its call is cancelled or the computer closes as the save finishes; only
`start_build_capture`, which saves nothing, checks after its last await, the
screenshot of the opened app, and ends cancelled. Each test below except the
first failed on the head that cancelled every aborted turn, with only the
tests added, and passes now.
- `buildCaptureRetryMcp.test.ts`: a close while `start_build_capture` waits
for that screenshot makes it return `cancelled`. Before the start's own
check, it returned success for the computer being removed. A close while
an export's bundle commits still returns the bundle; before, `cancelled`.
- `buildWitnessMcp.test.ts`: a close while the witness record commits still
returns the record, and a check cancelled while it commits gets the saved
record when asked again; a saved witness's repeated final check returns the
same record without checking or writing. Before, `cancelled`, and the
repeat was refused as ended.
- `DesktopBuildTurns.test.ts`: a running call that finishes after a close
keeps its result.
## Limits of this verification
- The witness acts were scripted by this agent from the task's prompt; no
separate solving agent ran, and the exported `verify.py` was not run.
- Only the local provider on amd64, and only Mousepad.
package.json
+2
View File
@@ -70,6 +70,7 @@
"bridge/ghidra/ReaGhidraPrepareDol.java",
"bridge/android/ReaJadxBridge.java",
"bridge/mitmproxy/capture.py",
"bridge/agentenv/capture.py",
"bridge/agentenv/computer.py",
"scripts/rea.mjs",
"scripts/lib/cli-process-environment.mjs",
@@ -112,6 +113,7 @@
"scripts": {
"release:notes": "node scripts/release-notes.mjs",
"deps:check": "node scripts/check-dependency-install.mjs",
"pack:build-computer-runtime": "node scripts/pack-build-computer-runtime.mjs",
"metadata:generate": "node scripts/run-exclusive.mjs artifacts node scripts/generate-package-metadata.mjs",
"metadata:check": "node scripts/generate-package-metadata.mjs --check",
"mcp-registry:check": "node scripts/verify-mcp-registry-metadata.mjs",
scripts/pack-build-computer-runtime.mjs
+54
View File
@@ -0,0 +1,54 @@
#!/usr/bin/env node
// Pack this checkout as the REA a build computer runs, the archive REA_BUILD_COMPUTER_REA_ARCHIVE names:
// `npm pack`, its package folder as rea-prod with the lockfile, production dependencies installed with
// `npm ci --omit=dev --omit=optional`, then rea-prod as a .tar.xz.
// Usage: node scripts/pack-build-computer-runtime.mjs <archive.tar.xz>
import { spawnSync } from "node:child_process";
import { copyFile, mkdtemp, readdir, rename, rm, stat } from "node:fs/promises";
import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
const root = join(dirname(fileURLToPath(import.meta.url)), "..");
const npm = process.platform === "win32" ? "npm.cmd" : "npm";
const run = (command, args, cwd) => {
const result = spawnSync(command, args, { cwd, stdio: "inherit" });
if (result.error !== undefined) throw result.error;
if (result.status !== 0)
throw new Error(
`${command} ${args.join(" ")} failed with status ${String(result.status)}`,
);
};
const [destination] = process.argv.slice(2);
if (destination === undefined || !destination.endsWith(".tar.xz")) {
process.stderr.write(
"Usage: node scripts/pack-build-computer-runtime.mjs <archive.tar.xz>\n",
);
process.exit(2);
}
const archive = resolve(destination);
const work = await mkdtemp(join(tmpdir(), "rea-build-runtime-"));
try {
run(npm, ["pack", "--pack-destination", work], root);
const [packed] = (await readdir(work)).filter((name) =>
name.endsWith(".tgz"),
);
if (packed === undefined) throw new Error("npm pack wrote no .tgz");
run("tar", ["-xzf", packed], work);
const runtime = join(work, "rea-prod");
await rename(join(work, "package"), runtime);
await copyFile(
join(root, "package-lock.json"),
join(runtime, "package-lock.json"),
);
run(npm, ["ci", "--omit=dev", "--omit=optional"], runtime);
run("tar", ["-cJf", archive, "-C", work, "rea-prod"], work);
const { size } = await stat(archive);
process.stdout.write(
`${archive} (${(size / 1024 / 1024).toFixed(1)} MiB): set REA_BUILD_COMPUTER_REA_ARCHIVE to it.\n`,
);
} finally {
await rm(work, { recursive: true, force: true });
}
src/application/desktop/DesktopApplicationCapture.test.ts
+74 -1
View File
@@ -14,7 +14,10 @@ import { join } from "node:path";
import { describe, expect, it, vi } from "vitest";
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
import { desktopApplicationCaptureSchema } from "../../domain/desktop/desktopCapture.js";
import {
desktopApplicationCaptureSchema,
desktopCaptureEventSchema,
} from "../../domain/desktop/desktopCapture.js";
import type { SessionRequest } from "../../desktop/DesktopApplicationSession.js";
import { OwnedFileHandle } from "../../filesystem/OwnedFileHandle.js";
import {
@@ -387,6 +390,76 @@ describe.skipIf(process.platform !== "linux")(
},
);
describe.skipIf(process.platform !== "linux")(
"captureDesktopApplication acknowledgements on Linux",
() => {
it("acknowledges each checkpoint once it is taken, with its counts, and a refused name", async () => {
const { root, home, environment } = await fixture();
const output = join(root, "out");
const statePath = join(home, ".config/fakeapp/state.ini");
const requests: SessionRequest[] = [
{ kind: "checkpoint", name: "seed" },
{ kind: "checkpoint", name: "seed" },
{ kind: "stop" },
];
const events: unknown[] = [];
const windowOpen = () =>
events.some(
(event) =>
desktopCaptureEventSchema.parse(event).kind === "window_open",
);
const nextRequest = (): SessionRequest | undefined =>
readFileSyncSafe(statePath) === "count=1\n" && windowOpen()
? requests.shift()
: undefined;
const evidence = await captureDesktopApplication(request(output), {
...host(environment, nextRequest),
report: async (_message, event) => {
if (event !== undefined) events.push(event);
},
});
if ("error" in evidence) throw new Error(JSON.stringify(evidence));
const parsed = events.map((event) =>
desktopCaptureEventSchema.parse(event),
);
expect(parsed.map((event) => event.kind)).toEqual([
"checkpoint",
"window_open",
"checkpoint",
"checkpoint_refused",
"checkpoint",
"checkpoint",
]);
const capture = desktopApplicationCaptureSchema.parse(
evidence.normalized_result,
);
const seed = capture.state.checkpoints[1];
// The acknowledgement comes after the checkpoint is taken and carries its counts.
expect(parsed[2]).toEqual({
kind: "checkpoint",
name: "seed",
trigger: "request",
at_ms: seed?.at_ms,
entries: seed?.entries,
changes: seed?.changes.length,
retained_files: 1,
retained_bytes: 8,
});
expect(parsed[3]).toEqual({
kind: "checkpoint_refused",
name: "seed",
reason: "A checkpoint named seed was already recorded.",
});
expect(parsed.slice(4)).toMatchObject([
{ name: "stop", trigger: "stop" },
{ name: "exit", trigger: "exit" },
]);
}, 30_000);
},
);
describe.skipIf(process.platform !== "linux")(
"captureDesktopApplication retained-file cleanup on Linux",
() => {
src/application/desktop/DesktopApplicationCapture.ts
+21 -4
View File
@@ -6,9 +6,11 @@ import {
projectAnalysisError,
type AnalysisErrorProjection,
} from "../../domain/analysisErrorProjection.js";
import type {
DesktopApplicationCapture,
DesktopSession,
import {
checkpointTaken,
type DesktopApplicationCapture,
type DesktopCaptureEvent,
type DesktopSession,
} from "../../domain/desktop/desktopCapture.js";
import { isWithinAny } from "../../domain/desktop/desktopStateDiff.js";
import { createEvidence, type Evidence } from "../../domain/evidence.js";
@@ -74,7 +76,11 @@ export interface DesktopCaptureHost {
/** Directory relative executables and paths are resolved from. */
readonly callerDirectory: string;
readonly nextRequest: () => SessionRequest | undefined;
readonly report: (message: string) => Promise<void>;
/** Progress text, with the event a driver waits on when there is one. */
readonly report: (
message: string,
event?: DesktopCaptureEvent,
) => Promise<void>;
}
/** Safe failure returned to the CLI adapter. */
@@ -255,8 +261,18 @@ const recordCapture = async (
store,
);
const recorder = new CheckpointRecorder(scope, store);
const acknowledge = async () => {
const taken = recorder.checkpoints.at(-1);
if (taken === undefined) return;
const event = checkpointTaken(taken);
await host.report(
`Recorded checkpoint ${taken.name}: ${String(event.changes)} change(s), ${String(event.retained_files)} file(s) retained.`,
event,
);
};
await host.report("Recording the starting state.");
await recorder.record("baseline", "baseline", 0, signal);
await acknowledge();
await host.report(`Launching ${inventory.argv.join(" ")}.`);
const outcome = await runDesktopSession({
argv: inventory.argv,
@@ -272,6 +288,7 @@ const recordCapture = async (
await host.report(`Recording checkpoint ${name}.`);
recorder.openedFiles = openedFiles;
await recorder.record(name, trigger, atMs, signal);
await acknowledge();
},
report: host.report,
signal,
src/application/desktop/DesktopApplicationExport.test.ts
+65 -1
View File
@@ -238,6 +238,70 @@ describe("exportDesktopApplication", () => {
});
});
describe("exportDesktopApplication and the desktop Trash", () => {
it("leaves a file the user moved to the Trash out of the seed and the owned paths, and says so", async () => {
const local = capture();
const trashed = `${HOME}/.local/share/Trash/files/today.txt`;
const info = `${HOME}/.local/share/Trash/info/today.txt.trashinfo`;
const infoText = "[Trash Info]\nPath=/home/ada/today.txt\n";
const [baseline, edited] = local.state.checkpoints;
if (baseline === undefined || edited === undefined)
throw new Error("the fixture has two checkpoints");
const folder = await writeCaptureFolder(
{
...local,
state: {
...local.state,
checkpoints: [
baseline,
{
...edited,
changes: [
...edited.changes,
change(trashed, { after: "milk\n" }),
change(info, { after: infoText }),
],
retained: [
...edited.retained,
retained(trashed, "milk\n"),
retained(info, infoText),
],
},
],
},
},
undefined,
[infoText],
);
const result = await exportDesktopApplication(
{
capture: folder,
directory: join(
await createTestTempDirectory("desktop-export-"),
"bundle",
),
name: undefined,
tasks: [],
seedCheckpoint: "edited",
},
readers,
);
if (result instanceof Error) throw result;
expect(result.normalized_result).toMatchObject({
paths: [".config/notes", ".local/share/notes", "Documents"],
seed_files: [
".config/notes/settings.ini",
".local/share/notes/notes.db",
"Documents/today.txt",
],
});
expect(result.limitations).toContain(
"The desktop Trash held 2 file(s) at the seed checkpoint, which are not seeded: ~/.local/share/Trash/files/today.txt, ~/.local/share/Trash/info/today.txt.trashinfo. Restore one the task needs and capture again.",
);
});
});
const TYPORA_SOURCE = "https://downloads.typora.io/linux ./ Packages";
const TYPORA_KEY = "typora public keyring\n";
const VERIFIED_KEY: AptRepository["signing_key"] = {
@@ -396,7 +460,7 @@ describe("exportDesktopApplication from a vendor repository", () => {
);
expect(module).toContain('"arch": ["amd64"]');
expect(module).toContain(
`command=["sh", "-c", "exec 'typora' '--no-sandbox' \\"$HOME\\"/'Documents/today.txt'"]`,
`command=["sh", "-c", "cd \\"$HOME\\" 2>/dev/null; exec 'typora' '--no-sandbox' \\"$HOME\\"/'Documents/today.txt'"]`,
);
expect(result.limitations.join("\n")).toContain(
"Chromium's sandbox is off inside the computer",
src/application/desktop/DesktopApplicationExport.ts
+53 -99
View File
@@ -1,13 +1,3 @@
import { mkdir } from "node:fs/promises";
import { dirname } from "node:path";
import { SafeOutputTree } from "../../artifacts/SafeOutputTree.js";
import {
ArtifactResourceScope,
type ArtifactCleanupAttempt,
} from "../../artifacts/ArtifactResourceScope.js";
import { outputDirectoryAdmissionRefusal } from "../../artifacts/SafeOutputTreeAdmission.js";
import { SafeOutputTreeCreationFailure } from "../../artifacts/SafeOutputTreeCreationFailure.js";
import { AnalysisError } from "../../domain/analysisErrorBase.js";
import {
DESKTOP_EXPORT_FRAMEWORK_VERSION,
@@ -41,7 +31,6 @@ import type { HostIdentity } from "../../domain/desktop/hostIdentity.js";
import { scrubDconf } from "../../domain/desktop/seedScrub.js";
import { isTextualFormat } from "../../domain/desktop/stateFormats.js";
import { createEvidence, type Evidence } from "../../domain/evidence.js";
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
import type { JsonValue } from "../../domain/jsonValue.js";
import {
inspectDesktopApplicationState,
@@ -56,6 +45,10 @@ import {
admitApplication,
exportEnvironmentName,
} from "./DesktopExportInstall.js";
import {
writeOutputTree,
writtenWithOpenHandles,
} from "./DesktopOutputTree.js";
import {
retainedCopies,
shippedSeedViews,
@@ -128,11 +121,6 @@ const seedDconf = (
};
type StatePath = DesktopApplicationState["paths"][number];
type FailedRelease = Extract<
ArtifactCleanupAttempt,
{ readonly kind: "failed" }
>;
/** Files under the owned paths that existed at the seed without retained bytes, in the state view or omitted. */
const unseededFiles = (
capture: DesktopApplicationCapture,
@@ -159,6 +147,20 @@ const unseededFiles = (
return [...new Set([...listed, ...omitted])].sort();
};
/** Files in the desktop Trash at the seed, which the state view never makes the app's, so the seed leaves them out. */
const trashedFiles = (
files: ReadonlyMap<string, StatePath>,
seed: number,
): readonly string[] =>
[...files]
.filter(
([, path]) =>
path.reasons.includes("desktop_trash") &&
versionAt(path, seed)?.present === true,
)
.map(([home]) => home)
.sort();
/** Decide the bundle from the capture and its state view at the seed, with each seed's digest. */
const planExport = (
request: DesktopExportRequest,
@@ -175,6 +177,7 @@ const planExport = (
readonly limitations: readonly string[];
readonly collisions: readonly HomePathCollision[];
readonly files: ReadonlyMap<string, StatePath>;
readonly trashed: readonly string[];
readonly dconf: { readonly groups: number; readonly holdsIdentity: boolean };
} => {
const { application, host } = capture;
@@ -241,7 +244,15 @@ const planExport = (
application.package,
electron,
);
return { plan, seeds, limitations, collisions, files, dconf: dconfScrub };
return {
plan,
seeds,
limitations,
collisions,
files,
trashed: trashedFiles(files, context.seed),
dconf: dconfScrub,
};
};
const EXPORT_LIMITATIONS = [
@@ -249,58 +260,6 @@ const EXPORT_LIMITATIONS = [
"Binary state files are seeded as captured; only text files, UTF-8 or UTF-16 with a byte-order mark, have the capture host's name, user@host and home path replaced.",
];
/**
* Write the bundle through the safe output tree, rolling an unpublished tree
* back on any failure so no partial copy of the user's data remains. A
* published bundle is kept; a failed release of its handles is returned.
*/
const writeBundle = async (
directory: string,
write: (tree: SafeOutputTree) => Promise<void>,
signal: AbortSignal | undefined,
): Promise<FailedRelease | undefined> => {
await mkdir(dirname(directory), { recursive: true, mode: 0o700 });
const resources = new ArtifactResourceScope();
let tree: SafeOutputTree | undefined;
let outcome:
| { readonly kind: "completed" }
| { readonly kind: "failed"; readonly cause: unknown };
try {
try {
tree = await SafeOutputTree.create(directory, process.platform, {
acceptExistingEmpty: true,
});
} catch (cause: unknown) {
if (cause instanceof SafeOutputTreeCreationFailure) tree = cause.tree;
throw cause;
}
await write(tree);
await tree.commit(signal);
outcome = { kind: "completed" };
} catch (cause: unknown) {
outcome = { kind: "failed", cause };
}
if (tree !== undefined) {
const released = await resources.release({
kind: "output-tree",
resource: directory,
tree,
});
if (released.kind === "failed") {
if (tree.published) return released;
throw resources.cleanupFailure(outcome, [released]);
}
if (tree.published) return undefined;
}
if (outcome.kind === "failed")
throw outputDirectoryAdmissionRefusal(
OPERATION,
"directory",
outcome.cause,
);
return undefined;
};
/** The bundle's generated files, the class module flagged when its dconf keys hold the identity, and its README, made once the seeds' scrub is known. */
const bundleText = (
plan: DesktopAppExportPlan,
@@ -345,6 +304,7 @@ const exportLimitations = (
readonly install: readonly string[];
readonly collisions: readonly HomePathCollision[];
readonly identity: HostIdentity;
readonly trashed: readonly string[];
},
): string[] => {
const opened = capture.state.unretained_opened_directories;
@@ -356,6 +316,11 @@ const exportLimitations = (
: [
`${String(plan.unseededFiles.length)} file(s) under the owned paths existed at the seed but were not retained, so they are not seeded.`,
]),
...(facts.trashed.length === 0
? []
: [
`The desktop Trash held ${String(facts.trashed.length)} file(s) at the seed checkpoint, which are not seeded: ${facts.trashed.map((home) => `~/${home}`).join(", ")}. Restore one the task needs and capture again.`,
]),
...(facts.identity.hostname === null || facts.identity.user === null
? [
"The capture does not record the host's name and user, so only the captured home is removed from the seed; capture again to remove them too.",
@@ -399,12 +364,12 @@ const runExport = async (
);
const { home, hostname, user } = capture.host;
const identity = { home, hostname: hostname ?? null, user: user ?? null };
const { plan, seeds, collisions, files, dconf, ...install } = planExport(
request,
capture,
state,
{ evidenceId: evidence.evidence_id, seed: ends.seed, identity },
);
const { plan, seeds, collisions, files, trashed, dconf, ...install } =
planExport(request, capture, state, {
evidenceId: evidence.evidence_id,
seed: ends.seed,
identity,
});
const copies = retainedCopies(plan, seeds);
// A task's seed checks read the seeds as the bundle ships them, scrubbed.
const seedViews =
@@ -425,12 +390,13 @@ const runExport = async (
const unclosed = await withRetainedContent(
request.capture,
(content) =>
writeBundle(
writeOutputTree(
request.directory,
async (tree) => {
const bundle = { generated, copies, identity, readme };
scrub = await writeFiles(tree, content, bundle, signal);
},
{ operation: OPERATION, field: "directory" },
signal,
),
SEED_MAX_BYTES,
@@ -439,6 +405,7 @@ const runExport = async (
install: install.limitations,
collisions,
identity,
trashed,
});
const result: DesktopExportResult = {
directory: request.directory,
@@ -465,28 +432,6 @@ const runExport = async (
return { result, captureEvidenceId: evidence.evidence_id, unclosed };
};
/** A complete bundle whose output handles could not be closed, reported with its Evidence. */
const writtenWithOpenHandles = (
directory: string,
evidence: Evidence,
release: FailedRelease,
): ProviderAdapterError => {
const reason =
release.cause instanceof Error
? release.cause.message
: String(release.cause);
const error = new ProviderAdapterError(DESKTOP_PROVIDER.id, OPERATION, {
userMessage: `The bundle was written to ${directory} and is complete, but its output handles could not be closed: ${reason}.`,
remediationAction:
"Use the written bundle; the handles are released when this process exits.",
cleanup: { reason, resources: [directory] },
partialObservation: evidence,
diagnostics: { output_directory: directory },
});
release.lease.bindCleanupError(error);
return error;
};
/** Export one capture folder as an agentenv computer app bundle, reported as Evidence. */
export const exportDesktopApplication = async (
request: DesktopExportRequest,
@@ -527,7 +472,16 @@ export const exportDesktopApplication = async (
});
return unclosed === undefined
? evidence
: writtenWithOpenHandles(request.directory, evidence, unclosed);
: writtenWithOpenHandles(
{
provider: DESKTOP_PROVIDER.id,
operation: OPERATION,
what: "bundle",
},
request.directory,
unclosed,
evidence,
);
} catch (cause: unknown) {
return desktopOperationFailure(cause, signal, OPERATION, "Desktop export");
}
src/application/desktop/DesktopBuildActionLog.ts
+17 -3
View File
@@ -65,6 +65,7 @@ const jsonBytes = (entry: BuildComputerAction): number =>
export class BuildComputerActionLog {
readonly #kept: { entry: BuildComputerAction; bytes: number }[] = [];
readonly #lastApplied: Partial<Record<ComputerAction, number>> = {};
readonly #lastUnsettled: Partial<Record<ComputerAction, number>> = {};
#total = 0;
#bytes = 0;
@@ -88,6 +89,11 @@ export class BuildComputerActionLog {
return { ...this.#lastApplied };
}
/** The index of the last action whose reply was lost, kept or not. */
get lastUnsettled(): Readonly<Partial<Record<ComputerAction, number>>> {
return { ...this.#lastUnsettled };
}
/** Log an action as it is sent, pending, and how to record how it ended. */
begin(
action: ComputerAction,
@@ -95,7 +101,10 @@ export class BuildComputerActionLog {
args: Readonly<Record<string, JsonValue>>,
): {
readonly index: number;
readonly settle: (outcome: Exclude<Outcome, "pending">) => void;
readonly settle: (
outcome: Exclude<Outcome, "pending">,
observation?: { readonly unsettled?: boolean },
) => void;
} {
const index = this.#total;
const fields = Object.entries(args).map(
@@ -120,12 +129,17 @@ export class BuildComputerActionLog {
this.#keep(this.#kept.length, entry);
return {
index,
settle: (outcome) => {
settle: (outcome, observation) => {
if (outcome === "applied")
this.#lastApplied[action] = Math.max(
index,
this.#lastApplied[action] ?? index,
);
if (observation?.unsettled === true)
this.#lastUnsettled[action] = Math.max(
index,
this.#lastUnsettled[action] ?? index,
);
// An action dropped while pending leaves only its outcome above.
const slot = index - this.dropped;
if (this.#kept[slot] !== undefined)
@@ -151,5 +165,5 @@ export class BuildComputerActionLog {
/** What later stages read of a computer's action log. */
export type ReadonlyActionLog = Pick<
BuildComputerActionLog,
"total" | "dropped" | "entries" | "lastApplied"
"total" | "dropped" | "entries" | "lastApplied" | "lastUnsettled"
>;
src/application/desktop/DesktopBuildCapture.ts
+362
View File
@@ -0,0 +1,362 @@
import { AnalysisError } from "../../domain/analysisErrorBase.js";
import type { BuildRequest } from "../../domain/desktop/appBuild.js";
import { BUILD_PACKAGE_FILE } from "../../domain/desktop/appBuild.js";
import {
END_CHECKPOINT,
SEED_CHECKPOINT,
} from "../../domain/desktop/buildCapture.js";
import {
captureProgressEvent,
type DesktopCaptureEvent,
} from "../../domain/desktop/desktopCapture.js";
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
import { err, ok, type Result } from "../../domain/result.js";
import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvComputerHelper.js";
import type { ControlledComputer } from "../../desktop/BuildComputerController.js";
import {
adapterError,
controller,
errorText,
refused,
WORK,
type Answer,
} from "./DesktopBuildCommands.js";
import {
exportBuild,
type BuildExportRequest,
type ExportedBuild,
} from "./DesktopBuildExport.js";
import {
summarizeCapture,
type StoppedCapture,
} from "./DesktopBuildSummary.js";
const START = "start_build_capture";
const MARK = "mark_build_checkpoint";
const STOP = "stop_build_capture";
const EXPORT = "export_build_environment";
const NOT_RUNNING = [
"No capture is running on this computer.",
"Start one with start_build_capture.",
] as const;
type CheckpointTaken = Omit<
Extract<DesktopCaptureEvent, { kind: "checkpoint" }>,
"kind"
>;
/** The capture's progress event in a line the controller waited for. */
const eventOf = (line: string): DesktopCaptureEvent | undefined =>
captureProgressEvent(line);
/** Why the capture is not answering, from what the controller saw instead of the event. */
const notAnswered = (operation: string, answer: Answer): AnalysisError =>
adapterError(
operation,
answer.error ??
(answer.exited === undefined
? `The capture did not answer in time. Its last progress: ${answer.progress ?? ""}`
: `The capture ended with exit status ${String(answer.exited)}. ${answer.output ?? ""} ${answer.progress ?? ""}`.trim()),
);
/**
* One capture in the computer. `unreleased` is a capture that may be
* running although its start failed, kept until a stop is confirmed;
* `stopped` keeps the stop's result once it was read, so a retried stop
* resumes or repeats it.
*/
interface CaptureRun {
readonly folder: string;
status: "running" | "unreleased" | "stopped" | "released";
/** Checkpoints the capture acknowledged, then those its own Evidence records. */
checkpoints: Set<string>;
/** The capture's package installs only from its file, so the export takes the request's URL. */
packageFile?: boolean;
outputBytes?: number;
stopped?: StoppedCapture;
exports: number;
}
/** The capture of a failed start may still run; REA still owns it. */
const unreleasedError = (
operation: string,
run: CaptureRun,
cause: AnalysisError | undefined,
) =>
new ProviderAdapterError(AGENTENV_PROVIDER, operation, {
userMessage: `The capture REA began in ~/${run.folder} could not be confirmed stopped${cause === undefined ? "" : ` after: ${cause.message}`}.`,
remediationAction:
"Call start_build_capture again, which retries its stop first, or close the computer.",
cleanup: {
reason: "its stop was not confirmed",
resources: [`~/${run.folder}`],
},
...(cause === undefined ? {} : { cause }),
});
/**
* REA's capture of a build computer's app, run inside the computer by REA's
* controller: start it, take named checkpoints with their acknowledgements,
* stop it and draft the task between `seed` and `end`, then export the
* bundle with the agent's task and bring it back to this host. Every step
* can be asked again after a lost answer.
*/
export class DesktopBuildCapture {
#count = 0;
#run: CaptureRun | undefined;
readonly #entry: string | undefined;
readonly #packageFile: { readonly url: string } | undefined;
constructor(
private readonly computer: ControlledComputer,
request: BuildRequest | undefined,
) {
this.#entry = request?.desktop_entry;
this.#packageFile =
request?.install.kind === "package_file"
? { url: request.install.url }
: undefined;
}
async start(
input: { readonly arguments?: readonly string[] },
signal: AbortSignal,
): Promise<
Result<
{
readonly desktop_entry: string;
readonly package_lists: number;
readonly package_lists_error: string | null;
},
AnalysisError
>
> {
const entry = this.#entry;
if (entry === undefined)
return err(
refused(
START,
"This computer was started from a bundle folder; REA captures the app of a computer started from a request, which installs REA inside it.",
"Start a computer from a request with start_build_computer and capture its app there.",
),
);
const previous = this.#run;
if (previous?.status === "running")
return err(
refused(
START,
"A capture is already running on this computer.",
"Stop it with stop_build_capture before starting another.",
),
);
if (
previous?.status === "unreleased" &&
!(await this.#release(previous, signal))
)
return err(unreleasedError(START, previous, undefined));
this.#count += 1;
const run: CaptureRun = {
folder: `${WORK}/captures/${String(this.#count)}`,
status: "running",
checkpoints: new Set(),
exports: 0,
};
this.#run = run;
const started = await this.#launch(run, entry, input, signal);
if (started.ok) return started;
// An answer naming no live capture means none runs; anything else may have left one running.
const released =
started.error.answer !== undefined &&
started.error.answer.timed_out !== true;
if (released) run.status = "released";
else if (!(await this.#release(run, signal)))
return err(unreleasedError(START, run, started.error.failure));
return err(started.error.failure);
}
async #launch(
run: CaptureRun,
entry: string,
input: { readonly arguments?: readonly string[] },
signal: AbortSignal,
): Promise<
Result<
{
readonly desktop_entry: string;
readonly package_lists: number;
readonly package_lists_error: string | null;
},
{ readonly failure: AnalysisError; readonly answer?: Answer }
>
> {
let answer: Answer;
try {
answer = await controller(
this.computer,
"start",
{
work: WORK,
capture: run.folder,
desktop_entry: entry,
arguments: [...(input.arguments ?? [])],
...(this.#packageFile === undefined
? {}
: { package_file: BUILD_PACKAGE_FILE }),
timeout_s: 120,
apt_timeout_s: 600,
},
{ timeoutS: 720, signal },
);
} catch (cause: unknown) {
return err({ failure: adapterError(START, errorText(cause), cause) });
}
const event = answer.line === undefined ? undefined : eventOf(answer.line);
if (event?.kind !== "window_open")
return err({ failure: notAnswered(START, answer), answer });
return ok({
desktop_entry: entry,
package_lists: answer.apt?.lists ?? 0,
package_lists_error: answer.apt?.error ?? null,
});
}
/** Stop a capture whose start failed; true once the computer confirms it ended or never launched. */
async #release(run: CaptureRun, signal: AbortSignal): Promise<boolean> {
try {
const answer = await controller(
this.computer,
"stop",
{ capture: run.folder, timeout_s: 120 },
{ timeoutS: 120, signal },
);
const confirmed =
answer.absent === true || typeof answer.exit_code === "number";
run.status = confirmed ? "released" : "unreleased";
return confirmed;
} catch {
run.status = "unreleased";
return false;
}
}
async mark(
name: string,
signal: AbortSignal,
): Promise<Result<CheckpointTaken, AnalysisError>> {
const run = this.#run;
if (run?.status !== "running") return err(refused(MARK, ...NOT_RUNNING));
if (run.checkpoints.has(name))
return err(
refused(
MARK,
`A checkpoint named ${name} was already taken.`,
"Each name is taken once per capture; use another name, or capture again to retake it.",
),
);
try {
// A checkpoint the capture already took, whose answer was lost, comes back from its log.
const answer = await controller(
this.computer,
"mark",
{ capture: run.folder, name, timeout_s: 600 },
{ timeoutS: 600, signal },
);
const event =
answer.line === undefined ? undefined : eventOf(answer.line);
if (event?.kind === "checkpoint_refused")
return err(refused(MARK, event.reason, "Use another checkpoint name."));
if (event?.kind !== "checkpoint") return err(notAnswered(MARK, answer));
run.checkpoints.add(name);
const { kind: _kind, ...taken } = event;
return ok(taken);
} catch (cause: unknown) {
return err(adapterError(MARK, errorText(cause), cause));
}
}
/** Stop the capture, read what it recorded and draft the task; asked again, it resumes or repeats the result. */
async stop(
signal: AbortSignal,
): Promise<Result<StoppedCapture, AnalysisError>> {
const run = this.#run;
if (run?.status === "unreleased")
return (await this.#release(run, signal))
? err(
refused(
STOP,
`The capture of the failed start in ~/${run.folder} is now stopped; no capture is running.`,
"Start one with start_build_capture.",
),
)
: err(unreleasedError(STOP, run, undefined));
if (run === undefined || run.status === "released")
return err(refused(STOP, ...NOT_RUNNING));
if (run.stopped !== undefined) return ok(run.stopped);
try {
if (run.outputBytes === undefined) {
const answer = await controller(
this.computer,
"stop",
{ capture: run.folder, timeout_s: 600 },
{ timeoutS: 600, signal },
);
if (answer.exit_code === undefined || answer.output_bytes === undefined)
return err(notAnswered(STOP, answer));
run.status = "stopped";
run.outputBytes = answer.output_bytes;
}
const summary = await summarizeCapture(
this.computer,
{ folder: run.folder, outputBytes: run.outputBytes },
signal,
);
if (!summary.ok) {
run.status = "released";
return summary;
}
run.checkpoints = summary.value.checkpoints;
run.packageFile = summary.value.packageFile;
run.stopped = summary.value.stopped;
return ok(summary.value.stopped);
} catch (cause: unknown) {
return err(
cause instanceof AnalysisError
? cause
: adapterError(STOP, errorText(cause), cause),
);
}
}
/** Export the bundle with the agent's task inside the computer and write it to a folder on this host. */
async export(
input: BuildExportRequest,
signal: AbortSignal,
): Promise<Result<ExportedBuild, AnalysisError>> {
const run = this.#run;
if (
run?.stopped === undefined ||
!run.checkpoints.has(SEED_CHECKPOINT) ||
!run.checkpoints.has(END_CHECKPOINT)
)
return err(
refused(
EXPORT,
`Export needs a stopped capture with ${SEED_CHECKPOINT} and ${END_CHECKPOINT} checkpoints; capture with start_build_capture, mark both and stop it.`,
"Capture with start_build_capture, mark seed and end, then call stop_build_capture.",
),
);
run.exports += 1;
return exportBuild(
this.computer,
{
folder: run.folder,
attempt: run.exports,
...(this.#packageFile === undefined || run.packageFile !== true
? {}
: { packageUrl: this.#packageFile.url }),
},
input,
signal,
);
}
}
src/application/desktop/DesktopBuildCommands.ts
+96
View File
@@ -0,0 +1,96 @@
import { z } from "zod";
import { AnalysisInputError } from "../../domain/analysisErrorCore.js";
import {
END_CHECKPOINT,
SEED_CHECKPOINT,
} from "../../domain/desktop/buildCapture.js";
import { parseEvidence } from "../../domain/evidence.js";
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvComputerHelper.js";
import {
runRea as runControllerRea,
type ControlledComputer,
} from "../../desktop/BuildComputerController.js";
export {
controller,
type Answer,
} from "../../desktop/BuildComputerController.js";
/** The work folder REA keeps under the computer user's home; every capture leaves it out. */
export const WORK = ".rea-build";
/** The checkpoint options the draft and the export read the seed and the end with. */
export const ENDS = [
"--seed-checkpoint",
SEED_CHECKPOINT,
"--end-checkpoint",
END_CHECKPOINT,
] as const;
const cliFailureSchema = z.looseObject({
code: z.string(),
message: z.string(),
remediation: z.looseObject({ action: z.string() }).optional(),
});
/** What REA reads from a command it ran in the computer: its Evidence result, or the failure it reported. */
export type CommandOutput =
| { readonly kind: "result"; readonly result: unknown }
| {
readonly kind: "failure";
readonly failure: z.infer<typeof cliFailureSchema>;
};
export const errorText = (cause: unknown): string =>
cause instanceof Error ? cause.message : String(cause);
export const adapterError = (
operation: string,
userMessage: string,
cause?: unknown,
): ProviderAdapterError =>
new ProviderAdapterError(AGENTENV_PROVIDER, operation, {
userMessage,
...(cause === undefined ? {} : { cause }),
});
/** A valid request the build's current state cannot take, with what to do instead. */
export const refused = (
operation: string,
userMessage: string,
remediationAction: string,
): AnalysisInputError =>
new AnalysisInputError(operation, {
userMessage,
remediationAction,
conformsToInputContract: true,
});
/** Run one `rea` command in the computer; the controller owns the reply's fields. */
export const runRea = (
computer: ControlledComputer,
argv: readonly string[],
options: {
readonly stdout: string;
readonly files?: Readonly<Record<string, string>>;
readonly signal: AbortSignal;
},
): Promise<number> =>
runControllerRea(computer, argv, { ...options, work: WORK });
/** The Evidence result, or the reported failure, a REA command wrote to a file of `bytes` in the computer. */
export const readOutput = async (
computer: ControlledComputer,
path: string,
bytes: number,
signal: AbortSignal,
): Promise<CommandOutput> => {
const content = (await computer.homeFiles([path], bytes, signal)).get(path);
if (content === undefined) throw new Error(`The computer has no ${path}.`);
const value: unknown = JSON.parse(content.toString("utf8"));
const failure = cliFailureSchema.safeParse(value);
return failure.success
? { kind: "failure", failure: failure.data }
: { kind: "result", result: parseEvidence(value).normalized_result };
};
src/application/desktop/DesktopBuildComputers.ts
+212 -43
View File
@@ -21,6 +21,7 @@ import { ProviderCleanupError } from "../../domain/providerCleanupError.js";
import { err, ok, type Result } from "../../domain/result.js";
import {
AgentEnvComputer,
AppDataRefusal,
ComputerStartFailure,
type DriverResult,
} from "../../desktop/AgentEnvComputer.js";
@@ -30,11 +31,15 @@ import {
type ComputerRelease,
} from "../../desktop/AgentEnvComputerHelper.js";
import type { PrivateRuntimeRoot } from "../../process/PrivateRuntimeRoot.js";
import type { ProgressReporter } from "../ProgressReporter.js";
import {
BuildComputerActionLog,
type ReadonlyActionLog,
} from "./DesktopBuildActionLog.js";
import { writeRequestBundle } from "./DesktopBuildBundle.js";
import { DesktopBuildCapture } from "./DesktopBuildCapture.js";
import { DesktopBuildTurns } from "./DesktopBuildTurns.js";
import { DesktopBuildWitness } from "./DesktopBuildWitness.js";
const START = "start_build_computer";
const ACT = "act_on_build_computer";
@@ -46,6 +51,14 @@ export interface ForwardedResult<Value> {
readonly content: DriverResult["content"];
}
/** What an action answers: its place in the log and the driver's structured result. */
type Acted = ForwardedResult<{
readonly computer_id: string;
readonly action: ComputerAction;
readonly action_index: number;
readonly result: Readonly<Record<string, JsonValue>> | null;
}>;
/** What later stages read of a live build computer: its action log and recording hooks. */
export interface BuildComputerSession {
readonly sandboxId: string;
@@ -62,9 +75,21 @@ interface HeldComputer {
readonly release: (() => Promise<ComputerRelease>) | undefined;
readonly workspace: PrivateRuntimeRoot | undefined;
readonly actions: BuildComputerActionLog;
/** REA's capture of the app and the task's witness, on a live computer. */
readonly steps: BuildComputerSteps | undefined;
/** Every action, step and close on the computer takes its turn here. */
readonly turns: DesktopBuildTurns;
closing: boolean;
}
/** What the build step tools work on: a live computer, its capture, its witness and the turns they take. */
export interface BuildComputerSteps {
readonly computer: AgentEnvComputer;
readonly capture: DesktopBuildCapture;
readonly witness: DesktopBuildWitness;
readonly turns: DesktopBuildTurns;
}
/** A start still in progress, which shutdown cancels and joins. */
interface PendingStart {
readonly settled: Promise<unknown>;
@@ -92,6 +117,110 @@ const failure = (
});
};
/** The text of a driver's answer, such as a refusal's message. */
const driverMessage = (content: DriverResult["content"]): string =>
content
.flatMap((block) => (typeof block.text === "string" ? [block.text] : []))
.join(" ");
/**
* What a reset did. `data/reset` completing is the observation that the seed
* was restored; readiness and a screenshot after it can still fail or be
* cancelled without erasing that observation.
*/
type ResetCall =
| { readonly kind: "unfinished"; readonly cause: unknown }
| { readonly kind: "interrupted"; readonly cause: unknown }
| { readonly kind: "done"; readonly result: DriverResult };
/**
* Restore the app's data to its seed with `data/reset`, which relaunches the
* app, and wait until the computer reports ready again; a screenshot after,
* when asked.
*/
const reset = async (
computer: AgentEnvComputer,
fields: Readonly<Record<string, JsonValue>>,
signal: AbortSignal,
): Promise<ResetCall> => {
const startedAt = Date.now();
try {
await computer.appData("data/reset", signal);
} catch (cause: unknown) {
return { kind: "unfinished", cause };
}
try {
await computer.waitUntilReady(signal);
const ready = { reset: true, ready_ms: Date.now() - startedAt };
if (fields.screenshot !== true)
return {
kind: "done",
result: { isError: false, content: [], structured: ready },
};
// The reset has happened: a screenshot that fails after it is reported beside it, and only an abort propagates.
const shot = await computer
.callTool("computer_screenshot", {}, signal)
.catch((cause: unknown) => {
if (signal.aborted) throw cause;
return cause instanceof Error ? cause.message : String(cause);
});
if (typeof shot === "string" || shot.isError)
return {
kind: "done",
result: {
isError: false,
content: [],
structured: {
...ready,
screenshot_error:
typeof shot === "string" ? shot : driverMessage(shot.content),
},
},
};
return {
kind: "done",
result: {
isError: false,
content: shot.content,
structured: { ...ready, frame: shot.structured },
},
};
} catch (cause: unknown) {
return { kind: "interrupted", cause };
}
};
/** A live computer as REA holds it: its action log, its capture and witness, and the turns all its calls take. */
const liveComputer = (
computer: AgentEnvComputer,
workspace: PrivateRuntimeRoot | undefined,
source: BuildSource,
): HeldComputer => {
const actions = new BuildComputerActionLog();
const turns = new DesktopBuildTurns();
return {
computer,
release: () => computer.close(),
workspace,
actions,
steps: {
computer,
capture: new DesktopBuildCapture(
computer,
source.kind === "request" ? source : undefined,
),
witness: new DesktopBuildWitness(
computer,
actions,
source.kind === "bundle" ? source.path : undefined,
),
turns,
},
turns,
closing: false,
};
};
/**
* Why a start failed, with the helper's output, and, when its teardown is
* unconfirmed, the id REA keeps that teardown under for close to retry.
@@ -153,6 +282,7 @@ export class DesktopBuildComputers {
readonly screenshot?: ScreenshotOptions | undefined;
},
signal: AbortSignal,
progress?: ProgressReporter,
): Promise<
Result<
ForwardedResult<{
@@ -171,7 +301,11 @@ export class DesktopBuildComputers {
}),
);
const stop = new AbortController();
const settled = this.#start(input, AbortSignal.any([signal, stop.signal]));
const settled = this.#start(
input,
AbortSignal.any([signal, stop.signal]),
progress,
);
const pending = { settled, stop };
this.#starts.add(pending);
try {
@@ -184,6 +318,7 @@ export class DesktopBuildComputers {
async #start(
input: Parameters<DesktopBuildComputers["start"]>[0],
signal: AbortSignal,
progress: ProgressReporter | undefined,
): ReturnType<DesktopBuildComputers["start"]> {
const { source } = input;
let target: {
@@ -209,26 +344,34 @@ export class DesktopBuildComputers {
signal,
operation: START,
});
computer = await AgentEnvComputer.connect(helper, signal);
const connected = await AgentEnvComputer.connect(
helper,
signal,
progress === undefined
? undefined
: (message) => {
void progress
.report({ phase: START, completed: 0, total: null, message })
.catch(() => undefined);
},
);
computer = connected.computer;
const shot = driverCall("screenshot", input.screenshot ?? {});
if (!shot.ok) throw new TypeError("screenshot options are its own");
const taken = await computer.callTool(
shot.value.tool,
shot.value.arguments,
signal,
);
const live = computer;
this.#computers.set(id, {
computer: live,
release: () => live.close(),
workspace: target.workspace,
actions: new BuildComputerActionLog(),
closing: false,
});
// The caller's region, format or quality apply to the drawn frame.
const taken =
Object.keys(shot.value.arguments).length === 0
? connected.frame
: await computer.callTool(
shot.value.tool,
shot.value.arguments,
signal,
);
this.#computers.set(id, liveComputer(computer, target.workspace, source));
return ok({
value: {
computer_id: id,
sandbox_id: live.sandboxId,
sandbox_id: computer.sandboxId,
environment: target.name,
screenshot: taken.isError ? null : taken.structured,
},
@@ -247,6 +390,8 @@ export class DesktopBuildComputers {
: () => owner.release(),
workspace: target.workspace,
actions: new BuildComputerActionLog(),
steps: undefined,
turns: new DesktopBuildTurns(),
closing: true,
});
const release = await this.#retire(id);
@@ -258,17 +403,7 @@ export class DesktopBuildComputers {
async act(
input: ComputerActionInput,
signal: AbortSignal,
): Promise<
Result<
ForwardedResult<{
readonly computer_id: string;
readonly action: ComputerAction;
readonly action_index: number;
readonly result: Readonly<Record<string, JsonValue>> | null;
}>,
AnalysisError
>
> {
): Promise<Result<Acted, AnalysisError>> {
const { computer_id: id, action, ...fields } = input;
const held = this.#computers.get(id);
const computer = held?.computer;
@@ -291,18 +426,47 @@ export class DesktopBuildComputers {
],
),
);
const logged = held.actions.begin(
action,
call.value.tool,
call.value.arguments,
return held.turns.run(ACT, signal, (turn) =>
this.#forward(held, computer, { id, action, ...call.value }, turn),
);
}
/** Log and send one action on its turn, so the log holds no action still pending while another call runs. */
async #forward(
held: HeldComputer,
computer: AgentEnvComputer,
call: {
readonly id: string;
readonly action: ComputerAction;
readonly tool: string;
readonly arguments: Readonly<Record<string, JsonValue>>;
},
signal: AbortSignal,
): Promise<Result<Acted, AnalysisError>> {
const { id, action } = call;
const logged = held.actions.begin(action, call.tool, call.arguments);
let result: DriverResult;
try {
result = await computer.callTool(
call.value.tool,
call.value.arguments,
signal,
);
if (action === "reset") {
const outcome = await reset(computer, call.arguments, signal);
if (outcome.kind === "unfinished") {
// A tagged app refusal is a failed reset. Any other lost reply,
// including a dropped connection after the seed was restored, stays
// unknown and blocks a near miss that would treat it as absent.
logged.settle("failed", {
unsettled: !(outcome.cause instanceof AppDataRefusal),
});
return err(failure(ACT, outcome.cause, signal));
}
// The seed restore is already observed. A later cancel or readiness
// failure must not make the witness treat that restore as absent.
logged.settle("applied");
if (outcome.kind === "interrupted")
return err(failure(ACT, outcome.cause, signal));
result = outcome.result;
} else {
result = await computer.callTool(call.tool, call.arguments, signal);
}
} catch (cause: unknown) {
logged.settle("failed");
return err(failure(ACT, cause, signal));
@@ -311,11 +475,7 @@ export class DesktopBuildComputers {
if (result.isError)
return err(
new AnalysisInputError(ACT, {
userMessage: `The computer refused the ${action}: ${result.content
.flatMap((block) =>
typeof block.text === "string" ? [block.text] : [],
)
.join(" ")}`,
userMessage: `The computer refused the ${action}: ${driverMessage(result.content)}`,
remediationAction:
"Follow the computer's message; actions on frame coordinates need a screenshot first.",
conformsToInputContract: true,
@@ -342,8 +502,11 @@ export class DesktopBuildComputers {
AnalysisError
>
> {
if (!this.#computers.has(id)) return err(unknownComputer(CLOSE));
const release = await this.#retire(id);
const held = this.#computers.get(id);
if (held === undefined) return err(unknownComputer(CLOSE));
held.closing = true;
// Tear down only once the call it cancels has ended; the calls waiting are refused.
const release = await held.turns.close(() => this.#retire(id));
return release.verified
? ok({ computer_id: id, teardown: release.teardown })
: err(
@@ -399,6 +562,12 @@ export class DesktopBuildComputers {
);
}
/** A live computer's capture and witness, for the build step tools; undefined for an unknown or closing id. */
steps(id: string): BuildComputerSteps | undefined {
const held = this.#computers.get(id);
return held === undefined || held.closing ? undefined : held.steps;
}
/** A live computer's action log and recording hooks, for the witness. */
session(id: string): BuildComputerSession | undefined {
const held = this.#computers.get(id);
src/application/desktop/DesktopBuildExport.ts
+199
View File
@@ -0,0 +1,199 @@
import { Readable } from "node:stream";
import { z } from "zod";
import { AnalysisError } from "../../domain/analysisErrorBase.js";
import type { desktopTaskSchema } from "../../domain/desktop/appTask.js";
import { jsonValueSchema, type JsonValue } from "../../domain/jsonValue.js";
import { err, ok, type Result } from "../../domain/result.js";
import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvComputerHelper.js";
import {
fetchHomeFolder,
type ControlledComputer,
} from "../../desktop/BuildComputerController.js";
import {
adapterError,
ENDS,
errorText,
readOutput,
refused,
runRea,
} from "./DesktopBuildCommands.js";
import {
writeOutputTree,
writtenWithOpenHandles,
} from "./DesktopOutputTree.js";
const EXPORT = "export_build_environment";
/** The most bytes the task takes in the controller's one JSON argument, which Linux caps at 128 KiB. */
const TASK_ARGUMENT_BYTES = 96 * 1024;
const exportResultSchema = z.looseObject({
environment: z.string(),
seed_files: z.array(z.string()),
scrubbed_files: z.array(z.string()),
unscrubbed_files: z.array(z.string()),
limitations: z.array(z.string()),
task: jsonValueSchema,
});
/** What the agent asks the export for: its task, the folder here and, optionally, the environment's name. */
export interface BuildExportRequest {
readonly task: z.output<typeof desktopTaskSchema>;
readonly directory: string;
readonly name?: string | undefined;
}
/** An exported bundle brought back to this host. */
export type ExportedBuild = {
readonly directory: string;
readonly environment: string;
readonly files: number;
readonly bytes: number;
readonly task: JsonValue;
readonly seed_files: string[];
readonly scrubbed_files: string[];
readonly unscrubbed_files: string[];
readonly limitations: string[];
};
/**
* Bring the bundle back straight into the caller's new or empty folder, each
* checked answer written before the next is fetched; a failure leaves no
* partial folder.
*/
const writeBundle = async (
computer: ControlledComputer,
bundle: string,
directory: string,
signal: AbortSignal,
): Promise<
Result<{ readonly files: number; readonly bytes: number }, AnalysisError>
> => {
let fetched = { files: 0, bytes: 0 };
const unclosed = await writeOutputTree(
directory,
async (tree) => {
fetched = await fetchHomeFolder(
computer,
bundle,
{
file: async (file) => {
await tree.write(
file.path,
Readable.from([file.bytes]),
{ bytes: file.bytes.length, sha256: file.sha256 },
signal,
);
},
},
signal,
);
},
{ operation: EXPORT, field: "directory" },
signal,
);
return unclosed === undefined
? ok(fetched)
: err(
writtenWithOpenHandles(
{ provider: AGENTENV_PROVIDER, operation: EXPORT, what: "bundle" },
directory,
unclosed,
),
);
};
/**
* Export a stopped capture in the computer with the agent's task, seeded at
* `seed` and checked at `end`, into a new folder of the work folder, then
* bring the bundle back and write it into the caller's folder. A task the
* export refuses is refused with the export's own reason.
*/
export const exportBuild = async (
computer: ControlledComputer,
capture: {
readonly folder: string;
readonly attempt: number;
/** The URL a computer downloads the app's `.deb` from, when the capture says apt does not install it. */
readonly packageUrl?: string;
},
request: BuildExportRequest,
signal: AbortSignal,
): Promise<Result<ExportedBuild, AnalysisError>> => {
const { folder } = capture;
const attempt = String(capture.attempt);
const bundle = `${folder}/bundle-${attempt}`;
const task = `${folder}/task-${attempt}.json`;
const stdout = `${folder}/export-${attempt}.json`;
const taskFile = `${JSON.stringify(request.task, null, 2)}\n`;
const taskBytes = Buffer.byteLength(JSON.stringify(taskFile));
if (taskBytes > TASK_ARGUMENT_BYTES)
return err(
refused(
EXPORT,
`The task takes ${String(taskBytes)} bytes in the command REA runs in the computer, more than the ${String(TASK_ARGUMENT_BYTES)} it allows a task.`,
"Shorten the task's prompt or checks and export again.",
),
);
try {
const bytes = await runRea(
computer,
[
"export-desktop-application",
`${folder}/capture`,
bundle,
"--task",
task,
...ENDS,
...(request.name === undefined ? [] : ["--name", request.name]),
...(capture.packageUrl === undefined
? []
: ["--package-url", capture.packageUrl]),
"--json",
],
{
stdout,
files: { [task]: taskFile },
signal,
},
);
const output = await readOutput(computer, stdout, bytes, signal);
if (output.kind === "failure")
return err(
output.failure.code === "invalid_request"
? refused(
EXPORT,
output.failure.message,
output.failure.remediation?.action ??
"Rewrite the task and export again.",
)
: adapterError(EXPORT, output.failure.message),
);
const result = exportResultSchema.parse(output.result);
const written = await writeBundle(
computer,
bundle,
request.directory,
signal,
);
if (!written.ok) return written;
return ok({
directory: request.directory,
environment: result.environment,
files: written.value.files,
bytes: written.value.bytes,
task: result.task,
seed_files: result.seed_files,
scrubbed_files: result.scrubbed_files,
unscrubbed_files: result.unscrubbed_files,
limitations: result.limitations,
});
} catch (cause: unknown) {
return err(
cause instanceof AnalysisError
? cause
: adapterError(EXPORT, errorText(cause), cause),
);
}
};
src/application/desktop/DesktopBuildSteps.ts
+158
View File
@@ -0,0 +1,158 @@
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
import {
AnalysisCancelledError,
AnalysisInputError,
} from "../../domain/analysisErrorCore.js";
import type { JsonValue } from "../../domain/jsonValue.js";
import { err, ok, type Result } from "../../domain/result.js";
import type { StoppedCapture } from "./DesktopBuildSummary.js";
import type {
BuildComputerSteps,
DesktopBuildComputers,
ForwardedResult,
} from "./DesktopBuildComputers.js";
import type { ExportedBuild } from "./DesktopBuildExport.js";
import type {
CheckedTask,
DesktopBuildWitness,
} from "./DesktopBuildWitness.js";
const unknownComputer = (operation: string) =>
new AnalysisInputError(operation, {
userMessage:
"No build computer has this id in this REA server; start one with start_build_computer.",
remediationAction: "Start one with start_build_computer.",
conformsToInputContract: true,
});
type Identified<Value> = { readonly computer_id: string } & Value;
/**
* The build steps on a held computer: REA's capture of its app, the draft,
* the export and the task's witness, each named by the computer's id.
*/
export class DesktopBuildSteps {
constructor(
private readonly computers: Pick<DesktopBuildComputers, "steps">,
) {}
/** Start REA's capture of the computer's app, inside the computer, and show the opened app. */
async startCapture(
input: {
readonly computer_id: string;
readonly arguments?: readonly string[] | undefined;
},
signal: AbortSignal,
): Promise<
Result<
ForwardedResult<
Identified<{
readonly desktop_entry: string;
readonly package_lists: number;
readonly package_lists_error: string | null;
readonly screenshot: Readonly<Record<string, JsonValue>> | null;
}>
>,
AnalysisError
>
> {
const steps = this.computers.steps(input.computer_id);
if (steps === undefined) return err(unknownComputer("start_build_capture"));
// The screenshot of the opened app takes the same turn as the start.
return steps.turns.run("start_build_capture", signal, async (turn) => {
const started = await steps.capture.start(
input.arguments === undefined ? {} : { arguments: input.arguments },
turn,
);
if (!started.ok) return started;
// A failed screenshot only leaves it out, but a close or abort meanwhile cancels the start: it saved nothing to report.
const shot = await steps.computer
.callTool("computer_screenshot", {}, turn)
.catch(() => undefined);
if (turn.aborted)
return err(
new AnalysisCancelledError("start_build_capture", {
cause: turn.reason,
}),
);
return ok({
value: {
computer_id: input.computer_id,
...started.value,
screenshot:
shot === undefined || shot.isError ? null : shot.structured,
},
content: shot === undefined || shot.isError ? [] : shot.content,
});
});
}
markCheckpoint(
input: { readonly computer_id: string; readonly name: string },
signal: AbortSignal,
) {
return this.#on(
input.computer_id,
"mark_build_checkpoint",
signal,
async (steps, turn) => {
const taken = await steps.capture.mark(input.name, turn);
return taken.ok ? ok({ checkpoint: taken.value }) : taken;
},
);
}
stopCapture(
input: { readonly computer_id: string },
signal: AbortSignal,
): Promise<Result<Identified<StoppedCapture>, AnalysisError>> {
return this.#on(
input.computer_id,
"stop_build_capture",
signal,
(steps, turn) => steps.capture.stop(turn),
);
}
exportEnvironment(
input: Parameters<BuildComputerSteps["capture"]["export"]>[0] & {
readonly computer_id: string;
},
signal: AbortSignal,
): Promise<Result<Identified<ExportedBuild>, AnalysisError>> {
const { computer_id: id, ...request } = input;
return this.#on(id, "export_build_environment", signal, (steps, turn) =>
steps.capture.export(request, turn),
);
}
checkTask(
input: Parameters<DesktopBuildWitness["check"]>[0] & {
readonly computer_id: string;
},
signal: AbortSignal,
): Promise<Result<Identified<CheckedTask>, AnalysisError>> {
const { computer_id: id, ...request } = input;
return this.#on(id, "check_build_task", signal, (steps, turn) =>
steps.witness.check(request, turn),
);
}
/** Run one step on a held computer, on its turn among the computer's calls, and name the computer in its result. */
async #on<Value extends object>(
id: string,
operation: string,
signal: AbortSignal,
run: (
steps: BuildComputerSteps,
signal: AbortSignal,
) => Promise<Result<Value, AnalysisError>>,
): Promise<Result<Identified<Value>, AnalysisError>> {
const steps = this.computers.steps(id);
if (steps === undefined) return err(unknownComputer(operation));
const result = await steps.turns.run(operation, signal, (turn) =>
run(steps, turn),
);
return result.ok ? ok({ computer_id: id, ...result.value }) : result;
}
}
src/application/desktop/DesktopBuildSummary.ts
+179
View File
@@ -0,0 +1,179 @@
import { z } from "zod";
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
import {
END_CHECKPOINT,
SEED_CHECKPOINT,
} from "../../domain/desktop/buildCapture.js";
import { installsFromPackageFile } from "../../domain/desktop/appExportInstall.js";
import { desktopApplicationCaptureSchema } from "../../domain/desktop/desktopCapture.js";
import {
jsonObjectSchema,
jsonValueSchema,
type JsonValue,
} from "../../domain/jsonValue.js";
import { err, ok, type Result } from "../../domain/result.js";
import type { ControlledComputer } from "../../desktop/BuildComputerController.js";
import {
adapterError,
ENDS,
readOutput,
runRea,
} from "./DesktopBuildCommands.js";
const STOP = "stop_build_capture";
const draftResultSchema = z.looseObject({
files: z.array(
z.looseObject({
file: z.string(),
role: z.string(),
change: jsonValueSchema,
}),
),
task: z.looseObject({
name: z.string(),
prompt: z.string(),
checks: z.array(jsonObjectSchema),
}),
});
/** A stopped capture: its checkpoints, what it found of the app and the draft between seed and end. */
export type StoppedCapture = {
readonly end_reason: string;
readonly checkpoints: {
readonly name: string;
readonly trigger: string;
readonly changes: number;
readonly retained_files: number;
}[];
readonly package: {
readonly kind: string;
readonly name: string | null;
readonly version: string | null;
readonly sources: string[];
};
readonly limitations: string[];
readonly draft: {
readonly seed_checkpoint: string;
readonly end_checkpoint: string;
readonly files: {
readonly file: string;
readonly role: string;
readonly change: JsonValue;
}[];
readonly task: {
readonly name: string;
readonly prompt: string;
readonly checks: Readonly<Record<string, JsonValue>>[];
};
} | null;
readonly draft_refusal?: string;
};
/** The task draft between `seed` and `end`, or why there is none. */
const draftTask = async (
computer: ControlledComputer,
folder: string,
checkpoints: ReadonlySet<string>,
signal: AbortSignal,
): Promise<Pick<StoppedCapture, "draft" | "draft_refusal">> => {
const missing = [SEED_CHECKPOINT, END_CHECKPOINT].filter(
(name) => !checkpoints.has(name),
);
if (missing.length > 0)
return {
draft: null,
draft_refusal: `No draft: the capture has no ${missing.join(" or ")} checkpoint; capture again and mark ${SEED_CHECKPOINT} once the seed is saved and ${END_CHECKPOINT} once the edit is.`,
};
const stdout = `${folder}/draft.json`;
const bytes = await runRea(
computer,
[
"describe-desktop-application-task",
`${folder}/capture`,
...ENDS,
"--json",
],
{ stdout, signal },
);
const output = await readOutput(computer, stdout, bytes, signal);
if (output.kind === "failure")
return { draft: null, draft_refusal: output.failure.message };
const draft = draftResultSchema.parse(output.result);
return {
draft: {
seed_checkpoint: SEED_CHECKPOINT,
end_checkpoint: END_CHECKPOINT,
files: draft.files.map(({ file, role, change }) => ({
file,
role,
change,
})),
task: {
name: draft.task.name,
prompt: draft.task.prompt,
checks: draft.task.checks,
},
},
};
};
/**
* What a capture that has exited recorded, read from its own Evidence: the
* checkpoints it holds, which the draft and the export trust over the
* acknowledgements REA happened to receive, whether the export installs the
* app from its package file, and the stop's summary with the draft between
* `seed` and `end`. A capture that reported a failure is one.
*/
export const summarizeCapture = async (
computer: ControlledComputer,
capture: { readonly folder: string; readonly outputBytes: number },
signal: AbortSignal,
): Promise<
Result<
{
readonly checkpoints: Set<string>;
readonly packageFile: boolean;
readonly stopped: StoppedCapture;
},
AnalysisError
>
> => {
const output = await readOutput(
computer,
`${capture.folder}/capture.out`,
capture.outputBytes,
signal,
);
if (output.kind === "failure")
return err(
adapterError(STOP, `The capture failed: ${output.failure.message}`),
);
const recorded = desktopApplicationCaptureSchema.parse(output.result);
const checkpoints = new Set(
recorded.state.checkpoints.map((checkpoint) => checkpoint.name),
);
const { package: installed } = recorded.application;
return ok({
checkpoints,
packageFile: installsFromPackageFile(installed, recorded.host.architecture),
stopped: {
end_reason: recorded.session.end_reason,
checkpoints: recorded.state.checkpoints.map((checkpoint) => ({
name: checkpoint.name,
trigger: checkpoint.trigger,
changes: checkpoint.changes.length,
retained_files: checkpoint.retained.length,
})),
package: {
kind: installed.kind,
name: installed.name,
version: installed.version,
sources: [...installed.sources],
},
limitations: [...recorded.limitations],
...(await draftTask(computer, capture.folder, checkpoints, signal)),
},
});
};
src/application/desktop/DesktopBuildTurns.test.ts
+99
View File
@@ -0,0 +1,99 @@
import { setImmediate as settled } from "node:timers/promises";
import { expect, it } from "vitest";
import { ok } from "../../domain/result.js";
import { DesktopBuildTurns } from "./DesktopBuildTurns.js";
/** A call that notes its start and runs until the test ends it. */
const gated = (log: string[], name: string) => {
let end = (): void => undefined;
const ended = new Promise<void>((resolve) => {
end = resolve;
});
return {
end,
work: async () => {
log.push(name);
await ended;
return ok(name);
},
};
};
const signal = () => new AbortController().signal;
it("runs one call at a time in the order they arrive, and frees the turn of a call that throws", async () => {
const turns = new DesktopBuildTurns();
const log: string[] = [];
const first = gated(log, "first");
const throwing = turns.run("first", signal(), async () => {
await first.work();
throw new Error("the driver broke");
});
const second = turns.run("second", signal(), async () => {
log.push("second");
return ok("second");
});
await settled();
expect(log).toEqual(["first"]);
first.end();
await expect(throwing).rejects.toThrow("the driver broke");
await expect(second).resolves.toEqual(ok("second"));
expect(log).toEqual(["first", "second"]);
});
it("keeps the place of a call cancelled while it waits, so the one after it still waits", async () => {
const turns = new DesktopBuildTurns();
const log: string[] = [];
const running = gated(log, "running");
const first = turns.run("running", signal(), running.work);
const caller = new AbortController();
const cancelled = turns.run("cancelled", caller.signal, async () =>
ok("ran"),
);
const last = turns.run("last", signal(), async () => {
log.push("last");
return ok("last");
});
await settled();
caller.abort(new Error("the client went away"));
expect(await cancelled).toMatchObject({
ok: false,
error: { _tag: "AnalysisCancelledError", operation: "cancelled" },
});
await settled();
expect(log).toEqual(["running"]);
running.end();
await first;
await last;
expect(log).toEqual(["running", "last"]);
});
it("on close cancels the call running, refuses those waiting and runs once the running one has ended", async () => {
const turns = new DesktopBuildTurns();
const log: string[] = [];
// It finishes after the abort, as a step whose output was already committed would.
const running = turns.run("running", signal(), async (turn) => {
await new Promise((resolve) => turn.addEventListener("abort", resolve));
await settled();
log.push("running ended");
return ok("finished");
});
const waiting = turns.run("waiting", signal(), async () => ok("ran"));
await settled();
const closed = turns.close(async () => {
log.push("closed");
return "down";
});
expect(await waiting).toMatchObject({
ok: false,
error: { _tag: "AnalysisCancelledError", operation: "waiting" },
});
expect(await closed).toBe("down");
expect(await running).toEqual(ok("finished"));
expect(log).toEqual(["running ended", "closed"]);
expect(
await turns.run("later", signal(), async () => ok("ran")),
).toMatchObject({ ok: false, error: { _tag: "AnalysisCancelledError" } });
});
src/application/desktop/DesktopBuildTurns.ts
+64
View File
@@ -0,0 +1,64 @@
import type { AnalysisError } from "../../domain/analysisErrorBase.js";
import { AnalysisCancelledError } from "../../domain/analysisErrorCore.js";
import { err, type Result } from "../../domain/result.js";
import { ABORTED, waitForAbortable } from "../../process/AbortablePromise.js";
/**
* One build computer's calls, each run alone in the order it arrived, so a
* witness stage is admitted, read and filed with no action or other step in
* between. Closing cancels the call running and refuses those waiting.
*/
export class DesktopBuildTurns {
#tail: Promise<void> = Promise.resolve();
readonly #closing = new AbortController();
/**
* Run `work` once every call before it has ended; its signal also aborts
* when the computer closes. What `work` returns is the call's result, even
* if its turn aborted after the work committed what that result reports; a
* step that commits nothing checks its signal before succeeding.
*/
async run<Value>(
operation: string,
signal: AbortSignal,
work: (signal: AbortSignal) => Promise<Result<Value, AnalysisError>>,
): Promise<Result<Value, AnalysisError>> {
const turn = AbortSignal.any([signal, this.#closing.signal]);
const { previous, release } = this.#enqueue();
if ((await waitForAbortable(previous, turn)) === ABORTED || turn.aborted) {
// A call that never ran keeps its place until the one before it ends, so the next still waits for that one.
void previous.then(release);
return err(new AnalysisCancelledError(operation, { cause: turn.reason }));
}
try {
return await work(turn);
} finally {
release();
}
}
/** Cancel the call running and those waiting, then run `work` once the running one has ended. */
async close<Value>(work: () => Promise<Value>): Promise<Value> {
this.#closing.abort(new Error("The build computer was closed."));
const { previous, release } = this.#enqueue();
await previous;
try {
return await work();
} finally {
release();
}
}
/** Take the next place: the end of the call before it, and the release of this one. */
#enqueue(): {
readonly previous: Promise<void>;
readonly release: () => void;
} {
const previous = this.#tail;
let release = (): void => undefined;
this.#tail = new Promise<void>((resolve) => {
release = resolve;
});
return { previous, release };
}
}
src/application/desktop/DesktopBuildWitness.ts
+583
View File
@@ -0,0 +1,583 @@
import { createHash, randomUUID } from "node:crypto";
import { readdir } from "node:fs/promises";
import { join } from "node:path";
import { isDeepStrictEqual } from "node:util";
import { z } from "zod";
import { AnalysisError } from "../../domain/analysisErrorBase.js";
import { AnalysisInputError } from "../../domain/analysisErrorCore.js";
import {
parseDesktopTask,
type desktopTaskSchema,
type DesktopTask,
} from "../../domain/desktop/appTask.js";
import {
evaluateTaskChecks,
type TaskCheckOutcome,
} from "../../domain/desktop/appTaskChecks.js";
import {
renderTaskSteps,
renderTaskVerifier,
taskVerifierArtifact,
} from "../../domain/desktop/appTaskPython.js";
import {
admitWitnessStage,
appStateViews,
stageExpectation,
stageVerdict,
unrestoredChecks,
unreadResetChecks,
WITNESS_STAGES,
type WitnessStage,
type WitnessStageRecord,
} from "../../domain/desktop/buildWitness.js";
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
import { err, ok, type Result } from "../../domain/result.js";
import { safeParseUtf8Json } from "../../domain/safeJson.js";
import type {
AgentEnvComputer,
RecordingManifest,
} from "../../desktop/AgentEnvComputer.js";
import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvComputerHelper.js";
import { readHostFile } from "../../desktop/HostFileRead.js";
import { isPortableBundleName } from "../environment/environmentBundle.js";
import type { ReadonlyActionLog } from "./DesktopBuildActionLog.js";
import { errorText } from "./DesktopBuildCommands.js";
import {
writeWitnessRecord,
type EndedWitness,
type WitnessRecord,
} from "./DesktopBuildWitnessRecord.js";
const CHECK = "check_build_task";
const FRESH =
"Close this computer and start a fresh one from the bundle with start_build_computer.";
/** The computer routes the witness reads. */
export type WitnessedComputer = Pick<
AgentEnvComputer,
"appData" | "startRecording" | "endRecording" | "recordingSegments"
>;
type WitnessSummary = {
readonly stage: WitnessStage;
readonly expectation: "some_fail" | "all_pass";
readonly held: boolean;
readonly status: "open" | "held" | "failed";
readonly next_stage: WitnessStage | null;
readonly record: WitnessRecord | null;
};
/** One check of the task against the app's live state, and the witness stage it filed, if any. */
export type CheckedTask = {
readonly task: string;
readonly passed: number;
readonly failed: number;
readonly results: TaskCheckOutcome[];
readonly witness: WitnessSummary | null;
};
const refused = (userMessage: string, remediationAction: string) =>
new AnalysisInputError(CHECK, {
userMessage,
remediationAction,
conformsToInputContract: true,
});
const taskDigest = (task: DesktopTask) =>
createHash("sha256").update(JSON.stringify(task)).digest("hex");
/** Whether a folder can take the record: absent, or empty. */
const usableFolder = async (directory: string): Promise<boolean> =>
readdir(directory).then(
(entries) => entries.length === 0,
(cause: unknown) =>
(cause as NodeJS.ErrnoException | undefined)?.code === "ENOENT",
);
/** The most bytes REA reads of each of the bundle's files for the task. */
const BUNDLE_FILE_BYTES = 1024 * 1024;
const stepsSchema = z.array(
z.looseObject({ type: z.string(), env_id: z.string().optional() }),
);
/** Why the bundle's steps and verifier for the task are not what the export writes for it; undefined when they are. */
const bundleTaskDifference = async (
bundle: string,
task: DesktopTask,
signal: AbortSignal,
): Promise<string | undefined> => {
const stepsPath = `tasks/${task.name}.json`;
const verifierPath = `artifacts/${taskVerifierArtifact(task)}/verify.py`;
const [steps, verifier] = await Promise.all(
[stepsPath, verifierPath].map((path) =>
readHostFile(join(bundle, path), BUNDLE_FILE_BYTES, signal),
),
);
if (!Buffer.isBuffer(steps) || !Buffer.isBuffer(verifier))
return `the bundle at ${bundle} has no readable ${Buffer.isBuffer(steps) ? verifierPath : stepsPath}.`;
const parsed = safeParseUtf8Json(steps);
const listed = parsed.ok ? parsed.value : undefined;
const app = stepsSchema
.safeParse(listed)
.data?.find((step) => step.type === "deploy_env")?.env_id;
if (
app === undefined ||
!isDeepStrictEqual(listed, JSON.parse(renderTaskSteps(task, app)))
)
return `the bundle's ${stepsPath} differs from this task's steps.`;
return verifier.toString("utf8") === renderTaskVerifier(task, app)
? undefined
: `the bundle's ${verifierPath} differs from this task's checks.`;
};
/**
* A witness from its seed on: `open` while stages are checked, `saving`
* once a stage ended it and until its record is written, which a repeat of
* that stage's check retries, and `saved` after, with the record a repeat
* returns again.
*/
interface OpenWitness extends EndedWitness {
state: "open" | "saving" | "saved";
record?: WitnessRecord;
/** A manifest whose segments still need to be fetched. */
pendingManifest?: RecordingManifest | undefined;
/**
* A published record whose handles are still open. The error already carries
* the record and the cleanup lease; every later close failure returns it.
*/
published?:
| {
readonly error: AnalysisError;
readonly release: () => Promise<void>;
}
| undefined;
}
/** What the witness says of one filed stage. */
const summaryOf = (
filed: WitnessStageRecord,
next: WitnessStage | null,
record: WitnessRecord | null,
): WitnessSummary => ({
stage: filed.stage,
expectation: filed.expectation,
held: filed.held,
status: !filed.held ? "failed" : record === null ? "open" : "held",
next_stage: next,
record,
});
/** The pass and fail counts of one set of results. */
const counted = (task: DesktopTask, results: TaskCheckOutcome[]) => {
const passed = results.filter((result) => result.passed).length;
return { task: task.name, passed, failed: results.length - passed, results };
};
/**
* Grades an exported task against a build computer's app, as the bundle's
* verifier does, and witnesses it: the seed, the edit, a near miss and the
* seed again after a reset, recorded, with the action log, as Evidence.
*/
/**
* Retry closing a published witness. A further failure returns the original
* error, which already carries the record and the cleanup lease.
*/
export const retryPublishedCleanup = async (
published: AnalysisError,
release: () => Promise<void>,
): Promise<Result<void, AnalysisError>> => {
try {
await release();
return ok(undefined);
} catch {
return err(published);
}
};
export class DesktopBuildWitness {
#witness: OpenWitness | undefined;
/**
* The request id of this computer's one witness recording, fixed before any
* start is sent: a seed retried after a lost reply asks for the same id,
* and the computer answers the recording it already began.
*/
readonly #recordingRequest = `rea-witness:${randomUUID()}`;
/** `bundle` is the folder the computer was started from, if it was; the witness takes only the task it holds. */
constructor(
private readonly computer: WitnessedComputer,
private readonly actions: ReadonlyActionLog,
private readonly bundle?: string,
) {}
async check(
input: {
readonly task: z.output<typeof desktopTaskSchema>;
readonly stage?: WitnessStage | undefined;
readonly directory?: string | undefined;
},
signal: AbortSignal,
): Promise<Result<CheckedTask, AnalysisError>> {
const parsed = parseDesktopTask(input.task, isPortableBundleName);
if (!parsed.ok)
return err(
refused(
`The task is refused: ${parsed.error}`,
"Pass the task as export_build_environment took it.",
),
);
const task = parsed.value;
const { stage } = input;
if (stage !== undefined && this.#witness?.state === "saving")
return this.#resave(this.#witness, task, stage, signal);
const saved =
stage === undefined
? undefined
: this.#savedAgain(task, stage, input.directory);
if (saved !== undefined) return saved;
const admitted =
stage === undefined
? undefined
: await this.#admit(task, stage, input.directory, signal);
if (admitted?.ok === false) return admitted;
const graded = await this.#grade(task, signal);
if (!graded.ok) return graded;
const results = graded.value;
const checked = counted(task, results);
if (stage === undefined || admitted === undefined)
return ok({ ...checked, witness: null });
const filed = await this.#file(stage, admitted.value, results, signal);
return filed.ok ? ok({ ...checked, witness: filed.value }) : filed;
}
/** The task's checks against the app's live views, one result per check. */
async #grade(
task: DesktopTask,
signal: AbortSignal,
): Promise<Result<TaskCheckOutcome[], AnalysisError>> {
try {
const views = appStateViews(
await this.computer.appData("data/get", signal),
);
return views.ok
? ok([...evaluateTaskChecks(task.checks, views.value)])
: err(
refused(
views.error,
"Start the computer from a bundle export_build_environment wrote.",
),
);
} catch (cause: unknown) {
return err(
cause instanceof AnalysisError
? cause
: new ProviderAdapterError(AGENTENV_PROVIDER, CHECK, {
userMessage: `Reading the app's state failed: ${errorText(cause)}`,
cause,
}),
);
}
}
/**
* File a stage's results; one that does not hold, or the last, ends the
* witness and writes its record. One its checks leave undecided files
* nothing and can be checked again.
*/
async #file(
stage: WitnessStage,
actionsBefore: number,
results: TaskCheckOutcome[],
signal: AbortSignal,
): Promise<Result<WitnessSummary, AnalysisError>> {
const witness = this.#witness;
if (witness === undefined) throw new TypeError("the stage was admitted");
const verdict = stageVerdict(stage, results, witness.stages);
if (verdict === "undecided") {
const unreadSeed =
stage === "after_reset"
? unreadResetChecks(witness.stages, results).map(
(result) => result.id,
)
: [];
return err(
refused(
`The checks cannot decide ${stage}: each failure that would settle it may come from a view that could not be read or is cut short (${[
...results.flatMap((result) =>
result.undecided === true ? [result.description] : [],
),
...unreadSeed.map(
(id) => `${id} could not be compared with the seed`,
),
].join("; ")}). Nothing was filed.`,
`Check ${stage} again once the app's files read whole; a view that is always cut short or unreadable needs a task whose checks read other files.`,
),
);
}
const held = verdict === "held";
const unrestored =
stage === "after_reset" && !held
? unrestoredChecks(witness.stages, results).map((result) => result.id)
: [];
const filed: WitnessStageRecord = {
stage,
actions_before: actionsBefore,
checked_at: new Date().toISOString(),
expectation: stageExpectation(stage),
held,
results,
...(unrestored.length === 0
? {}
: {
remediation: `The reset did not restore the seed: ${unrestored.join(", ")} came out unlike on the seed. Find what the app's data/reset leaves changed, export again and witness a fresh computer of the new bundle.`,
}),
};
witness.stages.push(filed);
if (held && stage !== WITNESS_STAGES.at(-1))
return ok(
summaryOf(filed, WITNESS_STAGES[witness.stages.length] ?? null, null),
);
witness.state = "saving";
return this.#save(witness, filed, signal);
}
/** Write the ended witness's record; until that succeeds the witness keeps every stage for a retry. */
async #save(
witness: OpenWitness,
filed: WitnessStageRecord,
signal: AbortSignal,
): Promise<Result<WitnessSummary, AnalysisError>> {
const published = witness.published;
if (witness.record !== undefined && published !== undefined) {
const released = await retryPublishedCleanup(
published.error,
published.release,
);
if (!released.ok) return released;
witness.published = undefined;
witness.state = "saved";
return ok(summaryOf(filed, null, witness.record));
}
const written = await writeWitnessRecord(
this.computer,
witness,
this.actions,
signal,
witness.pendingManifest,
);
if (written.kind === "retry") {
if (written.manifest !== undefined)
witness.pendingManifest = written.manifest;
return err(written.error);
}
witness.pendingManifest = undefined;
witness.record = written.record;
if (written.kind === "published") {
witness.published = { error: written.error, release: written.release };
return err(written.error);
}
witness.state = "saved";
return ok(summaryOf(filed, null, written.record));
}
/**
* The saved witness again, for a repeat of the check that ended it with the
* same task, such as after a cancelled call whose record was saved anyway;
* undefined for any other check.
*/
#savedAgain(
task: DesktopTask,
stage: WitnessStage,
directory: string | undefined,
): Result<CheckedTask, AnalysisError> | undefined {
const witness = this.#witness;
const filed = witness?.stages.at(-1);
if (
witness?.record === undefined ||
filed?.stage !== stage ||
taskDigest(task) !== witness.digest
)
return undefined;
// A folder the record was not saved to gets nothing written, so it is refused.
if (directory !== undefined && directory !== witness.directory)
return err(
refused(
`This witness's record was saved to ${witness.directory}, not ${directory}.`,
"Ask again without directory or with the folder it was saved to; another witness needs a fresh computer.",
),
);
return ok({
...counted(task, filed.results),
witness: summaryOf(filed, null, witness.record),
});
}
/** Repeat the check of the stage that ended the witness to retry writing its record, without checking again. */
async #resave(
witness: OpenWitness,
task: DesktopTask,
stage: WitnessStage,
signal: AbortSignal,
): Promise<Result<CheckedTask, AnalysisError>> {
const filed = witness.stages.at(-1);
if (
filed === undefined ||
filed.stage !== stage ||
taskDigest(task) !== witness.digest
)
return err(
refused(
"This witness has ended, but its record is not saved yet.",
`Check ${filed?.stage ?? "the last stage"} again with the same task to save it.`,
),
);
const saved = await this.#save(witness, filed, signal);
return saved.ok
? ok({ ...counted(task, filed.results), witness: saved.value })
: saved;
}
/** Open the witness at its seed, or admit the next stage of the open one; its value is where its actions end. */
async #admit(
task: DesktopTask,
stage: WitnessStage,
directory: string | undefined,
signal: AbortSignal,
): Promise<Result<number, AnalysisError>> {
const digest = taskDigest(task);
const opened =
stage === "seed"
? await this.#open(task, digest, directory, signal)
: this.#continuing(digest, directory);
if (!opened.ok) return opened;
const admitted = admitWitnessStage(
opened.value.stages,
stage,
this.actions,
);
return admitted.ok
? admitted
: err(
refused(
admitted.error,
"Do what the message says, then check this stage again.",
),
);
}
/** Open the witness with its folder and begin its recording. */
async #open(
task: DesktopTask,
digest: string,
directory: string | undefined,
signal: AbortSignal,
): Promise<Result<OpenWitness, AnalysisError>> {
const unfinished = this.#witness;
if (unfinished !== undefined)
return this.#retrySeed(unfinished, digest, directory);
if (directory === undefined)
return err(
refused(
"The seed stage opens the witness and needs directory.",
"Give directory, a new or empty folder on this host for the record and recording.",
),
);
if (!(await usableFolder(directory)))
return err(
refused(
`${directory} exists and is not empty.`,
"Give a new or empty folder.",
),
);
const differs =
this.bundle === undefined
? undefined
: await bundleTaskDifference(this.bundle, task, signal);
if (differs !== undefined)
return err(
refused(
`The task is not the bundle's ${task.name} task: ${differs}`,
"Give the task exactly as export_build_environment took it for this bundle.",
),
);
try {
const { recording_id: recordingId } = await this.computer.startRecording(
this.#recordingRequest,
signal,
);
const witness: OpenWitness = {
task,
digest,
directory,
recordingId,
stages: [],
state: "open",
};
this.#witness = witness;
return ok(witness);
} catch (cause: unknown) {
return err(
new ProviderAdapterError(AGENTENV_PROVIDER, CHECK, {
userMessage: `The witness could not start its recording: ${errorText(cause)}`,
cause,
}),
);
}
}
/**
* A seed whose check failed after the witness opened, such as on a failed
* read of the app's state, is checked again with the same task and folder,
* keeping the recording it began.
*/
#retrySeed(
witness: OpenWitness,
digest: string,
directory: string | undefined,
): Result<OpenWitness, AnalysisError> {
if (witness.state !== "open" || witness.stages.length > 0)
return err(refused("This computer was already witnessed.", FRESH));
return witness.digest === digest && witness.directory === directory
? ok(witness)
: err(
refused(
"A seed check for another task or folder opened this witness and did not finish.",
"Check the seed again with the task and directory it was first given.",
),
);
}
/** The open witness a later stage continues, for the same task and without a folder. */
#continuing(
digest: string,
directory: string | undefined,
): Result<OpenWitness, AnalysisError> {
const witness = this.#witness;
if (witness === undefined)
return err(
refused(
"No witness is open on this computer.",
"Check the seed stage first, with directory.",
),
);
if (witness.state !== "open")
return err(refused("This witness has ended.", FRESH));
if (directory !== undefined)
return err(
refused(
"Only the seed stage takes directory.",
"Leave directory out after the seed stage.",
),
);
return witness.digest === digest
? ok(witness)
: err(
refused(
"This task differs from the one the seed stage checked.",
"Give every stage the task the seed stage checked.",
),
);
}
}
src/application/desktop/DesktopBuildWitnessRecord.test.ts
+132
View File
@@ -0,0 +1,132 @@
import { access } from "node:fs/promises";
import { join } from "node:path";
import { describe, expect, it, vi } from "vitest";
import { AnalysisCleanupLease } from "../../artifacts/AnalysisCleanupLease.js";
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
import { retryPublishedCleanup } from "./DesktopBuildWitness.js";
import type { RecordingManifest } from "../../desktop/AgentEnvComputer.js";
import { parseDesktopTask } from "../../domain/desktop/appTask.js";
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
import {
writeWitnessRecord,
type EndedWitness,
} from "./DesktopBuildWitnessRecord.js";
const manifest: RecordingManifest = {
recording_id: "rec-1",
segments: [],
};
const witness = async (): Promise<EndedWitness> => {
const parsed = parseDesktopTask(
{
name: "add-eggs",
prompt: "Add eggs.",
checks: [
{
kind: "lines_present",
file: "Documents/today.txt",
lines: ["eggs"],
},
],
},
(name) => /^[a-z0-9]+(?:-[a-z0-9]+)*$/u.test(name),
);
if (!parsed.ok) throw new Error(parsed.error);
return {
task: parsed.value,
digest: "digest",
directory: await createTestTempDirectory("witness-record-"),
recordingId: "rec-1",
stages: [],
};
};
const log = { entries: [], dropped: 0 };
describe("published witness cleanup", () => {
it("returns the same published error when closing the handles fails again", async () => {
const published = new ProviderAdapterError("agentenv", "check_build_task", {
userMessage: "handles open",
cleanup: { reason: "handles open", resources: ["/tmp/witness"] },
});
const lease = new AnalysisCleanupLease(async () => {
throw new Error("still open");
}, ["/tmp/witness"]);
lease.bindCleanupError(published);
const failClose = async () => {
throw new Error("still open");
};
const again = await retryPublishedCleanup(published, failClose);
const third = await retryPublishedCleanup(published, failClose);
expect(again.ok).toBe(false);
expect(third.ok).toBe(false);
if (again.ok || third.ok) return;
expect(again.error).toBe(published);
expect(third.error).toBe(published);
expect(published.cleanup?.resources).toEqual(["/tmp/witness"]);
expect(lease.released).toBe(false);
});
});
describe("writeWitnessRecord", () => {
it("does not publish a record when the recording cannot be ended, and retries the end", async () => {
const ended = await witness();
const endRecording = vi
.fn()
.mockRejectedValueOnce(new Error("recording busy"))
.mockResolvedValue(manifest);
const recordingSegments = vi.fn().mockResolvedValue([]);
const computer = { endRecording, recordingSegments };
const signal = new AbortController().signal;
const first = await writeWitnessRecord(computer, ended, log, signal);
expect(first.kind).toBe("retry");
if (first.kind !== "retry") return;
expect(first.manifest).toBeUndefined();
expect(first.error.userMessage).toContain("recording busy");
await expect(
access(join(ended.directory, "witness.json")),
).rejects.toThrow();
const second = await writeWitnessRecord(computer, ended, log, signal);
expect(second.kind).toBe("saved");
expect(endRecording).toHaveBeenCalledTimes(2);
await expect(
access(join(ended.directory, "witness.json")),
).resolves.toBeUndefined();
});
it("keeps a completed manifest and fetches only its segments on the next try", async () => {
const ended = await witness();
const endRecording = vi.fn().mockResolvedValue(manifest);
const recordingSegments = vi
.fn()
.mockRejectedValueOnce(new Error("segment unread"))
.mockResolvedValue([]);
const computer = { endRecording, recordingSegments };
const signal = new AbortController().signal;
const first = await writeWitnessRecord(computer, ended, log, signal);
expect(first.kind).toBe("retry");
if (first.kind !== "retry") return;
expect(first.manifest).toEqual(manifest);
await expect(
access(join(ended.directory, "witness.json")),
).rejects.toThrow();
const second = await writeWitnessRecord(
computer,
ended,
log,
signal,
first.manifest,
);
expect(second.kind).toBe("saved");
expect(endRecording).toHaveBeenCalledTimes(1);
expect(recordingSegments).toHaveBeenCalledTimes(2);
});
});
src/application/desktop/DesktopBuildWitnessRecord.ts
+323
View File
@@ -0,0 +1,323 @@
import { createHash } from "node:crypto";
import { join } from "node:path";
import { Readable } from "node:stream";
import { AnalysisError } from "../../domain/analysisErrorBase.js";
import type { DesktopTask } from "../../domain/desktop/appTask.js";
import type {
WitnessStage,
WitnessStageRecord,
} from "../../domain/desktop/buildWitness.js";
import { createEvidence, type Evidence } from "../../domain/evidence.js";
import type { JsonValue } from "../../domain/jsonValue.js";
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
import type {
AgentEnvComputer,
RecordingManifest,
} from "../../desktop/AgentEnvComputer.js";
import { AGENTENV_PROVIDER } from "../../desktop/AgentEnvComputerHelper.js";
import {
LOGGED_ACTIONS,
LOGGED_BYTES,
LOGGED_LIST_ITEMS,
LOGGED_TEXT_BYTES,
type BuildComputerAction,
type ReadonlyActionLog,
} from "./DesktopBuildActionLog.js";
import { errorText } from "./DesktopBuildCommands.js";
import { DESKTOP_PROVIDER } from "./DesktopCaptureRefusals.js";
import {
writeOutputTree,
writtenWithOpenHandles,
} from "./DesktopOutputTree.js";
const CHECK = "check_build_task";
const RECORD = "witness.json";
/** The computer routes the witness record reads. */
export type RecordedComputer = Pick<
AgentEnvComputer,
"endRecording" | "recordingSegments"
>;
/** A logged action as the record keeps it: the log's entry, as JSON. */
type RecordedAction = Omit<BuildComputerAction, "truncated_arguments"> & {
readonly truncated_arguments?: string[];
};
type Recording = WitnessRecord["recording"];
type Segment = { readonly path: string; readonly bytes: Buffer };
/** The witness record: each stage's checks, the actions between them and the recording. */
export type WitnessRecord = {
readonly path: string;
readonly evidence_id: string;
readonly verdict: "held" | "failed";
readonly failed_stage: WitnessStage | null;
readonly task: { readonly name: string; readonly sha256: string };
readonly stages: WitnessStageRecord[];
readonly actions: RecordedAction[];
/** How many of the earliest logged actions the log no longer keeps. */
readonly actions_dropped: number;
readonly recording:
| {
readonly recording_id: string;
readonly segments: {
readonly path: string;
readonly bytes: number;
readonly sha256: string;
}[];
}
| { readonly error: string };
};
/** What the record is written from: the witnessed task, its stages and where its folder is. */
export interface EndedWitness {
readonly task: DesktopTask;
readonly digest: string;
readonly directory: string;
readonly recordingId: string;
readonly stages: WitnessStageRecord[];
}
const sha256 = (bytes: Buffer) =>
createHash("sha256").update(bytes).digest("hex");
/** A recording fetch that can resume after the manifest is known. */
type RecordingFetch =
| {
readonly ok: true;
readonly recording: Exclude<Recording, { error: string }>;
readonly segments: readonly Segment[];
}
| {
readonly ok: false;
readonly error: string;
readonly manifest?: RecordingManifest;
};
/**
* End the recording and bring its segments back. A failure before the
* manifest leaves nothing to resume; a failure after it keeps the manifest
* so the next try only fetches the segments.
*/
const fetchRecording = async (
computer: RecordedComputer,
recordingId: string,
pending: RecordingManifest | undefined,
signal: AbortSignal,
): Promise<RecordingFetch> => {
let manifest = pending;
try {
manifest ??= await computer.endRecording(recordingId, signal);
} catch (cause: unknown) {
return { ok: false, error: errorText(cause) };
}
try {
const segments = await computer.recordingSegments(manifest, signal);
return {
ok: true,
segments,
recording: {
recording_id: manifest.recording_id,
segments: segments.map((segment) => ({
path: join(
"recording",
segment.path.split("/").at(-1) ?? segment.path,
),
bytes: segment.bytes.length,
sha256: sha256(segment.bytes),
})),
},
};
} catch (cause: unknown) {
return { ok: false, error: errorText(cause), manifest };
}
};
/** Write the recording's segments and the record's Evidence into the witness folder. */
const writeRecordFolder = async (
directory: string,
files: readonly { readonly path: string; readonly bytes: Buffer }[],
evidence: Evidence,
signal: AbortSignal,
): Promise<
| undefined
| { readonly published: false; readonly error: AnalysisError }
| {
readonly published: true;
readonly error: AnalysisError;
readonly release: () => Promise<void>;
}
> => {
try {
const unclosed = await writeOutputTree(
directory,
async (tree) => {
for (const file of files)
await tree.write(
file.path,
Readable.from([file.bytes]),
{ bytes: file.bytes.length, sha256: sha256(file.bytes) },
signal,
);
},
{ operation: CHECK, field: "directory" },
signal,
);
return unclosed === undefined
? undefined
: {
published: true,
error: writtenWithOpenHandles(
{
provider: AGENTENV_PROVIDER,
operation: CHECK,
what: "witness record",
},
directory,
unclosed,
evidence,
),
release: () => unclosed.lease.release(),
};
} catch (cause: unknown) {
return {
published: false,
error:
cause instanceof AnalysisError
? cause
: new ProviderAdapterError(AGENTENV_PROVIDER, CHECK, {
userMessage: `The witness record could not be written: ${errorText(cause)}`,
cause,
partialObservation: evidence,
}),
};
}
};
/** What the record's action log leaves out: its earliest actions, and parts of long arguments. */
const logLimitations = (
actions: readonly RecordedAction[],
dropped: number,
): string[] => [
...(dropped === 0
? []
: [
`The action log keeps the latest ${String(LOGGED_ACTIONS)} actions within ${String(LOGGED_BYTES / 1024)} KiB, so the ${String(dropped)} earliest are not in the record; the stages' rules counted them.`,
]),
...(actions.some((action) => action.truncated_arguments !== undefined)
? [
`Arguments named in an action's truncated_arguments are cut in the record, each string to ${String(LOGGED_TEXT_BYTES)} bytes and each list to ${String(LOGGED_LIST_ITEMS)} items; the computer received them whole.`,
]
: []),
];
/** How a witness write ended: saved, retryable, or published with cleanup still open. */
export type WitnessWrite =
| { readonly kind: "saved"; readonly record: WitnessRecord }
| {
readonly kind: "retry";
readonly error: AnalysisError;
readonly manifest?: RecordingManifest;
}
| {
readonly kind: "published";
readonly record: WitnessRecord;
readonly error: AnalysisError;
readonly release: () => Promise<void>;
};
/**
* End the witness's recording, fetch it, and write the segments and
* `witness.json`, the record as Evidence, into the witness folder. A
* recording that has not been fetched is not written, so the next try can
* resume it. A published folder whose handles fail to close is returned with
* the record and the cleanup to retry.
*/
export const writeWitnessRecord = async (
computer: RecordedComputer,
witness: EndedWitness,
log: Pick<ReadonlyActionLog, "entries" | "dropped">,
signal: AbortSignal,
pendingManifest?: RecordingManifest,
): Promise<WitnessWrite> => {
const actions = log.entries.map(
({ truncated_arguments: cut, ...entry }): RecordedAction =>
cut === undefined ? entry : { ...entry, truncated_arguments: [...cut] },
);
const dropped = log.dropped;
const fetched = await fetchRecording(
computer,
witness.recordingId,
pendingManifest,
signal,
);
if (!fetched.ok)
return {
kind: "retry",
...(fetched.manifest === undefined ? {} : { manifest: fetched.manifest }),
error: new ProviderAdapterError(AGENTENV_PROVIDER, CHECK, {
userMessage: `The witness recording could not be saved: ${fetched.error}. Check this stage again to retry it.`,
cause: new Error(fetched.error),
}),
};
const { recording, segments } = fetched;
const failed = witness.stages.find((stage) => !stage.held);
const result = {
verdict: failed === undefined ? ("held" as const) : ("failed" as const),
failed_stage: failed?.stage ?? null,
task: { name: witness.task.name, sha256: witness.digest },
stages: witness.stages,
actions,
actions_dropped: dropped,
recording,
};
const evidence = createEvidence(undefined, DESKTOP_PROVIDER, {
predicateType: "rea.desktop-build-witness",
operation: CHECK,
parameters: { task: witness.task.name, stages: witness.stages.length },
result: result as unknown as JsonValue,
confidence: "observed",
authority: "controlled-replay",
limitations: [
"Checks read the app's data/get views with REA's evaluator, as the exported verifier does; the verifier itself did not run.",
...("error" in recording
? [`The recording is missing: ${recording.error}`]
: []),
...logLimitations(actions, dropped),
],
});
const files = [
...("segments" in recording
? recording.segments.flatMap((target, index) => {
const bytes = segments[index]?.bytes;
return bytes === undefined ? [] : [{ path: target.path, bytes }];
})
: []),
{
path: RECORD,
bytes: Buffer.from(`${JSON.stringify(evidence, null, 2)}\n`),
},
];
const unwritten = await writeRecordFolder(
witness.directory,
files,
evidence,
signal,
);
const record = {
path: join(witness.directory, RECORD),
evidence_id: evidence.evidence_id,
...result,
};
if (unwritten === undefined) return { kind: "saved", record };
if (unwritten.published)
return {
kind: "published",
record,
error: unwritten.error,
release: unwritten.release,
};
return { kind: "retry", error: unwritten.error };
};
src/application/desktop/DesktopOutputTree.ts
+98
View File
@@ -0,0 +1,98 @@
import { mkdir } from "node:fs/promises";
import { dirname } from "node:path";
import {
ArtifactResourceScope,
type ArtifactCleanupAttempt,
} from "../../artifacts/ArtifactResourceScope.js";
import { SafeOutputTree } from "../../artifacts/SafeOutputTree.js";
import { outputDirectoryAdmissionRefusal } from "../../artifacts/SafeOutputTreeAdmission.js";
import { SafeOutputTreeCreationFailure } from "../../artifacts/SafeOutputTreeCreationFailure.js";
import type { Evidence } from "../../domain/evidence.js";
import { ProviderAdapterError } from "../../domain/providerAdapterError.js";
/** A published tree whose output handles could not be released. */
export type FailedRelease = Extract<
ArtifactCleanupAttempt,
{ readonly kind: "failed" }
>;
/**
* Write a new or empty folder through the safe output tree, rolling an
* unpublished tree back on any failure so no partial copy remains. A
* published tree is kept; a failed release of its handles is returned. A
* refused destination names the caller's `field` under `operation`.
*/
export const writeOutputTree = async (
directory: string,
write: (tree: SafeOutputTree) => Promise<void>,
refusal: { readonly operation: string; readonly field: string },
signal: AbortSignal | undefined,
): Promise<FailedRelease | undefined> => {
await mkdir(dirname(directory), { recursive: true, mode: 0o700 });
const resources = new ArtifactResourceScope();
let tree: SafeOutputTree | undefined;
let outcome:
| { readonly kind: "completed" }
| { readonly kind: "failed"; readonly cause: unknown };
try {
try {
tree = await SafeOutputTree.create(directory, process.platform, {
acceptExistingEmpty: true,
});
} catch (cause: unknown) {
if (cause instanceof SafeOutputTreeCreationFailure) tree = cause.tree;
throw cause;
}
await write(tree);
await tree.commit(signal);
outcome = { kind: "completed" };
} catch (cause: unknown) {
outcome = { kind: "failed", cause };
}
if (tree !== undefined) {
const released = await resources.release({
kind: "output-tree",
resource: directory,
tree,
});
if (released.kind === "failed") {
if (tree.published) return released;
throw resources.cleanupFailure(outcome, [released]);
}
if (tree.published) return undefined;
}
if (outcome.kind === "failed")
throw outputDirectoryAdmissionRefusal(
refusal.operation,
refusal.field,
outcome.cause,
);
return undefined;
};
/** A complete folder whose output handles could not be closed, reported with what it holds. */
export const writtenWithOpenHandles = (
context: {
readonly provider: string;
readonly operation: string;
readonly what: string;
},
directory: string,
release: FailedRelease,
evidence?: Evidence,
): ProviderAdapterError => {
const reason =
release.cause instanceof Error
? release.cause.message
: String(release.cause);
const error = new ProviderAdapterError(context.provider, context.operation, {
userMessage: `The ${context.what} was written to ${directory} and is complete, but its output handles could not be closed: ${reason}.`,
remediationAction: `Use the written ${context.what}; the handles are released when this process exits.`,
cleanup: { reason, resources: [directory] },
...(evidence === undefined ? {} : { partialObservation: evidence }),
diagnostics: { output_directory: directory },
});
release.lease.bindCleanupError(error);
return error;
};
src/application/desktop/DesktopTaskExport.test.ts
+122
View File
@@ -4,12 +4,15 @@ import { join } from "node:path";
import { describe, expect, it } from "vitest";
import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js";
import { AnalysisInputError } from "../../domain/analysisErrorCore.js";
import { exportDesktopApplication } from "./DesktopApplicationExport.js";
import { draftDesktopApplicationTask } from "./DesktopTaskDraft.js";
import {
NOTE as NOTE_PATH,
capture,
change,
readers,
retained,
writeCaptureFolder,
} from "./desktopCapture.fixture.js";
@@ -83,7 +86,126 @@ const unretainedNoteCapture = () => {
};
};
/** A note of 6,000 lines, more than the 5,000 a text view keeps, and the edit that adds `milk` at its top. */
const LONG_NOTE = `${Array.from(
{ length: 6000 },
(_, index) => `note ${String(index + 1)}`,
).join("\n")}\n`;
const MILK_NOTE = `milk\n${LONG_NOTE}`;
/** The capture with the long note at the seed and `milk` added to it by the edit. */
const longNoteCapture = () => {
const base = capture();
return {
...base,
state: {
...base.state,
checkpoints: base.state.checkpoints.map((checkpoint) =>
checkpoint.trigger === "baseline"
? {
...checkpoint,
retained: [
...checkpoint.retained,
retained(NOTE_PATH, LONG_NOTE),
],
}
: {
...checkpoint,
changes: checkpoint.changes.map((entry) =>
entry.path === NOTE_PATH
? change(NOTE_PATH, { before: LONG_NOTE, after: MILK_NOTE })
: entry,
),
retained: checkpoint.retained.map((entry) =>
entry.path === NOTE_PATH
? retained(NOTE_PATH, MILK_NOTE)
: entry,
),
},
),
},
};
};
describe("exportDesktopApplication with a task's seed", () => {
it("refuses a task whose only seed failure misses a line its cut view may have left out", async () => {
const scratch = await createTestTempDirectory("desktop-task-");
const path = join(scratch, "task.json");
await writeFile(
path,
JSON.stringify(
task([
{
id: "has-milk",
kind: "lines_present",
file: NOTE,
lines: ["milk"],
},
]),
),
);
const directory = join(scratch, "bundle");
const result = await exportDesktopApplication(
{
capture: await writeCaptureFolder(longNoteCapture(), undefined, [
LONG_NOTE,
MILK_NOTE,
]),
directory,
name: undefined,
tasks: [path],
},
readers,
);
expect(result).toBeInstanceOf(AnalysisInputError);
const { issues, userMessage } = result as AnalysisInputError;
expect(issues.map(({ path }) => path)).toEqual([["task"]]);
// The undecided check by id, with its file and line, in any wording.
for (const fact of [JSON.stringify("has-milk"), NOTE, '"milk"'])
expect(userMessage).toContain(fact);
expect(await exists(directory)).toBe(false);
// A failure on what the seed's view shows still keeps it, and only that one counts.
await writeFile(
path,
JSON.stringify(
task([
{
id: "has-milk",
kind: "lines_present",
file: NOTE,
lines: ["milk"],
},
{
id: "row",
kind: "row_present",
file: DATABASE,
table: "notes",
where: { id: 2 },
},
]),
),
);
const kept = await exportDesktopApplication(
{
capture: await writeCaptureFolder(longNoteCapture(), undefined, [
LONG_NOTE,
MILK_NOTE,
]),
directory,
name: undefined,
tasks: [path],
},
readers,
);
if (kept instanceof Error) throw kept;
expect(kept.normalized_result).toMatchObject({
task: { failing_on_seed: ["row"] },
});
});
it("treats a file present but not retained at the baseline as absent on the seed, as the computer starts", async () => {
const scratch = await createTestTempDirectory("desktop-task-");
const path = join(scratch, "task.json");
src/application/desktop/DesktopTaskExport.ts
+16 -4
View File
@@ -250,15 +250,27 @@ export const admitDesktopTask = async (
"task",
"Write checks that the captured end state passes; describe-desktop-application-task shows its views.",
);
const failingOnSeed = evaluateTaskChecks(task.checks, views.seed)
.filter((outcome) => !outcome.passed)
.map((outcome) => outcome.id);
if (failingOnSeed.length === 0)
const seed = evaluateTaskChecks(task.checks, views.seed).filter(
(outcome) => !outcome.passed,
);
if (seed.length === 0)
throw refuse(
"Every check already passes on the seed, so the task would be done before the agent starts.",
"task",
"Add a check that the change the user made is needed for.",
);
// Only a failure on what the seed's view shows proves the task is not done yet.
const failingOnSeed = seed
.filter((outcome) => outcome.undecided !== true)
.map((outcome) => outcome.id);
if (failingOnSeed.length === 0)
throw refuse(
`No check fails on what the seed's views show: each failure there may come from a view that could not be read or is cut short (${seed
.map(({ id, description }) => `${JSON.stringify(id)}: ${description}`)
.join("; ")}).`,
"task",
"Add a check that fails on what the seed's views hold, such as a line or row of a file read whole; describe-desktop-application-task shows its views.",
);
return {
summary: {
name: task.name,
src/browser/PngVisualDiff.test.ts
+17 -1
View File
@@ -3,7 +3,7 @@ import { crc32, deflateSync } from "node:zlib";
import { describe, expect, it } from "vitest";
import { readPngHeaderDimensions } from "../domain/pngHeader.js";
import { comparePngScreenshots } from "./PngVisualDiff.js";
import { comparePngScreenshots, isOneColourPng } from "./PngVisualDiff.js";
import {
AnalysisResourceConstraintError,
AnalysisUnsupportedTargetError,
@@ -196,6 +196,22 @@ describe("PNG visual diff", () => {
});
});
describe("isOneColourPng", () => {
it("tells a one-colour frame from a drawn one, and an unreadable one from both", () => {
const base64 = (pixels: readonly number[]) =>
png(2, 2, pixels).toString("base64");
const black = [0, 0, 0, 255];
expect(
isOneColourPng(base64([...black, ...black, ...black, ...black])),
).toBe(true);
expect(
isOneColourPng(base64([...black, ...black, ...black, 0, 0, 1, 255])),
).toBe(false);
expect(isOneColourPng("AAAA")).toBeUndefined();
});
});
describe("PNG comparison memory budget", () => {
it("classifies legal huge dimensions as a memory constraint before inflate", () => {
const image = createWebScreenshotArtifact(headerOnlyPng(0x7fffffff, 1));
src/browser/PngVisualDiff.ts
+22 -2
View File
@@ -98,9 +98,29 @@ export const comparePngScreenshots = (
};
};
/**
* Whether every pixel of a PNG has one RGBA value, decoded with the
* comparison's checks and encodings within the working-memory budget it
* keeps for a pair; undefined when the PNG cannot be decoded so.
*/
export const isOneColourPng = (dataBase64: string): boolean | undefined => {
let rgba: Buffer;
try {
const plan = parsePng({ data_base64: dataBase64 }, "image");
admitPngPair(plan, plan);
rgba = decodePng(plan).rgba;
} catch {
return undefined;
}
const first = rgba.readUInt32BE(0);
for (let offset = 4; offset < rgba.length; offset += 4)
if (rgba.readUInt32BE(offset) !== first) return false;
return true;
};
const parsePng = (
artifact: WebScreenshotArtifact,
field: "before" | "after",
artifact: Pick<WebScreenshotArtifact, "data_base64">,
field: "before" | "after" | "image",
): ParsedPng => {
const bytes = decodeCanonicalBase64(artifact.data_base64);
if (bytes === undefined || !hasPngSignature(bytes))
src/cli/desktopCaptureCommands.ts
+13 -5
View File
@@ -14,7 +14,10 @@ import { absoluteCallerPath } from "../filesystem/callerPath.js";
import { cliDecimalField } from "./options.js";
import { CLI_COMMANDS } from "../cliCommandNames.js";
import { logCliCommand } from "../cliLogging.js";
import { DESKTOP_CAPTURE_DEFAULTS } from "../domain/desktop/desktopCapture.js";
import {
DESKTOP_CAPTURE_DEFAULTS,
type DesktopCaptureProgress,
} from "../domain/desktop/desktopCapture.js";
import { SessionRequestQueue } from "../desktop/DesktopSessionRequests.js";
import { withCommandCancellation } from "./commandCancellation.js";
import type { CliInstance } from "./types.js";
@@ -327,10 +330,15 @@ export const registerDesktopCaptureCommands = (
environment,
callerDirectory: process.cwd(),
nextRequest: () => requests.next(),
report: async (message) => {
process.stderr.write(
`${JSON.stringify({ rea_progress: { phase: "desktop_capture", message } })}\n`,
);
report: async (message, event) => {
const progress: DesktopCaptureProgress = {
rea_progress: {
phase: "desktop_capture",
message,
...(event === undefined ? {} : { event }),
},
};
process.stderr.write(`${JSON.stringify(progress)}\n`);
},
},
signal,
src/contracts/desktopBuildToolContracts.ts
+302 -3
View File
@@ -1,12 +1,25 @@
import { z } from "zod";
import { buildSourceSchema } from "../domain/desktop/appBuild.js";
import { desktopTaskSchema } from "../domain/desktop/appTask.js";
import {
buildCheckpointNameSchema,
END_CHECKPOINT,
SEED_CHECKPOINT,
} from "../domain/desktop/buildCapture.js";
import { WITNESS_STAGES } from "../domain/desktop/buildWitness.js";
import {
COMPUTER_ACTIONS,
computerActionInputSchema,
screenshotOptionsSchema,
} from "../domain/desktop/computerActions.js";
import { jsonObjectSchema } from "../domain/jsonValue.js";
import { desktopCheckpointSchema } from "../domain/desktop/desktopCapture.js";
import {
jsonObjectSchema,
jsonValueSchema,
type JsonValue,
} from "../domain/jsonValue.js";
import { absoluteLocalPathStringSchema } from "../domain/localPath.js";
import type { ToolContract } from "./toolContractTypes.js";
import { toolContractMetadata } from "./toolEffects.js";
@@ -17,6 +30,89 @@ const driverResult = jsonObjectSchema
"The driver's structured result, such as frame_id, width and height; null when it returned none.",
);
const EXAMPLE_ID = "00000000-0000-4000-8000-000000000001";
const EXAMPLE_TASK: Readonly<Record<string, JsonValue>> = {
name: "add-eggs",
prompt: "Add eggs to the shopping list in Documents/todo.txt and save it.",
checks: [
{ kind: "lines_present", file: "Documents/todo.txt", lines: ["eggs"] },
],
};
const checkResult = z.strictObject({
id: z.string(),
passed: z.boolean(),
description: z.string(),
undecided: z
.literal(true)
.optional()
.describe(
"Present on a failure for want of a readable or whole view, such as a file the app could not read just then or a view cut short; it shows nothing about the task, so a witness stage counts no such failure.",
),
});
const strings = z.array(z.string());
const stageRecord = z.strictObject({
stage: z.enum(WITNESS_STAGES),
actions_before: z
.int()
.nonnegative()
.describe("How many logged actions came before this stage's check."),
checked_at: z.string(),
expectation: z.enum(["some_fail", "all_pass"]),
held: z.boolean(),
results: z.array(checkResult),
remediation: z
.string()
.optional()
.describe(
"Why the stage did not hold when its results alone do not say, such as a reset that did not restore the seed, and what to do.",
),
});
const loggedAction = z.strictObject({
index: z.int().nonnegative(),
action: z.enum(COMPUTER_ACTIONS),
tool: z.string(),
arguments: jsonObjectSchema,
truncated_arguments: z
.array(z.string())
.optional()
.describe(
"Arguments this entry cuts, each string to 1 KiB and each list to 64 items; the computer received them whole.",
),
started_at: z.string(),
ended_at: z.string().nullable(),
outcome: z.enum(["pending", "applied", "refused", "failed"]),
});
const witnessRecord = z.strictObject({
path: z.string().describe("witness.json, the record as REA Evidence."),
evidence_id: z.string(),
verdict: z.enum(["held", "failed"]),
failed_stage: z.enum(WITNESS_STAGES).nullable(),
task: z.strictObject({ name: z.string(), sha256: z.string() }),
stages: z.array(stageRecord),
actions: z
.array(loggedAction)
.describe(
"The latest actions logged on the computer, up to 256 within 256 KiB, oldest first.",
),
actions_dropped: z
.int()
.nonnegative()
.describe(
"How many of the earliest logged actions the log no longer keeps; the stages' rules counted them.",
),
recording: z.union([
z.strictObject({
recording_id: z.string(),
segments: z.array(
z.strictObject({
path: z.string().describe("Relative to the witness folder."),
bytes: z.int().nonnegative(),
sha256: z.string(),
}),
),
}),
z.strictObject({ error: z.string() }),
]),
});
/** Session tools that keep an agentenv computer up and forward computer-use actions to it while an environment is built. */
export const DESKTOP_BUILD_TOOL_CONTRACTS = [
@@ -59,7 +155,7 @@ export const DESKTOP_BUILD_TOOL_CONTRACTS = [
name: "act_on_build_computer",
...toolContractMetadata("act_on_build_computer"),
kind: "application",
description: `Drive the app on a build computer: one action, ${COMPUTER_ACTIONS.join(", ")}, sent to the computer's computer_<action> tool with the fields given, which are the driver's own arguments. Points are pixels of the latest frame, so take a screenshot first, or space norm on a 0..norm_scale grid; count 2 double-clicks; keys names one chord in W3C names, and spellings such as ctrl+s are translated. Returns the driver's content unchanged, its images included, and its structured result; an action's screenshot is taken at once, so after one that opens a menu or a window, take it with a wait instead; a refusal, such as a stale frame, is an error with the computer's message. Each action is logged on the computer.`,
description: `Drive the app on a build computer: one action, ${COMPUTER_ACTIONS.join(", ")}. Each but reset is sent to the computer's computer_<action> tool with the fields given, which are the driver's own arguments; reset restores the app's data to its seed with data/reset, which relaunches the app, and returns once the computer is ready again; a screenshot asked of it that then fails leaves the reset done, its result naming why in screenshot_error. Points are pixels of the latest frame, so take a screenshot first, or space norm on a 0..norm_scale grid; count 2 double-clicks; keys names one chord in W3C names, and spellings such as ctrl+s are translated. Returns the driver's content unchanged, its images included, and its structured result; an action's screenshot is taken at once, so after one that opens a menu or a window, take it with a wait instead; a refusal, such as a stale frame, is an error with the computer's message. Each action is logged on the computer. Actions, capture steps and checks on one computer run one at a time, in the order they arrive; computers run side by side.`,
inputSchema: computerActionInputSchema,
outputSchema: z.strictObject({
computer_id: z.uuid(),
@@ -91,12 +187,215 @@ export const DESKTOP_BUILD_TOOL_CONTRACTS = [
},
],
},
{
name: "start_build_capture",
...toolContractMetadata("start_build_capture"),
kind: "application",
description: `Begin REA's capture of the app on a build computer started from a request, inside the computer: REA downloads the package lists as the computer's user so the capture finds the app's sources, launches the app from its desktop entry under capture (do not open it from the menu first) and returns once its window is open, with a screenshot. Then make the seed through the app and save it, mark_build_checkpoint ${SEED_CHECKPOINT}, make the change the task asks for and save it, mark_build_checkpoint ${END_CHECKPOINT}, and stop_build_capture.`,
inputSchema: z.strictObject({
computer_id: computerId,
arguments: z
.array(z.string().max(4096))
.max(32)
.optional()
.describe(
"Arguments appended to the app's launch command, such as a file to open.",
),
}),
outputSchema: z.strictObject({
computer_id: z.uuid(),
desktop_entry: z.string(),
package_lists: z
.int()
.nonnegative()
.describe(
"apt package lists found for the capture to read sources from.",
),
package_lists_error: z
.string()
.nullable()
.describe("Why downloading the package lists failed, if it did."),
screenshot: driverResult,
}),
examples: [
{
title: "Capture the requested app",
input: { computer_id: EXAMPLE_ID },
},
],
},
{
name: "mark_build_checkpoint",
...toolContractMetadata("mark_build_checkpoint"),
kind: "application",
description: `Take a named checkpoint of the app's state in the capture under way and wait for the capture to acknowledge it, with its counts: entries walked, paths changed since the checkpoint before, and files retained. Mark ${SEED_CHECKPOINT} once the seed is saved and ${END_CHECKPOINT} once the change is; a name already taken is refused.`,
inputSchema: z.strictObject({
computer_id: computerId,
name: buildCheckpointNameSchema,
}),
outputSchema: z.strictObject({
computer_id: z.uuid(),
checkpoint: z.strictObject({
name: z.string(),
trigger: desktopCheckpointSchema.shape.trigger,
at_ms: z.int().nonnegative(),
entries: z.int().nonnegative(),
changes: z.int().nonnegative(),
retained_files: z.int().nonnegative(),
retained_bytes: z.int().nonnegative(),
}),
}),
examples: [
{
title: "Mark the saved seed",
input: { computer_id: EXAMPLE_ID, name: SEED_CHECKPOINT },
},
],
},
{
name: "stop_build_capture",
...toolContractMetadata("stop_build_capture"),
kind: "application",
description: `Stop the capture: the app is closed and the capture finishes inside the computer. Returns its checkpoints, the package and sources it found, its limitations and, when ${SEED_CHECKPOINT} and ${END_CHECKPOINT} were marked, the task draft between them: the files that changed, with their net change, and suggested checks. Write the task from the draft, its prompt in the user's words, and give it to export_build_environment.`,
inputSchema: z.strictObject({ computer_id: computerId }),
outputSchema: z.strictObject({
computer_id: z.uuid(),
end_reason: z.string(),
checkpoints: z.array(
z.strictObject({
name: z.string(),
trigger: z.string(),
changes: z.int().nonnegative(),
retained_files: z.int().nonnegative(),
}),
),
package: z.strictObject({
kind: z.string(),
name: z.string().nullable(),
version: z.string().nullable(),
sources: strings,
}),
limitations: strings,
draft: z
.strictObject({
seed_checkpoint: z.string(),
end_checkpoint: z.string(),
files: z.array(
z.strictObject({
file: z.string(),
role: z.string(),
change: jsonValueSchema,
}),
),
task: z.strictObject({
name: z.string(),
prompt: z.string(),
checks: z.array(jsonObjectSchema),
}),
})
.nullable(),
draft_refusal: z.string().optional(),
}),
examples: [
{ title: "Stop and draft the task", input: { computer_id: EXAMPLE_ID } },
],
},
{
name: "export_build_environment",
...toolContractMetadata("export_build_environment"),
kind: "application",
description: `Export the stopped capture as an agentenv computer app bundle with the given task, inside the computer, seeded at ${SEED_CHECKPOINT} and checked at ${END_CHECKPOINT}, then copy the bundle to a new or empty folder on this host, every file checked against its digest. A task the export refuses, such as a check that already passes on the seed, is an error naming why. Witness the bundle next: start_build_computer from the folder and check_build_task.`,
inputSchema: z.strictObject({
computer_id: computerId,
task: desktopTaskSchema,
directory: absoluteLocalPathStringSchema.describe(
"A new or empty folder on this host for the bundle.",
),
name: z
.string()
.min(1)
.optional()
.describe("The environment's name; derived from the app by default."),
}),
outputSchema: z.strictObject({
computer_id: z.uuid(),
directory: z.string(),
environment: z.string(),
files: z.int().nonnegative(),
bytes: z.int().nonnegative(),
// The export's task summary: its name, checks and those failing on the seed.
task: jsonValueSchema,
seed_files: strings,
scrubbed_files: strings,
unscrubbed_files: strings,
limitations: strings,
}),
examples: [
{
title: "Export the bundle with the task",
input: {
computer_id: EXAMPLE_ID,
task: EXAMPLE_TASK,
directory: "/tmp/mousepad-bundle",
},
},
],
},
{
name: "check_build_task",
...toolContractMetadata("check_build_task"),
kind: "application",
description: `Grade a task's checks against the app's live state on a computer of an exported bundle: REA reads the app's data/get views and evaluates each check as the bundle's verifier does, one result per check. With stage, the check is part of the witness, in order: seed (give directory; REA begins recording) must fail a check, and on a computer started from a bundle folder it refuses a task whose steps and verifier in the bundle differ from what the export writes for it; after_edit, after acting on the app from the task's prompt alone, must pass every check; near_miss, after a deliberately almost-right change, must fail one; after_reset, after act_on_build_computer action reset, must give each check the result it had on the seed. Only a check that fails on what the app's view shows counts as failing: one that fails because the view could not be read or is cut short is undecided, and a stage only undecided checks would settle files nothing and is refused, to be checked again. A stage that does not hold, or after_reset, ends the witness: REA ends the recording and writes witness.json, the record as Evidence, with the recording's segments into directory, and returns the record.`,
inputSchema: z.strictObject({
computer_id: computerId,
task: desktopTaskSchema,
stage: z
.enum(WITNESS_STAGES)
.optional()
.describe(
"The witness stage this check files; none for a plain check.",
),
directory: absoluteLocalPathStringSchema
.optional()
.describe(
"seed only: a new or empty folder on this host for the record and recording.",
),
}),
outputSchema: z.strictObject({
computer_id: z.uuid(),
task: z.string(),
passed: z.int().nonnegative(),
failed: z.int().nonnegative(),
results: z.array(checkResult),
witness: z
.strictObject({
stage: z.enum(WITNESS_STAGES),
expectation: z.enum(["some_fail", "all_pass"]),
held: z.boolean(),
status: z.enum(["open", "held", "failed"]),
next_stage: z.enum(WITNESS_STAGES).nullable(),
record: witnessRecord.nullable(),
})
.nullable(),
}),
examples: [
{
title: "Check the seed and begin the witness",
input: {
computer_id: EXAMPLE_ID,
task: EXAMPLE_TASK,
stage: "seed",
directory: "/tmp/mousepad-witness",
},
},
],
},
{
name: "close_build_computer",
...toolContractMetadata("close_build_computer"),
kind: "application",
description:
"Tear a build computer down and discard its state. agentenv's teardown is tried first; if it leaves the computer up, REA removes that exact sandbox's containers, the one named agent-<sandbox id> and any labelled agentenv.sandbox=<sandbox id>, with REA_DOCKER_COMMAND. When neither confirms it, the error names what is left and the computer stays listed, so closing again retries.",
"Tear a build computer down and discard its state. agentenv's teardown is tried first; if it leaves the computer up, REA removes that exact sandbox's containers, the one named agent-<sandbox id> and any labelled agentenv.sandbox=<sandbox id>, with REA_DOCKER_COMMAND. When neither confirms it, the error names what is left and the computer stays listed, so closing again retries. Closing cancels the call in progress on the computer and refuses those waiting for it.",
inputSchema: z.strictObject({ computer_id: computerId }),
outputSchema: z.strictObject({
computer_id: z.uuid(),
src/contracts/toolEffects.test.ts
+1
View File
@@ -53,6 +53,7 @@ describe("tool effects", () => {
"rizin_debug_command",
"start_adb_activity",
"start_adb_app",
"start_build_capture",
"start_build_computer",
"start_rizin_debug_session",
"stop_adb_app",
src/contracts/toolEffects.ts
+36
View File
@@ -754,6 +754,42 @@ const TOOL_EFFECT_AUDIT: Readonly<Record<string, EffectAudit>> = {
executesForeignCode: true,
idempotent: false,
}),
// Downloads package lists and launches the app inside the computer.
start_build_capture: effects({
mutatesSession: true,
launchesProcess: true,
accessesNetwork: true,
changesUiState: true,
executesForeignCode: true,
idempotent: false,
}),
mark_build_checkpoint: effects({
mutatesSession: true,
accessesNetwork: true,
idempotent: false,
}),
// Closes the app and drafts the task with REA inside the computer.
stop_build_capture: effects({
mutatesSession: true,
launchesProcess: true,
accessesNetwork: true,
changesUiState: true,
idempotent: false,
}),
export_build_environment: effects({
mutatesSession: true,
writesFilesystem: true,
launchesProcess: true,
accessesNetwork: true,
idempotent: false,
}),
// A witness stage records the display and writes its record on this host.
check_build_task: effects({
mutatesSession: true,
writesFilesystem: true,
accessesNetwork: true,
idempotent: false,
}),
close_build_computer: effects({
mutatesSession: true,
launchesProcess: true,
src/desktop/AgentEnvComputer.ts
+240 -22
View File
@@ -1,4 +1,4 @@
import { createHash } from "node:crypto";
import { createHash, randomUUID } from "node:crypto";
import { setTimeout as delay } from "node:timers/promises";
import { getHeapStatistics } from "node:v8";
@@ -12,14 +12,24 @@ import type {
ComputerHelper,
ComputerRelease,
} from "./AgentEnvComputerHelper.js";
import { drawnScreenshot } from "./DrawnScreenshot.js";
import { awaitHttpReply, type HttpReply } from "./HttpReply.js";
const READY_TIMEOUT_MS = 180_000;
const REQUEST_TIMEOUT_MS = 120_000;
const PROGRESS_MS = 5_000;
const RECORDINGS = ".agentenv-recordings";
/** A control route's answer: readiness, or a recording's manifest, near 1.5 MB for a day of 15 s segments. */
const CONTROL_RESPONSE_BYTES = 4 * 1024 * 1024;
/** What a data/get answer holds around one file's base64 bytes. */
const ENVELOPE_BYTES = 64 * 1024;
/** An exec answer: two streams of at most 1 MiB each, which JSON escaping can grow sixfold. */
const EXEC_RESPONSE_BYTES = 16 * 1024 * 1024;
/** The base64 answer for `bytes` of files, with each file's envelope. */
const dataAnswerBytes = (bytes: number, files: number) =>
Math.ceil(bytes / 3) * 4 + ENVELOPE_BYTES * Math.max(files, 1);
/** The most REA reads into memory for one answer of unknown size: a quarter of the V8 heap, as for recordings. */
const heapQuarter = () => Math.floor(getHeapStatistics().heap_size_limit / 4);
const readinessSchema = z.object({
ready: z.boolean(),
@@ -67,6 +77,31 @@ const dataGetSchema = z.union([
}),
z.object({ error: z.looseObject({ message: z.string() }) }),
]);
const rpcSchema = z.union([
z.object({ result: z.looseObject({}) }),
z.object({ error: z.looseObject({ message: z.string() }) }),
]);
const execSchema = z.object({
exit_code: z.int().nullable(),
stdout: z.string(),
stderr: z.string(),
truncated: z.boolean(),
timed_out: z.boolean(),
});
/** A command's outcome through `computer-exec/v1`, its output without the control token. */
export type ComputerCommandResult = z.infer<typeof execSchema>;
const cardSchema = z.looseObject({
children_environments: z
.array(z.looseObject({ name: z.string(), url: z.string().optional() }))
.optional(),
});
/** How long a route may take and how much of its answer REA reads. */
interface RequestBounds {
readonly limit?: { readonly bytes: number; readonly what: string };
/** The deadline of a route that answers only once its work ends, which REA waits out past fetch's 300 s for headers. */
readonly timeoutMs?: number;
}
/** A driver tool's answer as REA forwards it: its content blocks unchanged and its structured result. */
export interface DriverResult {
@@ -77,7 +112,7 @@ export interface DriverResult {
/** A response body read only while it stays within `maxBytes`. */
const readBounded = async (
response: Response,
response: HttpReply,
maxBytes: number,
what: string,
): Promise<string> => {
@@ -105,6 +140,27 @@ export class ComputerStartFailure extends Error {
}
}
/** Every few seconds while a computer comes up, report the phase and agentenv's latest line. */
const progressHeartbeat = (
helper: ComputerHelper,
phase: { readonly name: string },
progress: ((message: string) => void) | undefined,
) => {
if (progress === undefined) return undefined;
const startedAt = Date.now();
return setInterval(() => {
const line = helper
.output()
.stderr.trimEnd()
.split("\n")
.at(-1)
?.slice(-200);
progress(
`${phase.name} (${String(Math.round((Date.now() - startedAt) / 1000))} s)${line === undefined || line === "" ? "" : `: ${line}`}`,
);
}, PROGRESS_MS);
};
const connectDriver = async (
mcpUrl: string,
signal: AbortSignal,
@@ -130,6 +186,22 @@ const connectDriver = async (
* session to its driver, and its control token, which never leaves this
* object.
*/
/**
* The app refused a data method. The refusal is the app's own error, not a
* lost reply. Callers must use this tag rather than the message text.
*/
export class AppDataRefusal extends Error {
readonly _tag = "AppDataRefusal" as const;
constructor(
readonly method: "data/get" | "data/reset",
readonly appMessage: string,
) {
super(`${method} failed: ${appMessage}`);
this.name = "AppDataRefusal";
}
}
export class AgentEnvComputer {
readonly sandboxId: string;
readonly #baseUrl: string;
@@ -153,20 +225,40 @@ export class AgentEnvComputer {
/**
* Connect to the computer a helper keeps up: wait for its handshake and
* readiness and open an MCP session to its driver. A failure throws a
* `ComputerStartFailure`, or the signal's reason when cancelled; the
* helper's owner releases it either way.
* readiness, open an MCP session to its driver and take its first frame
* once the desktop has drawn, reporting progress throughout. A failure
* throws a `ComputerStartFailure`, or the signal's reason when cancelled;
* the helper's owner releases it either way.
*/
static async connect(
helper: ComputerHelper,
signal: AbortSignal,
): Promise<AgentEnvComputer> {
progress?: (message: string) => void,
): Promise<{
readonly computer: AgentEnvComputer;
readonly frame: DriverResult;
}> {
const phase = { name: "agentenv is deploying the computer" };
const heartbeat = progressHeartbeat(helper, phase, progress);
try {
return await AgentEnvComputer.#connect(helper, signal, phase);
} finally {
clearInterval(heartbeat);
}
}
static async #connect(
helper: ComputerHelper,
signal: AbortSignal,
phase: { name: string },
): ReturnType<typeof AgentEnvComputer.connect> {
const handshake = await helper.handshake(signal);
if (handshake === undefined || "error" in handshake)
throw new ComputerStartFailure(
`agentenv did not bring the computer up: ${handshake?.error ?? "its helper ended without a handshake"}.`,
"Read the helper's output; `agent-env run <bundle> --task smoke --no-emulate` reproduces the deploy.",
);
phase.name = "waiting for the computer's readiness checks";
let client: StructuredContentMcpClient | undefined;
try {
client = await connectDriver(handshake.mcp_url, signal);
@@ -178,7 +270,12 @@ export class AgentEnvComputer {
client,
});
await computer.#waitReady(signal);
return computer;
phase.name = "waiting for the desktop to draw";
const frame = await drawnScreenshot(
(retry) => computer.callTool("computer_screenshot", {}, retry),
signal,
);
return { computer, frame };
} catch (cause: unknown) {
await client?.close().catch(() => undefined);
if (signal.aborted) throw signal.reason;
@@ -277,9 +374,11 @@ export class AgentEnvComputer {
},
signal,
{
// Base64 plus the JSON-RPC envelope and the part's metadata.
bytes: Math.ceil(segment.bytes / 3) * 4 + ENVELOPE_BYTES,
what: `The computer's answer for recording segment ${segment.path}`,
limit: {
// Base64 plus the JSON-RPC envelope and the part's metadata.
bytes: Math.ceil(segment.bytes / 3) * 4 + ENVELOPE_BYTES,
what: `The computer's answer for recording segment ${segment.path}`,
},
},
),
);
@@ -302,6 +401,114 @@ export class AgentEnvComputer {
return bytes;
}
/**
* The files under paths of the computer user's home, by home path, through
* the driver's `data/get`; its answer is read within `bytes` of files.
*/
async homeFiles(
paths: readonly string[],
bytes: number,
signal?: AbortSignal,
): Promise<ReadonlyMap<string, Buffer>> {
const answer = dataGetSchema.parse(
await this.#request(
"/agentenv",
{ jsonrpc: "2.0", id: 1, method: "data/get", params: { paths } },
signal,
{
limit: {
bytes: dataAnswerBytes(bytes, paths.length),
what: `The computer's answer for ${paths.join(", ")}`,
},
},
),
);
if ("error" in answer)
throw new Error(
`data/get failed: ${this.#redacted(answer.error.message)}`,
);
return new Map(
answer.result.parts.flatMap((part) => {
const path = Object.values(part.metadata ?? {})[0]?.path;
const encoded = part.file?.bytes;
return path === undefined || encoded === undefined
? []
: [[path, Buffer.from(encoded, "base64")] as const];
}),
);
}
/** Run a command in the computer's desktop session as its user; output is cut at the computer's cap. */
async exec(
argv: readonly string[],
timeoutS: number,
signal?: AbortSignal,
): Promise<ComputerCommandResult> {
const result = execSchema.parse(
await this.#request(
"/agentenv/ext/computer-exec/run",
{ argv, request_id: randomUUID(), timeout_s: timeoutS },
signal,
{
limit: {
bytes: EXEC_RESPONSE_BYTES,
what: `The computer's answer to ${argv[0] ?? "a command"}`,
},
timeoutMs: (timeoutS + 30) * 1000,
},
),
);
return {
...result,
stdout: this.#redacted(result.stdout),
stderr: this.#redacted(result.stderr),
};
}
/**
* The parts the computer's one app answers a data method with: `data/get`,
* read within a quarter of the V8 heap since it carries the app's files, or
* `data/reset`.
*/
async appData(
method: "data/get" | "data/reset",
signal?: AbortSignal,
): Promise<readonly unknown[]> {
const card = cardSchema.parse(
await this.#request("/.well-known/agent-env.json", undefined, signal),
);
const [app, ...others] = card.children_environments ?? [];
if (app === undefined || others.length > 0)
throw new Error(
`The computer lists ${String(card.children_environments?.length ?? 0)} apps; REA reads a computer with one app.`,
);
const path = new URL(app.url ?? `/svc/${app.name}/agentenv`, this.#baseUrl)
.pathname;
const answer = rpcSchema.parse(
await this.#request(
path,
{ jsonrpc: "2.0", id: 1, method, params: {} },
signal,
{
limit: {
bytes: heapQuarter(),
what: `The ${app.name} app's ${method} answer`,
},
},
),
);
if ("error" in answer)
throw new AppDataRefusal(method, this.#redacted(answer.error.message));
return method === "data/get"
? z.object({ parts: z.array(z.unknown()) }).parse(answer.result).parts
: [];
}
/** Wait until the computer's readiness checks pass, as after a reset relaunches its app. */
async waitUntilReady(signal: AbortSignal): Promise<void> {
await this.#waitReady(signal);
}
/** End REA's MCP session and bring the computer down; a failed release can be retried. */
async close(): Promise<ComputerRelease> {
await this.#client.close().catch(() => undefined);
@@ -331,18 +538,26 @@ export class AgentEnvComputer {
}
}
/** A framework-only route of the computer, with its control token; its answer is read within `limit`. */
#redacted(text: string): string {
return this.#token === null
? text
: text.replaceAll(this.#token, "[control token]");
}
/** A framework-only route of the computer, with its control token; its answer is read within `bounds.limit`. */
async #request(
path: string,
body: unknown,
signal: AbortSignal | undefined,
limit = {
bounds: RequestBounds = {},
): Promise<unknown> {
const limit = bounds.limit ?? {
bytes: CONTROL_RESPONSE_BYTES,
what: `The computer's answer to ${path}`,
},
): Promise<unknown> {
const timeout = AbortSignal.timeout(REQUEST_TIMEOUT_MS);
const response = await fetch(`${this.#baseUrl}${path}`, {
};
const timeout = AbortSignal.timeout(bounds.timeoutMs ?? REQUEST_TIMEOUT_MS);
const url = `${this.#baseUrl}${path}`;
const init = {
method: body === undefined ? "GET" : "POST",
headers: {
...(this.#token === null
@@ -351,14 +566,17 @@ export class AgentEnvComputer {
...(body === undefined ? {} : { "content-type": "application/json" }),
},
...(body === undefined ? {} : { body: JSON.stringify(body) }),
redirect: "error",
signal:
signal === undefined ? timeout : AbortSignal.any([signal, timeout]),
});
};
const deadline =
signal === undefined ? timeout : AbortSignal.any([signal, timeout]);
const response: HttpReply =
bounds.timeoutMs === undefined
? await fetch(url, { ...init, redirect: "error", signal: deadline })
: await awaitHttpReply(new URL(url), init, deadline);
const text = await readBounded(response, limit.bytes, limit.what);
if (!response.ok)
if (response.status < 200 || response.status > 299)
throw new Error(
`The computer answered ${path} with HTTP ${String(response.status)}: ${text.slice(0, 512)}`,
`The computer answered ${path} with HTTP ${String(response.status)}: ${this.#redacted(text.slice(0, 512))}`,
);
return JSON.parse(text) as unknown;
}
src/desktop/AgentEnvComputerHelper.ts
+7 -1
View File
@@ -220,7 +220,12 @@ export class ComputerHelper {
});
}
/** Spawn the helper for a bundle folder; it deploys the bundle's smoke task. */
/**
* Spawn the helper for a bundle folder; it deploys the bundle's smoke task.
* Python writes no bytecode, in the helper or the children that inherit its
* environment, such as agentenv's own import of the bundle's app class, so
* the bundle's files stay as they were shared.
*/
static async spawn(options: {
readonly bundle: string;
readonly environment: Readonly<NodeJS.ProcessEnv>;
@@ -238,6 +243,7 @@ export class ComputerHelper {
runId: randomUUID(),
expectedCommand: null,
hostEnvironment: environment,
env: { PYTHONDONTWRITEBYTECODE: "1" },
stdin: "pipe",
signal,
});
src/desktop/BuildComputerController.test.ts
+89
View File
@@ -0,0 +1,89 @@
import { createHash } from "node:crypto";
import { describe, expect, it } from "vitest";
import { fetchHomeFolder } from "./BuildComputerController.js";
const MIB = 1024 * 1024;
const sha256 = (bytes: Buffer) =>
createHash("sha256").update(bytes).digest("hex");
/** A computer whose controller lists `files` into a file beside the folder and whose data/get answers them, noting each fetch of a file. */
const computer = (files: ReadonlyMap<string, Buffer>, steps: string[]) => {
const listing = Buffer.from(
JSON.stringify({
files: [...files].map(([path, bytes]) => ({
path,
size: bytes.length,
sha256: sha256(bytes),
})),
skipped: [],
}),
);
return {
exec: () =>
Promise.resolve({
exit_code: 0,
stdout: `${JSON.stringify({ listing_bytes: listing.length })}\n`,
stderr: "",
truncated: false,
timed_out: false,
}),
homeFiles: (paths: readonly string[], bytes: number) => {
if (paths[0]?.endsWith(".listing.json") === true)
return Promise.resolve(new Map([[paths[0], listing]]));
steps.push(`get ${paths.join(",")} within ${String(bytes)}`);
return Promise.resolve(
new Map(
paths.map((path) => [path, files.get(path) ?? Buffer.alloc(0)]),
),
);
},
};
};
describe("fetchHomeFolder", () => {
it("hands over each checked batch before asking for the next, so one batch at a time is held", async () => {
const files = new Map([
[".rea-build/b/a", Buffer.alloc(5 * MIB, 1)],
[".rea-build/b/b", Buffer.alloc(4 * MIB, 2)],
[".rea-build/b/c", Buffer.alloc(1 * MIB, 3)],
]);
const steps: string[] = [];
await fetchHomeFolder(computer(files, steps), ".rea-build/b", {
file: (file) => {
steps.push(`write ${file.path}`);
return Promise.resolve();
},
});
expect(steps).toEqual([
`get .rea-build/b/a within ${String(5 * MIB)}`,
"write a",
`get .rea-build/b/b,.rea-build/b/c within ${String(5 * MIB)}`,
"write b",
"write c",
]);
});
it("refuses a file that differs from its listing before handing it over", async () => {
const files = new Map([[".rea-build/b/a", Buffer.from("seed")]]);
const steps: string[] = [];
const base = computer(files, steps);
const changed = {
...base,
homeFiles: (paths: readonly string[], bytes: number) =>
paths[0] === ".rea-build/b/a"
? Promise.resolve(new Map([[".rea-build/b/a", Buffer.from("SEED")]]))
: base.homeFiles(paths, bytes),
};
await expect(
fetchHomeFolder(changed, ".rea-build/b", {
file: () => {
steps.push("write");
return Promise.resolve();
},
}),
).rejects.toThrow(/differs from its listing/u);
expect(steps).toEqual([]);
});
});
src/desktop/BuildComputerController.ts
+237
View File
@@ -0,0 +1,237 @@
import { createHash } from "node:crypto";
import { readFile } from "node:fs/promises";
import { fileURLToPath } from "node:url";
import { z } from "zod";
import type { JsonValue } from "../domain/jsonValue.js";
import type { AgentEnvComputer } from "./AgentEnvComputer.js";
const CONTROLLER = fileURLToPath(
new URL("../../bridge/agentenv/capture.py", import.meta.url),
);
/** The most file bytes one `data/get` brings back, so a large bundle arrives in several answers. */
const FETCH_BATCH_BYTES = 8 * 1024 * 1024;
let source: Promise<string> | undefined;
/** The computer routes REA's build controller uses. */
export type ControlledComputer = Pick<AgentEnvComputer, "exec" | "homeFiles">;
/**
* Run one command of REA's build controller in the computer, as its user and
* in its home, and return the JSON object it answers with. The controller's
* source travels with the command, so nothing of it is installed.
*/
export const runBuildController = async (
computer: ControlledComputer,
command: "start" | "mark" | "stop" | "run" | "list",
argumentsJson: Readonly<Record<string, JsonValue>>,
options: { readonly timeoutS: number; readonly signal?: AbortSignal },
): Promise<Readonly<Record<string, unknown>>> => {
const { timeoutS, signal } = options;
source ??= readFile(CONTROLLER, "utf8");
const result = await computer.exec(
[
"python3",
"-I",
"-c",
await source,
command,
JSON.stringify(argumentsJson),
],
// The controller's own waits end first, so it can answer.
timeoutS + 60,
signal,
);
const line = result.stdout.trim().split("\n").at(-1) ?? "";
try {
const parsed = z
.record(z.string(), z.unknown())
.safeParse(JSON.parse(line));
if (parsed.success) return parsed.data;
} catch {
// Reported below with what the command wrote.
}
throw new Error(
`REA's build controller (${command}) ${result.timed_out ? "timed out" : `exited ${String(result.exit_code)}`} without answering: ${result.stderr.slice(-2048) || result.stdout.slice(-2048)}`,
);
};
/** What REA's build controller answers a command with. The wire format stays here. */
export const answerSchema = z.looseObject({
error: z.string().optional(),
absent: z.boolean().optional(),
line: z.string().optional(),
exited: z.int().optional(),
timed_out: z.boolean().optional(),
progress: z.string().optional(),
output: z.string().optional(),
exit_code: z.int().nullable().optional(),
output_bytes: z.int().nonnegative().optional(),
stdout_bytes: z.int().nonnegative().optional(),
stderr: z.string().optional(),
apt: z
.looseObject({ lists: z.int().nonnegative(), error: z.string().optional() })
.optional(),
});
export type Answer = z.infer<typeof answerSchema>;
/** One controller command, its answer parsed. */
export const controller = async (
computer: ControlledComputer,
command: "start" | "mark" | "stop",
argumentsJson: Readonly<Record<string, JsonValue>>,
options: { readonly timeoutS: number; readonly signal: AbortSignal },
): Promise<Answer> =>
answerSchema.parse(
await runBuildController(computer, command, argumentsJson, options),
);
/**
* Run one `rea` command in the computer with its output in a file under the
* work folder, after writing the files it reads; the output's size.
*/
export const runRea = async (
computer: ControlledComputer,
argv: readonly string[],
options: {
readonly work: string;
readonly stdout: string;
readonly files?: Readonly<Record<string, string>>;
readonly signal: AbortSignal;
},
): Promise<number> => {
const answer = answerSchema.parse(
await runBuildController(
computer,
"run",
{
work: options.work,
argv: [...argv],
stdout: options.stdout,
files: { ...options.files },
timeout_s: 1800,
},
{ timeoutS: 1800, signal: options.signal },
),
);
if (answer.exit_code === undefined || answer.stdout_bytes === undefined)
throw new Error(
answer.error ?? `rea ${argv[0] ?? ""} did not finish in the computer.`,
);
return answer.stdout_bytes;
};
const listedSchema = z.object({ listing_bytes: z.int().nonnegative() });
const listingSchema = z.object({
files: z.array(
z.object({
path: z.string().min(1),
size: z.int().nonnegative(),
sha256: z.string().regex(/^[0-9a-f]{64}$/u),
}),
),
skipped: z.array(z.string()),
});
/** One file brought back from the computer, by its path under the fetched folder. */
export interface FetchedFile {
readonly path: string;
readonly bytes: Buffer;
readonly sha256: string;
}
/**
* Every regular file under a home folder with its size and digest, and what
* else the folder holds. The controller writes the list to a file beside the
* folder and REA reads it through `data/get`, within the size it answers, as
* it reads REA's own outputs: a large bundle's list outgrows the 1 MiB a
* command's output is cut at.
*/
const listFolder = async (
computer: ControlledComputer,
folder: string,
signal: AbortSignal | undefined,
): Promise<z.infer<typeof listingSchema>> => {
const output = `${folder}.listing.json`;
const { listing_bytes: bytes } = listedSchema.parse(
await runBuildController(
computer,
"list",
{ root: folder, output },
{ timeoutS: 300, ...(signal === undefined ? {} : { signal }) },
),
);
const content = (await computer.homeFiles([output], bytes, signal)).get(
output,
);
if (content === undefined) throw new Error(`The computer has no ${output}.`);
return listingSchema.parse(JSON.parse(content.toString("utf8")));
};
/** The answers for one home folder: files listed, then fetched in batches of about `FETCH_BATCH_BYTES`. */
const batched = (files: z.infer<typeof listingSchema>["files"]) => {
const batches: { files: typeof files; bytes: number }[] = [];
for (const file of files) {
const last = batches.at(-1);
if (last === undefined || last.bytes + file.size > FETCH_BATCH_BYTES)
batches.push({ files: [file], bytes: file.size });
else {
last.files.push(file);
last.bytes += file.size;
}
}
return batches;
};
/**
* Bring every regular file under a home folder of the computer back to REA:
* listed with sizes and digests in the computer, fetched through `data/get`
* in answers of at most 8 MiB of files each (a larger file comes alone),
* each checked against its listing and handed to `sink` before the next
* answer is asked for, so one answer at a time is held. A link or special
* file is refused. The value counts the files and bytes handed over.
*/
export const fetchHomeFolder = async (
computer: ControlledComputer,
folder: string,
sink: { readonly file: (file: FetchedFile) => Promise<void> },
signal?: AbortSignal,
): Promise<{ readonly files: number; readonly bytes: number }> => {
const listing = await listFolder(computer, folder, signal);
if (listing.skipped.length > 0)
throw new Error(
`${folder} holds entries other than regular files: ${listing.skipped.join(", ")}.`,
);
let handed = { files: 0, bytes: 0 };
for (const { files: batch, bytes } of batched(listing.files)) {
const files = await computer.homeFiles(
batch.map((file) => file.path),
bytes,
signal,
);
for (const file of batch) {
const content = files.get(file.path);
const sha256 =
content === undefined
? undefined
: createHash("sha256").update(content).digest("hex");
if (
!file.path.startsWith(`${folder}/`) ||
content?.length !== file.size ||
sha256 !== file.sha256
)
throw new Error(
`${file.path} is missing from the computer's answer or differs from its listing.`,
);
await sink.file({
path: file.path.slice(folder.length + 1),
bytes: content,
sha256,
});
handed = { files: handed.files + 1, bytes: handed.bytes + file.size };
}
}
return handed;
};
src/desktop/DesktopApplicationSession.ts
+12 -2
View File
@@ -4,6 +4,7 @@ import { setTimeout as delay } from "node:timers/promises";
import { AnalysisCleanupLease } from "../artifacts/AnalysisCleanupLease.js";
import {
checkpointNameRefusal,
type DesktopCaptureEvent,
type DesktopSession,
} from "../domain/desktop/desktopCapture.js";
import { ownedCommandResources } from "../process/OwnedCommand.js";
@@ -81,7 +82,11 @@ export interface DesktopSessionPlan {
atMs: number,
openedFiles: ReadonlySet<string>,
) => Promise<void>;
readonly report: (message: string) => Promise<void>;
/** Progress text, with the event a driver waits on when there is one. */
readonly report: (
message: string,
event?: DesktopCaptureEvent,
) => Promise<void>;
readonly signal: AbortSignal | undefined;
}
@@ -134,6 +139,7 @@ const sampleSession = async (
observations.announced = true;
await plan.report(
"The application window is open. Send an empty line or `checkpoint <name>` to record its state, or `stop` to finish.",
{ kind: "window_open" },
);
}
return applicationWindows;
@@ -161,7 +167,11 @@ const requestCheckpoint = async (
): Promise<void> => {
const refusal = checkpointNameRefusal(name, taken);
if (refusal !== undefined) {
await plan.report(`${refusal} No checkpoint was recorded.`);
await plan.report(`${refusal} No checkpoint was recorded.`, {
kind: "checkpoint_refused",
name: name ?? "",
reason: refusal,
});
return;
}
const chosen = name ?? `checkpoint-${String(observations.now())}`;
src/desktop/DrawnScreenshot.ts
+52
View File
@@ -0,0 +1,52 @@
import { setTimeout as delay } from "node:timers/promises";
import { isOneColourPng } from "../browser/PngVisualDiff.js";
import type { JsonValue } from "../domain/jsonValue.js";
/** How often REA retakes a first screenshot the desktop has not drawn on, and the time the retries share. */
const DRAWN_POLL_MS = 500;
const DRAWN_WAIT_MS = 10_000;
/** A driver's screenshot answer, as far as the wait reads it. */
interface Screenshot {
readonly isError: boolean;
readonly content: readonly Readonly<Record<string, JsonValue>>[];
}
/** Whether a screenshot's PNG is one colour throughout, as a display is before the desktop draws. */
const undrawn = (shot: Screenshot): boolean => {
const image = shot.content.find(
(block) => block.type === "image" && block.mimeType === "image/png",
);
return typeof image?.data === "string" && isOneColourPng(image.data) === true;
};
/**
* A whole-screen PNG screenshot once the desktop has drawn: readiness can
* pass while the display is still one colour, so REA takes another every
* 500 ms while it is. Retries share 10 s, each given only what is left of
* it; when that runs out, the latest frame is returned.
*/
export const drawnScreenshot = async <Shot extends Screenshot>(
take: (signal: AbortSignal) => Promise<Shot>,
signal: AbortSignal,
): Promise<Shot> => {
const deadline = Date.now() + DRAWN_WAIT_MS;
let shot = await take(signal);
while (!shot.isError && undrawn(shot)) {
const left = deadline - Date.now();
if (left <= 0) break;
await delay(Math.min(DRAWN_POLL_MS, left), undefined, { signal });
const retry = AbortSignal.any([
signal,
AbortSignal.timeout(Math.max(1, deadline - Date.now())),
]);
try {
shot = await take(retry);
} catch (cause: unknown) {
if (signal.aborted || !retry.aborted) throw cause;
break;
}
}
return shot;
};
src/desktop/HttpReply.ts
+36
View File
@@ -0,0 +1,36 @@
import { request as httpRequest } from "node:http";
import { request as httpsRequest } from "node:https";
/** An HTTP answer's status and body. */
export interface HttpReply {
readonly status: number;
readonly body: AsyncIterable<Uint8Array> | null;
}
/** Send a request and wait for its answer until `signal` ends it, with none of fetch's 300 s limit on the wait for headers. */
export const awaitHttpReply = (
url: URL,
init: {
readonly method: string;
readonly headers: Readonly<Record<string, string>>;
readonly body?: string;
},
signal: AbortSignal,
): Promise<HttpReply> =>
new Promise((resolve, reject) => {
const body = init.body ?? "";
(url.protocol === "https:" ? httpsRequest : httpRequest)(
url,
{
method: init.method,
headers: {
...init.headers,
"content-length": String(Buffer.byteLength(body)),
},
signal,
},
(message) => resolve({ status: message.statusCode ?? 0, body: message }),
)
.on("error", reject)
.end(body);
});
src/domain/desktop/appBuild.test.ts
+9 -3
View File
@@ -10,6 +10,7 @@ import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirect
import { PACKAGE_METADATA } from "../../generatedPackageMetadata.js";
import {
BUILD_NODE_VERSION,
BUILD_PACKAGE_FILE,
buildSourceSchema,
renderBuildBundle,
type BuildBundleFile,
@@ -53,7 +54,7 @@ describe("build bundle renderer", () => {
'packages=Packages(apt=["mousepad", "libgtk-3-bin"])',
);
expect(module).toContain(
'launch=Launch(at="demand", command=["gtk-launch", "org.xfce.mousepad"]),',
`launch=Launch(at="demand", command=["sh", "-c", "cd \\"$HOME\\" 2>/dev/null; exec gtk-launch 'org.xfce.mousepad'"]),`,
);
expect(module).toContain('"arch": ["amd64", "arm64"]');
expect(module).toContain(
@@ -146,9 +147,14 @@ describe("build bundle install sources", () => {
);
expect(module).toContain('"arch": ["amd64"]');
expect(module).not.toContain('to="node-arm64.tar.xz"');
// REA's layer keeps the .deb for the capture before the install removes it.
expect(module).toContain(
'run=[RunScript(script="install-package.sh"), RunScript(script="install-rea.sh")]',
'run=[RunScript(script="install-rea.sh"), RunScript(script="install-package.sh")]',
);
expect(text(bundle.files, "envs/mousepad/install-rea.sh")).toContain(
`cp vendor/package.deb ${BUILD_PACKAGE_FILE}`,
);
expect(BUILD_PACKAGE_FILE).toBe("/opt/rea/package.deb");
expect(text(bundle.files, "envs/mousepad/install-package.sh")).toContain(
"dpkg -i 'vendor/package.deb'",
);
@@ -271,7 +277,7 @@ describe("installed launch paths", () => {
);
const module = text(bundle.files, "envs/typora/typora_app.py");
expect(module).toContain(
'launch=Launch(at="demand", command=["gtk-launch", "typora"]),',
`launch=Launch(at="demand", command=["sh", "-c", "cd \\"$HOME\\" 2>/dev/null; exec gtk-launch 'typora'"]),`,
);
expect(module).toContain(
'run=[RunScript(script="install-rea.sh"), RunScript(script="prepare-electron-launch.sh")]',
src/domain/desktop/appBuild.ts
+24 -6
View File
@@ -13,6 +13,7 @@ import {
renderEnvToml,
renderInstallScript,
renderSmokeTask,
shellQuote,
} from "./appExport.js";
import { INSTALL_SCRIPT, type ExportPackageFile } from "./appExportInstall.js";
import { isArmoredKey } from "./aptRepositories.js";
@@ -36,6 +37,8 @@ const COMPUTER_ARCHITECTURES = ["amd64", "arm64"] as const;
/** The package of `gtk-launch`, which opens the app from its desktop entry. */
const LAUNCHER_PACKAGE = "libgtk-3-bin";
const REA_INSTALL_SCRIPT = "install-rea.sh";
/** Where the build layer keeps a `.deb` the app came from, for the capture's `--package-file`. */
export const BUILD_PACKAGE_FILE = "/opt/rea/package.deb";
const ELECTRON_LAUNCH_SCRIPT = "prepare-electron-launch.sh";
const REA_PAYLOAD = "rea-payload.tar";
@@ -144,11 +147,22 @@ const download = (
): string =>
`Download(url=${py(url)}, sha256=${py(sha256)}, to=${py(to)}${arch === undefined ? "" : `, arch=${py(arch)}`})`;
/** The build-only step: unpack Node.js and REA into /opt/rea, as root and offline, and put both on PATH. */
const renderReaInstallScript = (): string => `#!/bin/sh
/**
* The build-only step: unpack Node.js and REA into /opt/rea, as root and
* offline, put both on PATH, and keep a copy of the app's `.deb`, which its
* own install step removes, for the capture to record.
*/
const renderReaInstallScript = (
packageFile: string | undefined,
): string => `#!/bin/sh
# Build-only layer, run as root with the network off: unpack Node.js and REA into /opt/rea and put both on PATH.
set -eu
mkdir -p /opt/rea/payload
mkdir -p /opt/rea/payload${
packageFile === undefined
? ""
: `
cp ${packageFile} ${BUILD_PACKAGE_FILE}`
}
python3 - <<'PY'
import glob
import tarfile
@@ -244,9 +258,10 @@ const buildInstall = (
key,
packageFile,
downloads,
// REA's layer runs first, so it can keep the .deb the install removes.
scripts: [
...(packageFile === null ? [] : [INSTALL_SCRIPT]),
REA_INSTALL_SCRIPT,
...(packageFile === null ? [] : [INSTALL_SCRIPT]),
...(request.electron === true ? [ELECTRON_LAUNCH_SCRIPT] : []),
],
};
@@ -328,7 +343,7 @@ from agentenv_protocol.computer import AptSource, ComputerAppEnvironment, Downlo
packages=Packages(apt=${pyList(install.packages)}),${sources}
downloads=[${install.downloads.join(", ")}],
run=[${install.scripts.map((script) => `RunScript(script=${py(script)})`).join(", ")}],
launch=Launch(at="demand", command=${pyList(["gtk-launch", entry])}${window}),
launch=Launch(at="demand", command=${pyList(["sh", "-c", `cd "$HOME" 2>/dev/null; exec gtk-launch ${shellQuote(entry)}`])}${window}),
)
class ${plan.className}(ComputerAppEnvironment):
"""${entry}, opened on demand for building: it keeps no seed and serves no data operations."""
@@ -364,7 +379,10 @@ export const renderBuildBundle = (
appModuleFile(name),
renderBuildModule(request, entry, plan, install),
),
file(REA_INSTALL_SCRIPT, renderReaInstallScript()),
file(
REA_INSTALL_SCRIPT,
renderReaInstallScript(install.packageFile?.path ?? undefined),
),
...(request.electron === true
? [file(ELECTRON_LAUNCH_SCRIPT, renderElectronLaunchScript(entry))]
: []),
src/domain/desktop/appExport.test.ts
+16 -9
View File
@@ -99,9 +99,11 @@ describe("ownedPaths", () => {
});
describe("launchCommand", () => {
it("keeps commands without home paths and respells home arguments under $HOME", () => {
it("starts in the captured working directory, respelling the home and home arguments under $HOME", () => {
expect(launchCommand(["mousepad"], "/home/ada", "/home/ada")).toEqual([
"mousepad",
"sh",
"-c",
`cd "$HOME" 2>/dev/null; exec 'mousepad'`,
]);
expect(
launchCommand(
@@ -109,7 +111,11 @@ describe("launchCommand", () => {
"/home/ada",
"/home/ada",
),
).toEqual(["sh", "-c", `exec 'mousepad' "$HOME"/'Documents/it'"'"'s.txt'`]);
).toEqual([
"sh",
"-c",
`cd "$HOME" 2>/dev/null; exec 'mousepad' "$HOME"/'Documents/it'"'"'s.txt'`,
]);
});
it("respells the home itself and option values under it", () => {
@@ -122,11 +128,11 @@ describe("launchCommand", () => {
).toEqual([
"sh",
"-c",
`exec 'notes' "$HOME" '--config='"$HOME"/'.notes' '--x=/home/adam'`,
`cd '/' 2>/dev/null; exec 'notes' "$HOME" '--config='"$HOME"/'.notes' '--x=/home/adam'`,
]);
});
it("tries the captured working directory, the home included, only for relative arguments", () => {
it("never leaves the app in the framework's own folder, whatever its arguments", () => {
expect(
launchCommand(
["notes", "today.md"],
@@ -138,12 +144,13 @@ describe("launchCommand", () => {
"-c",
`cd "$HOME"/'project/docs' 2>/dev/null; exec 'notes' 'today.md'`,
]);
expect(
launchCommand(["notes", "today.md"], "/home/ada", "/home/ada"),
).toEqual(["sh", "-c", `cd "$HOME" 2>/dev/null; exec 'notes' 'today.md'`]);
expect(
launchCommand(["notes", "--wrap"], "/home/ada", "/home/ada/project"),
).toEqual(["notes", "--wrap"]);
).toEqual([
"sh",
"-c",
`cd "$HOME"/'project' 2>/dev/null; exec 'notes' '--wrap'`,
]);
expect(launchCommand(["./app"], "/home/ada", "/opt/app")).toEqual([
"sh",
"-c",
src/domain/desktop/appExport.ts
+11 -21
View File
@@ -223,14 +223,18 @@ export const exportedStateFiles = (
/** Text safe inside a Python triple-quoted docstring. */
const docText = (value: string): string => value.replace(/["\\]/gu, "'");
const shellQuote = (value: string): string =>
/** A value as one single-quoted shell word. */
export const shellQuote = (value: string): string =>
`'${value.replaceAll("'", `'"'"'`)}'`;
/**
* The launch command for the computer. Arguments that are or start with a
* path under the captured home, and `<option>=<path>` values that do, are
* respelled under the computer user's `$HOME` through `sh -c`, since the
* declared command is not expanded by a shell.
* The launch command for the computer, through `sh -c` since the declared
* command is not expanded by a shell. It starts in the captured working
* directory, which the framework would otherwise replace with the app's own
* folder, where an open or save dialog would show build files. That
* directory, arguments that are or start with a path under the captured home,
* and `<option>=<path>` values that do, are respelled under the computer
* user's `$HOME`.
*/
export const launchCommand = (
argv: readonly string[],
@@ -254,25 +258,11 @@ export const launchCommand = (
: `${shellQuote(argument.slice(0, equals + 1))}${value}`)
);
};
// The framework launches from the app's folder; a missing working directory must not stop the boot.
// A relative program path such as `./app` needs that directory too.
const relative = argv.some(
(argument, index) =>
argument.length > 0 &&
!argument.startsWith("/") &&
!argument.startsWith("-") &&
(index > 0 || argument.includes("/")),
);
const respelled = argv.map(respell);
if (!relative && respelled.every((argument) => argument === undefined))
return argv;
const command = `exec ${argv.map((argument, index) => respelled[index] ?? shellQuote(argument)).join(" ")}`;
const directory = underHome(workingDirectory) ?? shellQuote(workingDirectory);
return [
"sh",
"-c",
relative ? `cd ${directory} 2>/dev/null; ${command}` : command,
];
// A missing working directory must not stop the boot.
return ["sh", "-c", `cd ${directory} 2>/dev/null; ${command}`];
};
/** A Python string literal; JSON escapes are valid Python escapes. */
src/domain/desktop/appExportInstall.test.ts
+34
View File
@@ -5,6 +5,7 @@ import {
electronLaunchArguments,
exportInstall,
fromUbuntuNobleArchive,
installsFromPackageFile,
repositoryRefusal,
} from "./appExportInstall.js";
import { parseDesktopEntry, tokenizeDesktopExec } from "./desktopEntry.js";
@@ -768,6 +769,39 @@ describe("exportInstall from a package file", () => {
});
});
describe("installsFromPackageFile", () => {
it("is true only for a package the export installs from its file rather than from apt", () => {
const archive = {
sources: [
"http://archive.ubuntu.com/ubuntu noble/universe amd64 Packages",
],
source_origins: ["Ubuntu"],
};
expect(installsFromPackageFile(varicad(), "x64")).toBe(true);
expect(installsFromPackageFile(varicad(archive), "x64")).toBe(false);
expect(
installsFromPackageFile(
varicad({
sources: [TYPORA_SOURCE],
source_origins: ["typora.io"],
apt_repositories: [repository()],
}),
"x64",
),
).toBe(false);
expect(installsFromPackageFile(varicad({ kind: "snap" }), "x64")).toBe(
false,
);
// The export takes the URL exactly when the package needs it.
expect(
exportInstall(varicad(archive), "varicad", "x64", VARICAD_URL).ok,
).toBe(false);
expect(exportInstall(varicad(), "varicad", "x64", VARICAD_URL).ok).toBe(
true,
);
});
});
describe("exportInstall refusals for a package file", () => {
it("refuses a local install without --package-url, saying how to capture and export one", () => {
expect(refusedWith(varicad(), undefined)).toEqual({
src/domain/desktop/appExportInstall.ts
+53 -22
View File
@@ -384,12 +384,53 @@ const packageFileInstall = (
};
/**
* Install from the Ubuntu 24.04 archive by name, or from each vendor
* repository listing the installed version whose signing key the capture
* verified and retained, taking the package from the first one's release
* for the capture host's architecture. A package neither installs, given
* `--package-url`, is downloaded from it and installed from its captured
* package file; anything else is refused with the reason.
* Install a Debian package by name from the Ubuntu 24.04 archive, or from
* each vendor repository listing the installed version whose signing key the
* capture verified and retained, taking the package from the first one's
* release for the capture host's architecture; or why apt cannot.
*/
const aptInstall = (
installed: DesktopPackage,
aptPackage: string,
environment: string,
hostArchitecture: string,
): ReturnType<typeof vendorInstall> =>
// An archive package's declared dependencies are complete; a vendor package's may not be.
fromUbuntuNobleArchive(installed.sources, installed.source_origins)
? ok({
aptPackage,
release: null,
...suppliedLibraries(installed, []),
libraryPackages: [],
sources: [],
skipped: [],
architectures: [...COMPUTER_ARCHITECTURES.values()],
packageFile: null,
})
: vendorInstall(installed, aptPackage, environment, hostArchitecture);
/**
* Whether the export installs the captured package from its package file,
* and so takes `--package-url`: a Debian package apt does not install by
* name. The environment name only names key files, so any name answers.
*/
export const installsFromPackageFile = (
installed: DesktopPackage,
hostArchitecture: string,
): boolean =>
installed.kind === "deb" &&
installed.name !== null &&
!aptInstall(
installed,
installed.name.split(":")[0] ?? installed.name,
"app",
hostArchitecture,
).ok;
/**
* Install by name from apt where it can; a package apt does not install,
* given `--package-url`, is downloaded from it and installed from its
* captured package file; anything else is refused with the reason.
*/
const packageInstall = (
installed: DesktopPackage,
@@ -404,22 +445,12 @@ const packageInstall = (
),
);
const aptPackage = installed.name.split(":")[0] ?? installed.name;
// An archive package's declared dependencies are complete; a vendor package's may not be.
const named = fromUbuntuNobleArchive(
installed.sources,
installed.source_origins,
)
? ok({
aptPackage,
release: null,
...suppliedLibraries(installed, []),
libraryPackages: [],
sources: [],
skipped: [],
architectures: [...COMPUTER_ARCHITECTURES.values()],
packageFile: null,
})
: vendorInstall(installed, aptPackage, environment, hostArchitecture);
const named = aptInstall(
installed,
aptPackage,
environment,
hostArchitecture,
);
if (named.ok)
return packageUrl === undefined
? named
src/domain/desktop/appTaskChecks.fixture.ts
+44
View File
@@ -794,9 +794,52 @@ const TEXT_CASES: readonly (readonly [DesktopTaskCheckInput, boolean])[] = [
},
false,
],
[
{
kind: "text_contains",
file: "Documents/cut.xml",
pointer: "/root/children/2",
value: "later",
},
false,
],
[
{
kind: "text_equals",
file: "Documents/listing.zip",
pointer: "/entries/1/view/root/children/2",
value: "later",
},
false,
],
];
/** Every kind of check against those views, each with the outcome both evaluators must give. */
/** Failures a cut view may have caused, which both evaluators fail alike and REA marks undecided. */
const CUT_VIEW_CASES: readonly (readonly [DesktopTaskCheckInput, boolean])[] = [
[{ kind: "lines_present", file: "Documents/long.txt", lines: ["z"] }, false],
[{ kind: "row_present", file: db, table: "big", where: { id: 9 } }, false],
[
{
kind: "value_equals",
file: "Documents/long.txt",
pointer: "/lines/5500",
value: "z",
},
false,
],
[
{
kind: "value_equals",
file: "Documents/many.zip",
pointer: "/entries/1100/name",
value: "a.txt",
},
false,
],
];
export const CHECK_CASES: readonly (readonly [
DesktopTaskCheckInput,
boolean,
@@ -1045,4 +1088,5 @@ export const CHECK_CASES: readonly (readonly [
...SELECTOR_CASES,
...TOLERANCE_CASES,
...TEXT_CASES,
...CUT_VIEW_CASES,
];
src/domain/desktop/appTaskChecks.test.ts
+233 -38
View File
@@ -1,11 +1,30 @@
import { describe, expect, it } from "vitest";
import { parseDesktopTask } from "./appTask.js";
import { evaluateTaskChecks } from "./appTaskChecks.js";
import {
evaluateTaskChecks,
type TaskCheckOutcome,
type TaskFileView,
} from "./appTaskChecks.js";
import { CHECK_CASES, CHECK_VIEWS } from "./appTaskChecks.fixture.js";
import { textView } from "./stateFormats.js";
const portable = (name: string) => /^[a-z0-9]+(?:-[a-z0-9]+)*$/u.test(name);
/** A value as a description shows it: caller data and what the view holds, as JSON. */
const quoted = (value: unknown) => JSON.stringify(value);
/** An outcome's id, result and undecided mark exactly, and the facts its description reports in any wording. */
const expectOutcome = (
found: TaskCheckOutcome | undefined,
decision: Omit<TaskCheckOutcome, "description">,
facts: readonly string[],
) => {
const { description, ...decided } = found ?? { description: undefined };
expect(decided).toEqual(decision);
for (const fact of facts) expect(description).toContain(fact);
};
const task = (checks: readonly unknown[]) =>
parseDesktopTask(
{ name: "edit-notes", prompt: "Edit my notes.", checks },
@@ -261,6 +280,168 @@ describe("evaluateTaskChecks", () => {
'2 line(s) of Documents/todo.txt match "buy milk", expected at least 3',
);
});
it("marks a failure for want of a readable or whole view as undecided, apart from what a view shows", () => {
const outcomes = evaluateTaskChecks(
[
{
id: "unread",
kind: "lines_absent",
file: "Documents/huge.txt",
lines: ["x"],
},
{
id: "table",
kind: "row_present",
file: ".local/share/notes/notes.db",
table: "broken",
where: { id: 1 },
},
{
id: "cut",
kind: "lines_absent",
file: "Documents/long.txt",
lines: ["z"],
},
{
id: "shown",
kind: "lines_present",
file: "Documents/todo.txt",
lines: ["x"],
},
{
id: "binary",
kind: "lines_present",
file: "Documents/photo.png",
lines: ["x"],
},
] as never,
new Map(Object.entries(CHECK_VIEWS)),
);
expect(
outcomes.map(({ id, passed, undecided }) => [id, passed, undecided]),
).toEqual([
["unread", false, true],
["table", false, true],
["cut", false, true],
["shown", false, undefined],
["binary", false, undefined],
]);
});
});
describe("evaluateTaskChecks on cut views", () => {
const note = (index: number) => `note ${String(index + 1)}`;
const notes = Array.from({ length: 6000 }, (_, index) => note(index));
const rows = (count: number) =>
Array.from({ length: count }, (_, index) => [index + 1, note(index)]);
const views = new Map<string, TaskFileView>([
["notes.txt", { format: "text", view: textView(notes.join("\n")) }],
[
"notes.db",
{
format: "sqlite",
view: {
tables: {
entries: {
columns: ["id", "body"],
primary_key: ["id"],
rows: rows(500),
truncated: true,
},
},
tables_truncated: false,
},
},
],
[
"page.xml",
{
format: "xml",
view: {
root: { name: "doc", attributes: {}, children: ["kept"] },
truncated: true,
},
},
],
[
"notes.zip",
{
format: "zip",
view: { entries: [], entry_count: 1200, truncated: true },
},
],
]);
const outcome = (check: Record<string, unknown>) =>
evaluateTaskChecks([{ id: "c", ...check } as never], views)[0];
it("leaves lines_present undecided on a line past the 5,000 a text view keeps, and decided on one it keeps", () => {
expectOutcome(
outcome({
kind: "lines_present",
file: "notes.txt",
lines: ["note 5500"],
}),
{ id: "c", passed: false, undecided: true },
["notes.txt", quoted("note 5500")],
);
expectOutcome(
outcome({
kind: "lines_present",
file: "notes.txt",
lines: ["note 5000"],
}),
{ id: "c", passed: true },
["notes.txt", quoted("note 5000")],
);
});
it("leaves row_present undecided on a row past those a table's view keeps, and decided on one it keeps", () => {
expectOutcome(
outcome({
kind: "row_present",
file: "notes.db",
table: "entries",
where: { id: 501 },
}),
{ id: "c", passed: false, undecided: true },
["notes.db", "entries", quoted({ id: 501 })],
);
expectOutcome(
outcome({
kind: "row_present",
file: "notes.db",
table: "entries",
where: { id: 500 },
}),
{ id: "c", passed: true },
["notes.db", "entries", quoted({ id: 500 })],
);
});
it("leaves value_equals undecided where its pointer names nothing in a cut view, and decided on a value it keeps", () => {
for (const [file, pointer] of [
["notes.txt", "/lines/5499"],
["notes.db", "/tables/entries/rows/500/1"],
["page.xml", "/root/children/1"],
["notes.zip", "/entries/1100/name"],
] as const)
expectOutcome(
outcome({ kind: "value_equals", file, pointer, value: "note 5500" }),
{ id: "c", passed: false, undecided: true },
[file, pointer],
);
expectOutcome(
outcome({
kind: "value_equals",
file: "notes.txt",
pointer: "/lines/0",
value: "note 2",
}),
{ id: "c", passed: false },
["notes.txt", "/lines/0", quoted("note 1"), quoted("note 2")],
);
});
});
describe("evaluateTaskChecks selectors and tolerance", () => {
@@ -273,29 +454,33 @@ describe("evaluateTaskChecks selectors and tolerance", () => {
)[0];
it("leaves a selector undecided where the array it searches may be cut short, and a position decided", () => {
const cut =
"the view of Documents/listing.zip is truncated, so what it leaves out is unknown";
const listing = "Documents/listing.zip";
const cut = { id: "c", passed: false, undecided: true } as const;
expect(
expectOutcome(
outcome({
kind: "value_equals",
file: "Documents/listing.zip",
file: listing,
pointer: "/entries/[name=a.json]/view/x",
value: 1,
}),
).toEqual({ id: "c", passed: false, description: cut });
expect(
cut,
[listing],
);
expectOutcome(
outcome({
kind: "value_equals",
file: "Documents/listing.zip",
file: listing,
pointer: "/entries/1/view/root/children/[name=p]/attributes/id",
value: "1",
}),
).toEqual({ id: "c", passed: false, description: cut });
cut,
[listing],
);
expect(
outcome({
kind: "value_equals",
file: "Documents/listing.zip",
file: listing,
pointer: "/entries/0/view/x",
value: 1,
})?.passed,
@@ -303,18 +488,15 @@ describe("evaluateTaskChecks selectors and tolerance", () => {
});
it("fails a check into an entry's view that could not be read, where JSON null passes", () => {
expect(
expectOutcome(
outcome({
kind: "value_equals",
pointer: "/entries/[name=huge.brp]/view",
value: null,
}),
).toEqual({
id: "c",
passed: false,
description:
"Documents/plate.FCStd entry huge.brp has no view: larger than the 16 MiB an entry's view reads",
});
{ id: "c", passed: false, undecided: true },
[plate, "huge.brp", "larger than the 16 MiB an entry's view reads"],
);
expect(
outcome({
kind: "value_equals",
@@ -322,31 +504,25 @@ describe("evaluateTaskChecks selectors and tolerance", () => {
value: null,
})?.passed,
).toBe(true);
expect(
expectOutcome(
outcome({
kind: "value_equals",
pointer: "/entries/[name=thumbnails~1Thumbnail.png]/view",
value: null,
}),
).toEqual({
id: "c",
passed: false,
description:
"Documents/plate.FCStd entry thumbnails/Thumbnail.png is binary and has no view",
});
{ id: "c", passed: false },
[plate, "thumbnails/Thumbnail.png", "binary"],
);
});
it("fails a selector that matches several members or, on the way, none, and says which", () => {
expect(
outcome({
kind: "value_equals",
pointer: `${objects}/[name=Object][@type=Part::Box]/attributes/name`,
value: "Plate",
}),
).toMatchObject({
passed: false,
description: `${plate} at ${objects}/[name=Object][@type=Part::Box]/attributes/name: [name=Object][@type=Part::Box] matches 2 members, not one`,
});
const boxes = `${objects}/[name=Object][@type=Part::Box]/attributes/name`;
expectOutcome(
outcome({ kind: "value_equals", pointer: boxes, value: "Plate" }),
{ id: "c", passed: false },
[plate, boxes, "2"],
);
expect(
outcome({
kind: "value_absent",
@@ -372,15 +548,15 @@ describe("evaluateTaskChecks selectors and tolerance", () => {
it("compares numbers and numeral strings within a tolerance, and nothing else", () => {
const radius = `${objects}/[name=Object][@name=Hole]/children/[name=Property][@name=Radius]/children/[name=Float]/attributes/value`;
expect(
expectOutcome(
outcome({
kind: "value_equals",
pointer: radius,
value: 5.1,
tolerance: 0.01,
})?.description,
).toBe(
`${plate} at ${radius} is "5.0000000000000000", not within 0.01 of 5.1`,
}),
{ id: "c", passed: false },
[plate, radius, quoted("5.0000000000000000"), "0.01", "5.1"],
);
expect(
outcome({
@@ -518,6 +694,25 @@ describe("evaluateTaskChecks whole text", () => {
value: "kept",
})?.passed,
).toBe(true);
const missing = outcome({
kind: "text_contains",
file: cut,
pointer: "/root/children/2",
value: "later",
});
expect(missing?.passed).toBe(false);
expect(missing?.undecided).toBe(true);
expect(missing?.description).toBe(
"the view of Documents/cut.xml is truncated, so what it leaves out is unknown",
);
const zipped = outcome({
kind: "text_equals",
file: "Documents/listing.zip",
pointer: "/entries/1/view/root/children/2",
value: "later",
});
expect(zipped?.passed).toBe(false);
expect(zipped?.undecided).toBe(true);
});
});
src/domain/desktop/appTaskChecks.ts
+41 -15
View File
@@ -21,12 +21,17 @@ export interface TaskFileView {
readonly error?: string | null;
}
/** How one check came out against one set of views. */
export interface TaskCheckOutcome {
/**
* How one check came out against one set of views. `undecided` marks a failure
* that shows nothing about the task: its file or table could not be read, or
* what would turn it may lie in what its view left out.
*/
export type TaskCheckOutcome = {
readonly id: string;
readonly passed: boolean;
readonly description: string;
}
readonly undecided?: true;
};
type Verdict = Omit<TaskCheckOutcome, "id">;
type Check<Kind extends DesktopTaskCheck["kind"]> = Extract<
@@ -43,8 +48,16 @@ const shown = (value: JsonValue): string => {
const text = JSON.stringify(value);
return text.length > 200 ? `${text.slice(0, 200)}…` : text;
};
/** A failure for want of a readable or whole view, not for what the view holds. */
const unread = (description: string): Verdict => ({
...fail(description),
undecided: true,
});
const undecided = (file: string): Verdict =>
fail(`the view of ${file} is truncated, so what it leaves out is unknown`);
unread(`the view of ${file} is truncated, so what it leaves out is unknown`);
/** A failure to find what a check looks for, undecided when its view is cut, since it may lie past what the view kept. */
const notFound = (description: string, cut: boolean): Verdict =>
cut ? unread(description) : fail(description);
const isRecord = (
value: JsonValue | null | undefined,
@@ -119,7 +132,7 @@ const viewOf = (
): { view: JsonValue | null; truncated: boolean } | Verdict => {
if (entry === undefined) return { view: null, truncated: false };
if (typeof entry.error === "string")
return fail(`${check.file} has no view: ${entry.error}`);
return unread(`${check.file} has no view: ${entry.error}`);
if (entry.format === null || entry.format === "binary")
return fail(`${check.file} is binary and has no view`);
if (format !== undefined && entry.format !== format)
@@ -164,7 +177,7 @@ const lineVerdict = (
const missing = check.lines.find((line) => !lines.includes(line));
return missing === undefined
? pass(`${file} has the line(s) ${shown(check.lines)}`)
: fail(`${file} has no line ${shown(missing)}`);
: notFound(`${file} has no line ${shown(missing)}`, read.truncated);
}
if (read.truncated) return undecided(file);
if (check.kind === "lines_absent") {
@@ -215,8 +228,10 @@ const unreadOrPartial = (
): Verdict | undefined => {
const zip = format === "zip";
const viewless = zip ? viewlessEntryAt(read.view, check.pointer) : undefined;
if (viewless !== undefined)
return fail(`${check.file} entry ${viewless.name} ${viewless.reason}`);
if (viewless !== undefined) {
const description = `${check.file} entry ${viewless.name} ${viewless.reason}`;
return viewless.unread ? unread(description) : fail(description);
}
// A selector tells one member from several only in an array the view holds whole.
const partial = (array: string) =>
zip ? zipArrayLeavesOut(read.view, array) : read.truncated;
@@ -257,7 +272,13 @@ const valueVerdict = (
check.value !== null;
if (whole && truncated) return undecided(check.file);
if (check.kind === "value_equals") {
if (located.kind === "none") return noValue(at, located);
if (located.kind === "none")
return notFound(
located.reason === undefined
? `${at} has no value`
: `${at}: ${located.reason}`,
truncated,
);
if (check.tolerance !== undefined && typeof check.value === "number")
return nearVerdict(at, located.value, check.value, check.tolerance);
return sameJson(located.value, check.value)
@@ -362,7 +383,12 @@ const textVerdict = (
const problem = zip ? notXmlEntry(check, read.view) : undefined;
if (problem !== undefined) return fail(problem);
if (located.kind === "unresolved") return fail(`${at}: ${located.problem}`);
if (located.kind === "none") return noValue(at, located);
const truncated = zip
? zipLeavesOutAt(read.view, check.pointer)
: read.truncated;
// A child past the kept nodes of a cut view may still hold the text.
if (located.kind === "none")
return truncated ? undecided(check.file) : noValue(at, located);
const found = located.value;
if (
typeof found === "string"
@@ -373,9 +399,6 @@ const textVerdict = (
`${at} is ${shown(found)}, not an element, a text node or an attribute value`,
);
// A text node is whole; an element's text is whole only if its view left nothing out.
const truncated = zip
? zipLeavesOutAt(read.view, check.pointer)
: read.truncated;
if (typeof found !== "string" && truncated) return undecided(check.file);
return comparedText(check, at, elementText(found));
};
@@ -396,7 +419,7 @@ const tableOf = (
return undecided(check.file);
const read = isRecord(table) ? table : {};
if (typeof read.error === "string")
return fail(
return unread(
`${check.file} table ${check.table} could not be read: ${read.error}`,
);
return {
@@ -444,7 +467,10 @@ const rowVerdict = (
if (check.kind === "row_present")
return n > 0
? pass(`${name} has a row with ${shown(check.where)}`)
: fail(`${name} has no row with ${shown(check.where)}`);
: notFound(
`${name} has no row with ${shown(check.where)}`,
table.truncated,
);
if (table.truncated) return undecided(check.file);
if (check.kind === "row_absent")
return n === 0
src/domain/desktop/appTaskPython.ts
+4 -2
View File
@@ -335,13 +335,15 @@ def _whole_text(check, entry):
return False, problem
if kind == "unresolved":
return False, f"{at}: {found}"
if form == "zip":
truncated = _zip_leaves_out(view, check["pointer"])
if kind == "none":
if truncated:
return _undecided(check)
return False, f"{at} has no value" if found is None else f"{at}: {found}"
if not (_text_at(view, check["pointer"]) if isinstance(found, str) else _is_element(found)):
return False, f"{at} is {_shown(found)}, not an element, a text node or an attribute value"
# A text node is whole; an element's text is whole only if its view left nothing out.
if form == "zip":
truncated = _zip_leaves_out(view, check["pointer"])
if not isinstance(found, str) and truncated:
return _undecided(check)
return _compared_text(check, at, _element_text(found))
src/domain/desktop/buildCapture.ts
+19
View File
@@ -0,0 +1,19 @@
import { z } from "zod";
import { RESERVED_CHECKPOINT_NAMES } from "./desktopCapture.js";
/** The checkpoints a build's draft and export read the seed and the end at. */
export const SEED_CHECKPOINT = "seed";
export const END_CHECKPOINT = "end";
/** A checkpoint name REA sends on one request line: not one the capture keeps for its own checkpoints. */
export const buildCheckpointNameSchema = z
.string()
.regex(/^[a-z0-9][a-z0-9._\-]{0,62}$/u)
.refine((name) => !RESERVED_CHECKPOINT_NAMES.has(name), {
message:
"baseline, stop, time-limit and exit are the capture's own checkpoints",
})
.describe(
`Lowercase letters, digits, dot, dash and underscore. Name the seed ${SEED_CHECKPOINT} and the end ${END_CHECKPOINT}: the draft and the export read them.`,
);
src/domain/desktop/buildWitness.test.ts
+386
View File
@@ -0,0 +1,386 @@
import { describe, expect, it } from "vitest";
import {
admitWitnessStage,
appStateViews,
stageVerdict,
type WitnessActionLog,
type WitnessStageRecord,
} from "./buildWitness.js";
import type { ComputerAction } from "./computerActions.js";
type Logged = {
readonly action: ComputerAction;
readonly outcome: "applied" | "refused" | "failed";
};
const action = (
name: ComputerAction,
outcome: Logged["outcome"] = "applied",
): Logged => ({ action: name, outcome });
/** The log the witness reads of these actions, in order: their count and the last applied of each kind. */
const logOf = (actions: readonly Logged[]): WitnessActionLog => ({
total: actions.length,
lastApplied: Object.fromEntries(
actions.flatMap(({ action: name, outcome }, index) =>
outcome === "applied" ? [[name, index]] : [],
),
),
});
const admit = (
stages: readonly WitnessStageRecord[],
stage: WitnessStageRecord["stage"],
actions: readonly Logged[],
) => admitWitnessStage(stages, stage, logOf(actions));
const staged = (
stage: WitnessStageRecord["stage"],
actionsBefore: number,
): WitnessStageRecord => ({
stage,
actions_before: actionsBefore,
checked_at: "2026-11-05T00:00:00.000Z",
expectation: stage === "after_edit" ? "all_pass" : "some_fail",
held: true,
results: [],
});
describe("build witness rules", () => {
it("expects a failing check on the seed, after the near miss and after reset, and every check passing after the edit", () => {
const failing = [
{ id: "a", passed: true, description: "" },
{ id: "b", passed: false, description: "" },
];
const passing = [{ id: "a", passed: true, description: "" }];
expect(stageVerdict("seed", failing)).toBe("held");
expect(stageVerdict("seed", passing)).toBe("not_held");
expect(stageVerdict("after_edit", passing)).toBe("held");
expect(stageVerdict("after_edit", failing)).toBe("not_held");
expect(stageVerdict("after_edit", [])).toBe("not_held");
expect(stageVerdict("near_miss", failing)).toBe("held");
expect(stageVerdict("near_miss", passing)).toBe("not_held");
const seed = [{ ...staged("seed", 0), results: failing }];
expect(stageVerdict("after_reset", failing, seed)).toBe("held");
expect(stageVerdict("after_reset", passing, seed)).toBe("not_held");
});
it("holds after reset only when each check comes out as it did on the seed", () => {
const outcome = (id: string, passed: boolean) => ({
id,
passed,
description: "",
});
const seed = [
{
...staged("seed", 0),
results: [outcome("eggs", false), outcome("no-egg", true)],
},
];
expect(
stageVerdict(
"after_reset",
[outcome("eggs", false), outcome("no-egg", true)],
seed,
),
).toBe("held");
// A reset that left the near miss fails a check, but not as the seed did.
expect(
stageVerdict(
"after_reset",
[outcome("eggs", false), outcome("no-egg", false)],
seed,
),
).toBe("not_held");
// A view unread after the reset cannot show the difference.
expect(
stageVerdict(
"after_reset",
[
outcome("eggs", false),
{ ...outcome("no-egg", false), undecided: true as const },
],
seed,
),
).toBe("undecided");
});
it("takes no unreadable or partial view as a failure, leaving a stage it alone would settle undecided", () => {
const unread = {
id: "u",
passed: false,
description: "",
undecided: true as const,
};
const passing = { id: "a", passed: true, description: "" };
const failing = { id: "b", passed: false, description: "" };
const seed = [
{ ...staged("seed", 0), results: [passing, unread, failing] },
];
for (const stage of ["seed", "near_miss"] as const) {
expect(stageVerdict(stage, [passing, unread], seed)).toBe("undecided");
expect(stageVerdict(stage, [unread, failing], seed)).toBe("held");
}
// after_reset must read every check. An unread one does not match the seed.
expect(stageVerdict("after_reset", [passing, unread], seed)).toBe(
"undecided",
);
expect(stageVerdict("after_reset", [unread, failing], seed)).toBe(
"undecided",
);
expect(stageVerdict("after_edit", [passing, unread])).toBe("undecided");
expect(stageVerdict("after_edit", [unread, failing])).toBe("not_held");
});
it("takes the stages in order, each after the actions it follows", () => {
expect(admit([], "seed", [])).toEqual({ ok: true, value: 0 });
expect(admit([], "after_edit", [])).toMatchObject({
ok: false,
error: expect.stringContaining("seed"),
});
const seed = [staged("seed", 1)];
const log = [action("screenshot")];
// Only looking and waiting since the seed is no edit.
expect(
admit(seed, "after_edit", [
...log,
action("screenshot"),
action("wait"),
action("type", "refused"),
]),
).toMatchObject({
ok: false,
error: expect.stringContaining("act on the app"),
});
expect(admit(seed, "after_edit", [...log, action("type")])).toEqual({
ok: true,
value: 2,
});
expect(admit(seed, "seed", log)).toMatchObject({
ok: false,
error: expect.stringContaining("after_edit"),
});
const edited = [...seed, staged("after_edit", 2)];
const nearMiss = [...edited, staged("near_miss", 4)];
const acted = [action("screenshot"), action("type"), action("key")];
expect(admit(edited, "near_miss", [...acted, action("key")])).toMatchObject(
{ ok: true },
);
// A reset must be the last change before the check after it.
expect(
admit(nearMiss, "after_reset", [...acted, action("key"), action("type")]),
).toMatchObject({
ok: false,
error: expect.stringContaining("reset"),
});
expect(
admit(nearMiss, "after_reset", [
...acted,
action("key"),
action("reset"),
action("type"),
]),
).toMatchObject({ ok: false });
expect(
admit(nearMiss, "after_reset", [
...acted,
action("key"),
action("reset"),
action("screenshot"),
]),
).toMatchObject({ ok: true });
expect(
admit([...nearMiss, staged("after_reset", 6)], "after_reset", []),
).toMatchObject({ ok: false, error: expect.stringContaining("ended") });
});
});
describe("build witness reset compared with an unreadable seed", () => {
it("leaves after_reset undecided when an unreadable seed check later passes", () => {
const unread = {
id: "note",
passed: false,
description: "",
undecided: true as const,
};
const failing = { id: "eggs", passed: false, description: "" };
const seed = [{ ...staged("seed", 0), results: [unread, failing] }];
const passed = { id: "note", passed: true, description: "" };
const eggsPass = { id: "eggs", passed: true, description: "" };
expect(stageVerdict("after_reset", [passed, failing], seed)).toBe(
"undecided",
);
expect(stageVerdict("after_reset", [unread, failing], seed)).toBe(
"undecided",
);
expect(stageVerdict("after_reset", [passed, eggsPass], seed)).toBe(
"not_held",
);
});
});
describe("build witness unread reset", () => {
it("stays undecided when a decided seed failure is unreadable after the reset and another check still fails", () => {
const seedFailure = { id: "a", passed: false, description: "" };
const stillFailing = { id: "b", passed: false, description: "" };
const unreadAfter = { ...seedFailure, undecided: true as const };
const restored = { id: "a", passed: true, description: "" };
const unreadOther = { ...stillFailing, undecided: true as const };
const seed = [
{ ...staged("seed", 0), results: [seedFailure, stillFailing] },
];
expect(stageVerdict("after_reset", [unreadAfter, stillFailing], seed)).toBe(
"undecided",
);
expect(stageVerdict("after_reset", [restored, unreadOther], seed)).toBe(
"not_held",
);
});
it("treats a reset whose reply was lost as blocking the near miss until a later input", () => {
const edited = [staged("seed", 0), staged("after_edit", 1)];
const log = {
total: 3,
lastApplied: { type: 1 },
lastUnsettled: { reset: 2 },
};
expect(admitWitnessStage(edited, "near_miss", log)).toMatchObject({
ok: false,
error: expect.stringContaining("lost its reply"),
});
expect(
admitWitnessStage(edited, "near_miss", {
total: 4,
lastApplied: { type: 3 },
lastUnsettled: { reset: 2 },
}),
).toEqual({ ok: true, value: 4 });
const nearMiss = [...edited, staged("near_miss", 2)];
expect(admitWitnessStage(nearMiss, "after_reset", log)).toMatchObject({
ok: false,
error: expect.stringContaining("reply was lost"),
});
});
});
describe("build witness edits", () => {
it("never takes a reset as the edit or the near miss", () => {
const seed = [staged("seed", 0)];
// An edit made only of a reset, or undone by one, is no edit.
expect(admit(seed, "after_edit", [action("reset")])).toMatchObject({
ok: false,
error: expect.stringContaining("act on the app"),
});
expect(
admit(seed, "after_edit", [action("type"), action("reset")]),
).toMatchObject({ ok: false });
const edited = [...seed, staged("after_edit", 1)];
const log = [action("type")];
// Reset after a passing edit, check near_miss on the seed: refused.
expect(admit(edited, "near_miss", [...log, action("reset")])).toMatchObject(
{ ok: false, error: expect.stringContaining("reset") },
);
expect(
admit(edited, "near_miss", [
...log,
action("key"),
action("reset"),
action("screenshot"),
]),
).toMatchObject({ ok: false });
// A near miss made after a reset is still a change of the app.
expect(
admit(edited, "near_miss", [...log, action("reset"), action("type")]),
).toEqual({ ok: true, value: 3 });
});
});
describe("build witness over a long session", () => {
it("reads the count and the last applied actions, so it holds after the log drops its oldest entries", () => {
const seed = [staged("seed", 500)];
expect(
admitWitnessStage(seed, "after_edit", {
total: 1000,
lastApplied: { type: 999, reset: 10 },
}),
).toEqual({ ok: true, value: 1000 });
expect(
admitWitnessStage(seed, "after_edit", {
total: 1000,
lastApplied: { type: 400 },
}),
).toMatchObject({
ok: false,
error: expect.stringContaining("act on the app"),
});
const nearMiss = [
...seed,
staged("after_edit", 600),
staged("near_miss", 800),
];
expect(
admitWitnessStage(nearMiss, "after_reset", {
total: 1000,
lastApplied: { key: 700, reset: 900 },
}),
).toEqual({ ok: true, value: 1000 });
expect(
admitWitnessStage(nearMiss, "after_reset", {
total: 1000,
lastApplied: { key: 950, reset: 900 },
}),
).toMatchObject({ ok: false, error: expect.stringContaining("reset") });
});
});
describe("build witness state views", () => {
it("reads the app's data/get state into the views the checks read, keeping each file's error", () => {
expect(
appStateViews([
{ kind: "file", file: { bytes: "eA==" } },
{
kind: "data",
data: {
state: {
"Documents/todo.txt": {
format: "text",
view: { lines: ["milk"], truncated: false },
truncated: false,
},
"Documents/big.bin": {
format: null,
view: null,
error: "larger than 64 MiB, the most the view reads",
},
},
},
},
]),
).toEqual({
ok: true,
value: new Map([
[
"Documents/todo.txt",
{
format: "text",
view: { lines: ["milk"], truncated: false },
error: null,
},
],
[
"Documents/big.bin",
{
format: null,
view: null,
error: "larger than 64 MiB, the most the view reads",
},
],
]),
});
expect(appStateViews([{ kind: "file" }])).toMatchObject({
ok: false,
error: expect.stringContaining("no state views"),
});
});
});
src/domain/desktop/buildWitness.ts
+249
View File
@@ -0,0 +1,249 @@
import { z } from "zod";
import { jsonValueSchema } from "../jsonValue.js";
import { err, ok, type Result } from "../result.js";
import type { TaskCheckOutcome, TaskFileView } from "./appTaskChecks.js";
import type { ComputerAction } from "./computerActions.js";
/**
* The witness of an exported task, in order: the seed must fail a check, the
* edit pass every one, a near miss fail one again, and the seed that
* `data/reset` restores must give each check its seed result.
*/
export const WITNESS_STAGES = [
"seed",
"after_edit",
"near_miss",
"after_reset",
] as const;
export type WitnessStage = (typeof WITNESS_STAGES)[number];
/** What a stage's checks must show for the witness to hold. */
export type WitnessExpectation = "some_fail" | "all_pass";
/**
* What the witness reads of a computer's action log: how many actions it
* logged, and the index of the last applied action of each kind, which hold
* however many entries the log has dropped.
*/
export interface WitnessActionLog {
readonly total: number;
readonly lastApplied: Readonly<Partial<Record<ComputerAction, number>>>;
/**
* The index of the last reset whose reply was lost. It is not applied, and
* it is not absent: a near miss cannot use the input before it.
*/
readonly lastUnsettled?: Readonly<Partial<Record<ComputerAction, number>>>;
}
/** One stage's checks, with how many logged actions came before it. */
export type WitnessStageRecord = {
readonly stage: WitnessStage;
readonly actions_before: number;
readonly checked_at: string;
readonly expectation: WitnessExpectation;
readonly held: boolean;
readonly results: TaskCheckOutcome[];
/** Why a stage did not hold when its results alone do not say, and what to do. */
readonly remediation?: string;
};
const EXPECTATIONS: Readonly<Record<WitnessStage, WitnessExpectation>> = {
seed: "some_fail",
after_edit: "all_pass",
near_miss: "some_fail",
after_reset: "some_fail",
};
/** Actions that change the app through its own interface; a reset restores the seed instead. */
const INPUT = ["click", "drag", "scroll", "type", "key"] as const;
export const stageExpectation = (stage: WitnessStage): WitnessExpectation =>
EXPECTATIONS[stage];
/** What a stage's results show of it: that it held, that it did not, or, when only undecided checks could tell, nothing. */
export type StageVerdict = "held" | "not_held" | "undecided";
type ComparedOutcome = Pick<TaskCheckOutcome, "id" | "passed" | "undecided">;
const seedResults = (
stages: readonly WitnessStageRecord[],
): Map<string, Pick<TaskCheckOutcome, "passed" | "undecided">> =>
new Map(
stages
.find((filed) => filed.stage === "seed")
?.results.map((result) => [result.id, result]),
);
/**
* The checks whose decided result after a reset differs from a decided seed
* result. A seed result that was unreadable is not a failure to compare
* against; a reset that restores the seed leaves none.
*/
export const unrestoredChecks = <Outcome extends ComparedOutcome>(
stages: readonly WitnessStageRecord[],
results: readonly Outcome[],
): Outcome[] => {
const seed = seedResults(stages);
return results.filter((result) => {
const filed = seed.get(result.id);
if (filed === undefined) return true;
if (filed.undecided === true) return false;
return filed.passed !== result.passed;
});
};
/**
* Checks whose seed comparison is unknown: either the filed seed result or the
* result after the reset was not read. An unreadable `false` is not the same
* result as a decided `false`.
*/
export const unreadResetChecks = <Outcome extends ComparedOutcome>(
stages: readonly WitnessStageRecord[],
results: readonly Outcome[],
): Outcome[] => {
const seed = seedResults(stages);
return results.filter((result) => {
const filed = seed.get(result.id);
if (filed === undefined) return false;
return filed.undecided === true || result.undecided === true;
});
};
/**
* Whether a stage's results show what the witness needs of it, given the
* stages filed before it. Only a check that failed on what its view shows
* counts as a failure; one that failed for want of a readable or whole view
* leaves the stage undecided when no such failure settles it. After a reset,
* each check must also come out as it did on the seed.
*/
export const stageVerdict = (
stage: WitnessStage,
results: readonly Pick<TaskCheckOutcome, "id" | "passed" | "undecided">[],
stages: readonly WitnessStageRecord[] = [],
): StageVerdict => {
if (stage === "after_reset") {
const unrestored = unrestoredChecks(stages, results);
if (unrestored.some((result) => result.undecided !== true))
return "not_held";
if (unrestored.length > 0 || unreadResetChecks(stages, results).length > 0)
return "undecided";
}
const failed = results.filter((result) => !result.passed);
const shown = failed.some((result) => result.undecided !== true);
if (EXPECTATIONS[stage] === "some_fail")
return shown ? "held" : failed.length > 0 ? "undecided" : "not_held";
if (shown || results.length === 0) return "not_held";
return failed.length > 0 ? "undecided" : "held";
};
/** Why an edit or a near miss is not one: no input since the stage before, or a reset after the last input. */
const notAnEdit = (
stage: WitnessStage,
since: string,
latest: {
readonly input: number;
readonly reset: number;
readonly unsettledReset: number;
readonly from: number;
},
): string | undefined => {
if (latest.input < latest.from)
return `Before checking ${stage}, act on the app with act_on_build_computer: no click, drag, scroll, type or key was applied since ${since}, and a reset does not count.`;
if (
latest.unsettledReset > latest.reset &&
latest.unsettledReset > latest.input
)
return `A reset since ${since} lost its reply, so whether it restored the seed is unknown. Reset again, or act after a reset whose reply you have, before checking ${stage}.`;
if (latest.reset > latest.input)
return `A reset since ${since} put the seed back; act on the app after it before checking ${stage}.`;
return undefined;
};
/**
* Whether `stage` may be checked now, given the stages already checked and
* the action log; its value is where this stage's actions end. The edit and
* the near miss each need an applied click, drag, scroll, type or key since
* the stage before, with no reset after the last of them; the check after
* reset needs a reset since the stage before as the last change.
*/
export const admitWitnessStage = (
stages: readonly WitnessStageRecord[],
stage: WitnessStage,
log: WitnessActionLog,
): Result<number, string> => {
const expected = WITNESS_STAGES[stages.length];
if (expected === undefined)
return err(
"This witness has ended; start a fresh computer from the bundle to witness again.",
);
if (stage !== expected)
return err(
`The witness checks ${WITNESS_STAGES.join(", ")} in that order; the next stage is ${expected}.`,
);
const latest = {
from: stages.at(-1)?.actions_before ?? 0,
input: Math.max(-1, ...INPUT.map((kind) => log.lastApplied[kind] ?? -1)),
reset: log.lastApplied.reset ?? -1,
unsettledReset: log.lastUnsettled?.reset ?? -1,
};
const edit =
stage === "after_edit" || stage === "near_miss"
? notAnEdit(stage, stages.at(-1)?.stage ?? "the start", latest)
: undefined;
if (edit !== undefined) return err(edit);
if (
stage === "after_reset" &&
latest.unsettledReset > latest.reset &&
latest.unsettledReset >= latest.from
)
return err(
"The last reset's reply was lost, so whether it restored the seed is unknown. Reset again, then check after_reset once that reset is applied.",
);
if (
stage === "after_reset" &&
(latest.reset < latest.from || latest.reset < latest.input)
)
return err(
"Before checking after_reset, reset the app with act_on_build_computer action reset, and change nothing after it.",
);
return ok(log.total);
};
const fileViewSchema = z.looseObject({
format: z.string().nullable(),
view: jsonValueSchema.nullable(),
error: z.string().nullish(),
});
const statePartSchema = z.looseObject({
kind: z.literal("data"),
data: z.looseObject({ state: z.record(z.string(), fileViewSchema) }),
});
/**
* The typed views an exported app's `data/get` returns beside its files,
* keyed by home path: the shape the task checks read.
*/
export const appStateViews = (
parts: readonly unknown[],
): Result<ReadonlyMap<string, TaskFileView>, string> => {
for (const part of parts) {
const parsed = statePartSchema.safeParse(part);
if (parsed.success)
return ok(
new Map(
Object.entries(parsed.data.data.state).map(([path, entry]) => [
path,
{
format: entry.format,
view: entry.view,
error: entry.error ?? null,
},
]),
),
);
}
return err(
"The app's data/get returned no state views; witness a bundle exported by REA, whose app serves them.",
);
};
src/domain/desktop/computerActions.test.ts
+11
View File
@@ -67,4 +67,15 @@ describe("computer actions", () => {
error: { field: "x", accepted: ["text", "screenshot"] },
});
});
it("resets the app's data rather than calling a driver tool, taking only a screenshot afterwards", () => {
expect(driverCall("reset", { screenshot: true })).toEqual({
ok: true,
value: { tool: "data/reset", arguments: { screenshot: true } },
});
expect(driverCall("reset", { keys: ["Enter"] })).toEqual({
ok: false,
error: { field: "keys", accepted: ["screenshot"] },
});
});
});
src/domain/desktop/computerActions.ts
+15 -4
View File
@@ -4,7 +4,11 @@ import { fixedTupleSchema } from "../fixedTupleSchema.js";
import type { JsonValue } from "../jsonValue.js";
import { err, ok, type Result } from "../result.js";
/** Actions REA forwards to an agentenv computer's driver, each as its `computer_<action>` tool. */
/**
* Actions on a build computer: each but `reset` is forwarded to the driver
* as its `computer_<action>` tool; `reset` restores the app's data to its
* seed with `data/reset`.
*/
export const COMPUTER_ACTIONS = [
"screenshot",
"click",
@@ -13,6 +17,7 @@ export const COMPUTER_ACTIONS = [
"type",
"key",
"wait",
"reset",
] as const;
export type ComputerAction = (typeof COMPUTER_ACTIONS)[number];
@@ -42,6 +47,7 @@ const ACTION_FIELDS: Readonly<Record<ComputerAction, readonly string[]>> = {
type: ["text", "screenshot"],
key: ["keys", "repeat", "screenshot"],
wait: ["ms", "screenshot"],
reset: ["screenshot"],
};
const coordinate = z
@@ -66,7 +72,9 @@ export const computerActionInputSchema = z.strictObject({
computer_id: z.uuid().describe("The id start_build_computer returned."),
action: z
.enum(COMPUTER_ACTIONS)
.describe("The driver's computer_<action> tool to call."),
.describe(
"The driver's computer_<action> tool to call, or reset to restore the app's data to its seed.",
),
x: coordinate.optional(),
y: coordinate.optional(),
frame_id: z
@@ -171,7 +179,7 @@ export const w3cKeys = (keys: readonly string[]): string[] =>
ONE_CHARACTER.test(key) ? [key] : key.split(/\+(?!$)/u).map(w3cKey),
);
/** The driver tool and arguments for an action, or the first field the action does not take. */
/** The tool and arguments for an action, `data/reset` for a reset, or the first field the action does not take. */
export const driverCall = (
action: ComputerAction,
fields: Readonly<Record<string, JsonValue | undefined>>,
@@ -193,5 +201,8 @@ export const driverCall = (
: value,
]),
);
return ok({ tool: `computer_${action}`, arguments: translated });
return ok({
tool: action === "reset" ? "data/reset" : `computer_${action}`,
arguments: translated,
});
};
src/domain/desktop/desktopCapture.ts
+57
View File
@@ -48,6 +48,63 @@ export const desktopCheckpointSchema = z.strictObject({
});
export type DesktopCheckpoint = z.infer<typeof desktopCheckpointSchema>;
/**
* What a running capture reports beside its progress text, for a driver to
* wait on: the window opening, a checkpoint once it has been taken, with its
* counts, and a requested name it refused.
*/
export const desktopCaptureEventSchema = z.discriminatedUnion("kind", [
z.strictObject({ kind: z.literal("window_open") }),
z.strictObject({
kind: z.literal("checkpoint"),
name: z.string().min(1),
trigger: desktopCheckpointSchema.shape.trigger,
at_ms: milliseconds,
entries: z.int().min(0),
changes: z.int().min(0),
retained_files: z.int().min(0),
retained_bytes: z.int().min(0),
}),
z.strictObject({
kind: z.literal("checkpoint_refused"),
name: z.string(),
reason: z.string().min(1),
}),
]);
export type DesktopCaptureEvent = z.infer<typeof desktopCaptureEventSchema>;
/** One progress line `capture-desktop-application` writes to stderr. */
export const desktopCaptureProgressSchema = z.strictObject({
rea_progress: z.strictObject({
phase: z.literal("desktop_capture"),
message: z.string(),
event: desktopCaptureEventSchema.optional(),
}),
});
export type DesktopCaptureProgress = z.infer<
typeof desktopCaptureProgressSchema
>;
/** The capture event in one progress line, when that line carries one. */
export const captureProgressEvent = (
line: string,
): DesktopCaptureEvent | undefined =>
desktopCaptureProgressSchema.parse(JSON.parse(line)).rea_progress.event;
/** The acknowledgement of a taken checkpoint, from what it recorded. */
export const checkpointTaken = (
checkpoint: DesktopCheckpoint,
): Extract<DesktopCaptureEvent, { kind: "checkpoint" }> => ({
kind: "checkpoint",
name: checkpoint.name,
trigger: checkpoint.trigger,
at_ms: checkpoint.at_ms,
entries: checkpoint.entries,
changes: checkpoint.changes.length,
retained_files: checkpoint.retained.length,
retained_bytes: checkpoint.retained.reduce((sum, file) => sum + file.size, 0),
});
const desktopEntrySchema = z.strictObject({
id: z.string().nullable(),
path: z.string().min(1),
src/domain/desktop/desktopState.ts
+1
View File
@@ -78,6 +78,7 @@ export const desktopStatePathSchema = z.strictObject({
"selected_path",
"dconf_application_keys",
"desktop_service",
"desktop_trash",
]),
),
/** The latest readable version's format. */
src/domain/desktop/stateRoles.test.ts
+34
View File
@@ -33,6 +33,17 @@ describe("stateRole", () => {
"shared_desktop",
);
});
it("keeps a file in the desktop Trash shared, whatever else names it", () => {
expect(
stateRole([
"opened_by_application",
"named_in_window_title",
"selected_path",
"desktop_trash",
]),
).toBe("shared_desktop");
});
});
describe("stateReasons", () => {
@@ -56,6 +67,29 @@ describe("stateReasons", () => {
expect(reasons("/home/ada/Documents/day.txt")).toEqual([]);
expect(reasons("/home/ada/Documents/today.tx")).toEqual([]);
});
it("marks the desktop Trash, where a file the user deleted keeps its name", () => {
const facts: DesktopRoleFacts = {
home: "/home/ada",
stateDirectories: [],
stateTails: [],
selectedRoots: [],
titles: ["poster.svg - Inkscape"],
openedFiles: new Set(),
};
const reasons = (path: string) => stateReasons(facts, path, [], false);
expect(reasons("/home/ada/.local/share/Trash/files/poster.svg")).toEqual([
"named_in_window_title",
"desktop_trash",
]);
expect(
reasons("/home/ada/.local/share/Trash/info/poster.svg.trashinfo"),
).toEqual(["desktop_trash"]);
expect(reasons("/home/ada/.local/share/Trashcan/poster.svg")).toEqual([
"named_in_window_title",
]);
});
});
describe("stateReasons for state directories", () => {
src/domain/desktop/stateRoles.ts
+12 -4
View File
@@ -11,7 +11,8 @@ export type DesktopStateReason =
| "named_in_window_title"
| "selected_path"
| "dconf_application_keys"
| "desktop_service";
| "desktop_service"
| "desktop_trash";
/** How a later step should treat a path. */
export type DesktopStateRole =
@@ -30,11 +31,13 @@ const SHARED_DESKTOP_PATHS = [
".config/pulse",
".local/share/recently-used.xbel",
".local/share/gvfs-metadata",
".local/share/Trash",
".Xauthority",
".ICEauthority",
];
/** The freedesktop Trash in the home, where files the user deleted wait under their own names. */
const DESKTOP_TRASH = ".local/share/Trash";
/**
* Lowercase names that identify the application in dconf keys. The desktop
* entry's display name is left out: generic names such as "Settings" would
@@ -203,16 +206,21 @@ export const stateReasons = (
if (dconfApplicationKeys) reasons.push("dconf_application_keys");
if (relative !== null && isWithinAny(relative, SHARED_DESKTOP_PATHS))
reasons.push("desktop_service");
if (relative !== null && isWithinAny(relative, [DESKTOP_TRASH]))
reasons.push("desktop_trash");
return reasons;
};
/**
* The role that follows from a path's reasons, strongest first. A file the
* application holds open is a document when a title or the caller names it.
* The role that follows from a path's reasons, strongest first. A file in the
* desktop Trash is shared whatever else names it, since the user deleted it.
* A file the application holds open is a document when a title or the caller
* names it.
*/
export const stateRole = (
reasons: readonly DesktopStateReason[],
): DesktopStateRole => {
if (reasons.includes("desktop_trash")) return "shared_desktop";
const shared = reasons.includes("desktop_service");
if (
reasons.includes("dconf_application_keys") ||
src/domain/desktop/zipView.ts
+12 -3
View File
@@ -161,18 +161,27 @@ export const entryWithoutView = (
return VIEW_FORMATS.has(entry.format ?? null) ? undefined : "has no view";
};
/** The entry whose view a pointer goes into when it has none, so its `view` of `null` is no JSON value, and why. */
/**
* The entry whose view a pointer goes into when it has none, so its `view` of
* `null` is no JSON value, and why; `unread` when reading its view failed.
*/
export const viewlessEntryAt = (
view: JsonValue | null,
pointer: string,
): { readonly name: string; readonly reason: string } | undefined => {
):
| { readonly name: string; readonly reason: string; readonly unread: boolean }
| undefined => {
const entry = zipEntryAt(view, pointer);
if (typeof entry !== "object" || pointer.split("/")[3] !== "view")
return undefined;
const reason = entryWithoutView(entry);
return reason === undefined
? undefined
: { name: typeof entry.name === "string" ? entry.name : "", reason };
: {
name: typeof entry.name === "string" ? entry.name : "",
reason,
unread: typeof entry.view_error === "string",
};
};
/**
src/server/registerDesktopBuildTools.ts
+131 -51
View File
@@ -9,6 +9,7 @@ import type {
DesktopBuildComputers,
ForwardedResult,
} from "../application/desktop/DesktopBuildComputers.js";
import { DesktopBuildSteps } from "../application/desktop/DesktopBuildSteps.js";
import { toolContract } from "../contracts/toolContracts.js";
import type { ToolContract } from "../contracts/toolContractTypes.js";
import type { AnalysisError } from "../domain/analysisErrorBase.js";
@@ -16,6 +17,7 @@ import { analysisInputErrorFromIssues } from "../domain/inputIssueProjection.js"
import type { JsonValue } from "../domain/jsonValue.js";
import { err, ok, type Result } from "../domain/result.js";
import type { EvidenceMcpServer } from "./EvidenceMcpServer.js";
import { mcpProgressReporter } from "./mcpProgress.js";
import { logToolExecution } from "./toolLogging.js";
import { applicationToolRegistrationOptions } from "./toolRegistrationOptions.js";
@@ -34,66 +36,144 @@ const admit = <Schema extends z.ZodType>(
);
};
/** Register the build computer session tools on the REA MCP server. */
export const registerDesktopBuildTools = (
server: EvidenceMcpServer,
computers: Pick<DesktopBuildComputers, "start" | "act" | "close">,
logger: Logger,
const HEARTBEAT_MS = 5_000;
/** Report progress every few seconds while a long build step works, so a client's request timeout can reset. */
const withHeartbeat = async <Value>(
context: ServerContext,
phase: string,
run: () => Promise<Value>,
): Promise<Value> => {
const progress = mcpProgressReporter(context);
const startedAt = Date.now();
const timer = setInterval(() => {
void progress
.report({
phase,
completed: 0,
total: null,
message: `working (${String(Math.round((Date.now() - startedAt) / 1000))} s)`,
})
.catch(() => undefined);
}, HEARTBEAT_MS);
try {
return await run();
} finally {
clearInterval(timer);
}
};
/** Registration of one build tool: the server, its logger and the tool's contract. */
interface Registration<Schema extends z.ZodType> {
readonly server: EvidenceMcpServer;
readonly logger: Logger;
readonly contract: ToolContract & {
readonly name: string;
readonly inputSchema: Schema;
};
}
/** Register a tool whose result carries a forwarded driver answer, such as a screenshot, after REA's JSON. */
const registerForwarded = <Schema extends z.ZodType>(
{ server, logger, contract }: Registration<Schema>,
run: (
input: z.output<Schema>,
context: ServerContext,
) => Promise<Result<ForwardedResult<JsonValue>, AnalysisError>>,
): void => {
const forward = (
contract: ToolContract,
result: Result<ForwardedResult<JsonValue>, AnalysisError>,
): CallToolResult =>
result.ok
? server.delivery.toForwardedToolResult(
ok(result.value.value),
contract,
result.value.content,
)
: server.delivery.toCallToolResult(result, contract);
const start = toolContract("start_build_computer");
server.registerTool(
start.name,
applicationToolRegistrationOptions(start),
async (raw: unknown, context: ServerContext) => {
const input = admit(start, raw);
if (!input.ok) return server.delivery.toCallToolResult(input, start);
return forward(
start,
await logToolExecution(logger, start.name, () =>
computers.start(input.value, context.mcpReq.signal),
),
contract.name,
applicationToolRegistrationOptions(contract),
async (raw: unknown, context: ServerContext): Promise<CallToolResult> => {
const input = admit(contract, raw);
if (!input.ok) return server.delivery.toCallToolResult(input, contract);
const result = await logToolExecution(logger, contract.name, () =>
run(input.value, context),
);
return result.ok
? server.delivery.toForwardedToolResult(
ok(result.value.value),
contract,
result.value.content,
)
: server.delivery.toCallToolResult(result, contract);
},
);
const act = toolContract("act_on_build_computer");
};
/** Register a tool with a plain JSON result. */
const registerPlain = <Schema extends z.ZodType>(
{ server, logger, contract }: Registration<Schema>,
run: (
input: z.output<Schema>,
context: ServerContext,
) => Promise<Result<JsonValue, AnalysisError>>,
): void => {
server.registerTool(
act.name,
applicationToolRegistrationOptions(act),
async (raw: unknown, context: ServerContext) => {
const input = admit(act, raw);
if (!input.ok) return server.delivery.toCallToolResult(input, act);
return forward(
act,
await logToolExecution(logger, act.name, () =>
computers.act(input.value, context.mcpReq.signal),
),
);
},
);
const close = toolContract("close_build_computer");
server.registerTool(
close.name,
applicationToolRegistrationOptions(close),
async (raw: unknown) => {
const input = admit(close, raw);
if (!input.ok) return server.delivery.toCallToolResult(input, close);
contract.name,
applicationToolRegistrationOptions(contract),
async (raw: unknown, context: ServerContext): Promise<CallToolResult> => {
const input = admit(contract, raw);
if (!input.ok) return server.delivery.toCallToolResult(input, contract);
return server.delivery.toCallToolResult(
await logToolExecution(logger, close.name, () =>
computers.close(input.value.computer_id),
await logToolExecution(logger, contract.name, () =>
run(input.value, context),
),
close,
contract,
);
},
);
};
/** Register the build computer session tools and the build steps on them on the REA MCP server. */
export const registerDesktopBuildTools = (
server: EvidenceMcpServer,
computers: Pick<DesktopBuildComputers, "start" | "act" | "close" | "steps">,
logger: Logger,
): void => {
const tool = <Name extends Parameters<typeof toolContract>[0]>(
name: Name,
) => ({
server,
logger,
contract: toolContract(name),
});
const steps = new DesktopBuildSteps(computers);
registerForwarded(tool("start_build_computer"), (input, context) =>
computers.start(input, context.mcpReq.signal, mcpProgressReporter(context)),
);
// An action can wait for the call before it on its computer, and a reset for the relaunched app, which can take minutes.
registerForwarded(tool("act_on_build_computer"), (input, context) =>
withHeartbeat(context, "act_on_build_computer", () =>
computers.act(input, context.mcpReq.signal),
),
);
registerForwarded(tool("start_build_capture"), (input, context) =>
withHeartbeat(context, "start_build_capture", () =>
steps.startCapture(input, context.mcpReq.signal),
),
);
registerPlain(tool("mark_build_checkpoint"), (input, context) =>
withHeartbeat(context, "mark_build_checkpoint", () =>
steps.markCheckpoint(input, context.mcpReq.signal),
),
);
registerPlain(tool("stop_build_capture"), (input, context) =>
withHeartbeat(context, "stop_build_capture", () =>
steps.stopCapture(input, context.mcpReq.signal),
),
);
registerPlain(tool("export_build_environment"), (input, context) =>
withHeartbeat(context, "export_build_environment", () =>
steps.exportEnvironment(input, context.mcpReq.signal),
),
);
registerPlain(tool("check_build_task"), (input, context) =>
withHeartbeat(context, "check_build_task", () =>
steps.checkTask(input, context.mcpReq.signal),
),
);
registerPlain(tool("close_build_computer"), (input) =>
computers.close(input.computer_id),
);
};
tests/boundary/cli/cliToolCoverage.test.ts
+5
View File
@@ -100,6 +100,11 @@ const MCP_TOOLS_WITHOUT_DEDICATED_CLI = Object.freeze([
"start_build_computer",
"act_on_build_computer",
"close_build_computer",
"start_build_capture",
"mark_build_checkpoint",
"stop_build_capture",
"export_build_environment",
"check_build_task",
]);
describe("CLI canonical tool coverage", () => {
tests/boundary/mcp/buildCaptureMcp.test.ts
+235
View File
@@ -0,0 +1,235 @@
import { readdir, readFile } from "node:fs/promises";
import { join } from "node:path";
import { expect, it } from "vitest";
import { toolContract } from "../../../src/contracts/toolContracts.js";
import {
BUILD_REQUEST as REQUEST,
BUILD_TASK as TASK,
buildCaptureFakes as fakes,
} from "../../fixtures/buildCaptureFakes.js";
import {
bundleFolder,
callOk,
expectNoToken,
startBuildComputer,
} from "../../fixtures/buildComputerMcp.js";
import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
const files = async (root: string, prefix = ""): Promise<string[]> =>
(
await Promise.all(
(await readdir(join(root, prefix), { withFileTypes: true })).map(
async (entry) =>
entry.isDirectory()
? files(root, join(prefix, entry.name))
: [join(prefix, entry.name)],
),
)
)
.flat()
.sort();
it("captures the app inside the computer, acknowledges checkpoints, drafts the task and brings the exported bundle back", async () => {
const { rea, outputs, computer, client } = await fakes();
const id = await startBuildComputer(client, REQUEST);
const started = await callOk(client, "start_build_capture", {
computer_id: id,
});
expect(
toolContract("start_build_capture").outputSchema.parse(
started.structuredContent,
),
).toMatchObject({
desktop_entry: "org.example.notes",
package_lists: 1,
package_lists_error: null,
});
expect(started.content.some((block) => block.type === "image")).toBe(true);
const mark = (name: string) =>
client.callTool({
name: "mark_build_checkpoint",
arguments: { computer_id: id, name },
});
const seed = await mark("seed");
expect(
toolContract("mark_build_checkpoint").outputSchema.parse(
seed.structuredContent,
).checkpoint,
).toEqual({
name: "seed",
trigger: "request",
at_ms: 1000,
entries: 5,
changes: 1,
retained_files: 1,
retained_bytes: 5,
});
expect(parseMcpToolError(await mark("seed")).error.message).toContain(
"already taken",
);
expect(parseMcpToolError(await mark("exit")).error.code).toBe(
"invalid_request",
);
await callOk(client, "act_on_build_computer", {
computer_id: id,
action: "type",
text: "eggs\n",
});
await mark("end");
const stopped = toolContract("stop_build_capture").outputSchema.parse(
(await callOk(client, "stop_build_capture", { computer_id: id }))
.structuredContent,
);
expect(stopped.checkpoints.map(({ name }) => name)).toEqual([
"baseline",
"seed",
"end",
"stop",
"exit",
]);
expect(stopped.draft).toMatchObject({
seed_checkpoint: "seed",
end_checkpoint: "end",
files: [{ file: "Documents/today.txt" }],
task: {
checks: [
{ kind: "lines_present", file: "Documents/today.txt", lines: ["eggs"] },
],
},
});
const directory = join(await createTestTempDirectory("build-out-"), "notes");
const exported = toolContract("export_build_environment").outputSchema.parse(
(
await callOk(client, "export_build_environment", {
computer_id: id,
task: TASK,
directory,
})
).structuredContent,
);
const { bundle } = outputs;
expect(await files(directory)).toEqual(await files(bundle));
for (const path of await files(directory))
expect(
(await readFile(join(directory, path))).equals(
await readFile(join(bundle, path)),
),
path,
).toBe(true);
expect(exported).toMatchObject({
environment: "notes",
files: (await files(directory)).length,
seed_files: [".config/notes/settings.ini", "Documents/today.txt"],
task: { name: "add-eggs", failing_on_seed: [expect.any(String)] },
});
expect(await rea.notes("tasks.jsonl")).toEqual([TASK]);
// REA ran as the computer's user in its home, with the user's own package
// lists and without the control token the computer's routes take.
const calls = (await rea.notes("calls.jsonl")) as {
argv: string[];
cwd: string;
apt_config: string;
agentenv: string[];
}[];
expect(calls.map(({ argv }) => argv[0])).toEqual([
"capture-desktop-application",
"describe-desktop-application-task",
"export-desktop-application",
]);
for (const call of calls)
expect(call).toMatchObject({
agentenv: [],
apt_config: join(rea.home, ".rea-build/apt/apt.conf"),
});
expect(calls[0]?.argv).toEqual(
expect.arrayContaining([
"--desktop-entry",
"org.example.notes",
"--exclude",
join(rea.home, ".rea-build"),
]),
);
expect(calls.slice(1).map(({ argv }) => argv)).toEqual([
expect.arrayContaining([
"--seed-checkpoint",
"seed",
"--end-checkpoint",
"end",
]),
expect.arrayContaining([
"--seed-checkpoint",
"seed",
"--end-checkpoint",
"end",
]),
]);
expectNoToken(computer, [started, seed, stopped, exported]);
});
it("refuses a task too large for one command argument before the export runs in the computer", async () => {
const { rea, client } = await fakes();
const id = await startBuildComputer(client, REQUEST);
await callOk(client, "start_build_capture", { computer_id: id });
for (const name of ["seed", "end"])
await callOk(client, "mark_build_checkpoint", { computer_id: id, name });
await callOk(client, "stop_build_capture", { computer_id: id });
const { error } = parseMcpToolError(
await client.callTool({
name: "export_build_environment",
arguments: {
computer_id: id,
task: { ...TASK, prompt: "Add eggs. ".repeat(10 * 1024) },
directory: join(await createTestTempDirectory("build-out-"), "b"),
},
}),
);
expect(error.code).toBe("invalid_request");
const calls = (await rea.notes("calls.jsonl")) as { argv: string[] }[];
expect(calls.map(({ argv }) => argv[0])).not.toContain(
"export-desktop-application",
);
});
it("refuses capture steps out of order and a capture of a bundle's computer", async () => {
const bundled = await fakes();
const fromBundle = await startBuildComputer(bundled.client, {
kind: "bundle",
path: await bundleFolder(),
});
expect(
parseMcpToolError(
await bundled.client.callTool({
name: "start_build_capture",
arguments: { computer_id: fromBundle },
}),
).error.message,
).toContain("request");
const { client } = await fakes();
const call = (name: string, args: Record<string, unknown>) =>
client.callTool({ name, arguments: args }).then(parseMcpToolError);
const id = await startBuildComputer(client, REQUEST);
expect(
(await call("mark_build_checkpoint", { computer_id: id, name: "seed" }))
.error.message,
).toContain("No capture is running");
expect(
(await call("stop_build_capture", { computer_id: id })).error.message,
).toContain("No capture is running");
expect(
(
await call("export_build_environment", {
computer_id: id,
task: TASK,
directory: join(await createTestTempDirectory("build-out-"), "b"),
})
).error.message,
).toContain("stopped capture");
});
tests/boundary/mcp/buildCaptureRetryMcp.test.ts
+311
View File
@@ -0,0 +1,311 @@
import { mkdir, readdir, writeFile } from "node:fs/promises";
import { join } from "node:path";
import type { Client } from "@modelcontextprotocol/client";
import { expect, it, vi } from "vitest";
import { toolContract } from "../../../src/contracts/toolContracts.js";
import {
VARICAD_URL,
varicadPackage,
} from "../../../src/domain/desktop/debianPackageFile.fixture.js";
import {
BUILD_REQUEST,
BUILD_TASK,
buildCaptureFakes as fakes,
} from "../../fixtures/buildCaptureFakes.js";
import { callOk, startBuildComputer } from "../../fixtures/buildComputerMcp.js";
import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
import { cancelDuringCommit } from "../../fixtures/outputTreeCommit.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
const call = (client: Client, name: string, args: Record<string, unknown>) =>
client.callTool({ name, arguments: args });
/** Start a capture on the computer and mark seed and end. */
const captureSeedAndEnd = async (client: Client, id: string) => {
await callOk(client, "start_build_capture", { computer_id: id });
for (const name of ["seed", "end"])
await callOk(client, "mark_build_checkpoint", { computer_id: id, name });
};
const captureCalls = async (
rea: Awaited<ReturnType<typeof fakes>>["rea"],
): Promise<string[][]> =>
((await rea.notes("calls.jsonl")) as { argv: string[] }[])
.map(({ argv }) => argv)
.filter((argv) => argv[0] === "capture-desktop-application");
it("stops a capture whose start reply was lost, and retries that stop before starting another", async () => {
const { rea, client } = await fakes({
dropReplies: ["start", "stop", "stop"],
});
const id = await startBuildComputer(client, BUILD_REQUEST);
// The capture started in the computer, but REA never heard so.
const lost = parseMcpToolError(
await call(client, "start_build_capture", { computer_id: id }),
);
expect(lost.error.code).not.toBe("invalid_request");
// Its stop ran too, unheard; REA keeps the capture until a stop is confirmed.
const unconfirmed = parseMcpToolError(
await call(client, "start_build_capture", { computer_id: id }),
);
expect(unconfirmed.error.message).toContain("captures/1");
expect(await captureCalls(rea)).toHaveLength(1);
// The third stop is heard: the first capture is released, and a new one starts.
const started = await callOk(client, "start_build_capture", {
computer_id: id,
});
expect(started.structuredContent).toMatchObject({ package_lists: 1 });
const calls = await captureCalls(rea);
expect(calls).toHaveLength(2);
expect(calls[1]).toContain(join(rea.home, ".rea-build/captures/2/capture"));
expect(await rea.notes("stopped.jsonl")).toHaveLength(1);
await callOk(client, "stop_build_capture", { computer_id: id });
});
it("returns a checkpoint taken in the computer when its reply was lost and the name is asked again", async () => {
const { client } = await fakes({ dropReplies: ["mark"] });
const id = await startBuildComputer(client, BUILD_REQUEST);
await callOk(client, "start_build_capture", { computer_id: id });
const mark = (name: string) =>
call(client, "mark_build_checkpoint", { computer_id: id, name });
expect(parseMcpToolError(await mark("seed")).error.code).not.toBe(
"invalid_request",
);
const retried = await mark("seed");
expect(retried.isError, JSON.stringify(retried)).not.toBe(true);
expect(retried.structuredContent).toMatchObject({
checkpoint: { name: "seed", at_ms: 1000, changes: 1 },
});
await callOk(client, "mark_build_checkpoint", {
computer_id: id,
name: "end",
});
const stopped = await callOk(client, "stop_build_capture", {
computer_id: id,
});
expect(stopped.structuredContent).toMatchObject({
draft: { seed_checkpoint: "seed", end_checkpoint: "end" },
});
});
it("finishes a stop whose draft reply was lost when stop is asked again, and repeats its result", async () => {
const { rea, client } = await fakes({ dropReplies: ["run"] });
const id = await startBuildComputer(client, BUILD_REQUEST);
await captureSeedAndEnd(client, id);
const stop = () => call(client, "stop_build_capture", { computer_id: id });
expect((await stop()).isError).toBe(true);
const first = await stop();
expect(first.isError, JSON.stringify(first)).not.toBe(true);
const stopped = toolContract("stop_build_capture").outputSchema.parse(
first.structuredContent,
);
expect(stopped.draft?.files).toEqual([
expect.objectContaining({ file: "Documents/today.txt" }),
]);
expect((await stop()).structuredContent).toEqual(first.structuredContent);
expect(await rea.notes("stopped.jsonl")).toHaveLength(1);
const exported = await call(client, "export_build_environment", {
computer_id: id,
task: BUILD_TASK,
directory: join(await createTestTempDirectory("build-out-"), "bundle"),
});
expect(exported.isError, JSON.stringify(exported)).not.toBe(true);
});
it("reports a failed package list download with apt's own message", async () => {
const { client } = await fakes({ env: { FAKE_APT_EXIT: "100" } });
const id = await startBuildComputer(client, BUILD_REQUEST);
const started = await callOk(client, "start_build_capture", {
computer_id: id,
});
const value = toolContract("start_build_capture").outputSchema.parse(
started.structuredContent,
);
expect(value.package_lists).toBe(0);
expect(value.package_lists_error).toContain("exited 100");
expect(value.package_lists_error).toContain("Failed to fetch");
await callOk(client, "stop_build_capture", { computer_id: id });
});
it("downloads the package lists again when an update exits 0 with no list", async () => {
const { client } = await fakes({
env: { FAKE_APT_EMPTY: "1", FAKE_APT_IGNORE_ERROR_MODE: "1" },
});
const id = await startBuildComputer(client, BUILD_REQUEST);
for (const attempt of [1, 2]) {
const value = toolContract("start_build_capture").outputSchema.parse(
(await callOk(client, "start_build_capture", { computer_id: id }))
.structuredContent,
);
expect(value.package_lists, `start ${String(attempt)}`).toBe(0);
expect(value.package_lists_error, `start ${String(attempt)}`).toContain(
"without a package list",
);
await callOk(client, "stop_build_capture", { computer_id: id });
}
});
it("retries an empty apt update that Error-Mode turns into a failure", async () => {
const { client } = await fakes({ env: { FAKE_APT_EMPTY: "1" } });
const id = await startBuildComputer(client, BUILD_REQUEST);
for (const attempt of [1, 2]) {
const value = toolContract("start_build_capture").outputSchema.parse(
(await callOk(client, "start_build_capture", { computer_id: id }))
.structuredContent,
);
expect(value.package_lists, `start ${String(attempt)}`).toBe(0);
expect(value.package_lists_error, `start ${String(attempt)}`).toContain(
"exited 100",
);
await callOk(client, "stop_build_capture", { computer_id: id });
}
});
it("downloads the package lists again on the next start after an update that fetched some and failed", async () => {
const { client } = await fakes({
env: { FAKE_APT_EXIT: "100", FAKE_APT_PARTIAL: "1" },
});
const id = await startBuildComputer(client, BUILD_REQUEST);
for (const attempt of [1, 2]) {
const value = toolContract("start_build_capture").outputSchema.parse(
(await callOk(client, "start_build_capture", { computer_id: id }))
.structuredContent,
);
expect(value.package_lists, `start ${String(attempt)}`).toBe(1);
expect(value.package_lists_error, `start ${String(attempt)}`).toContain(
"exited 100",
);
await callOk(client, "stop_build_capture", { computer_id: id });
}
});
it("cancels a capture's start when the computer closes while it waits for the opened app's screenshot", async () => {
// The start's own screenshot answers at once; the capture's waits until its request is cancelled.
const { computer, client } = await fakes({ screenshotDelaysMs: [0, 60_000] });
const id = await startBuildComputer(client, BUILD_REQUEST);
const screenshots = () =>
computer.calls.filter(({ name }) => name === "computer_screenshot").length;
expect(screenshots()).toBe(1);
const started = call(client, "start_build_capture", { computer_id: id });
await vi.waitFor(() => expect(screenshots()).toBe(2), { timeout: 30_000 });
await callOk(client, "close_build_computer", { computer_id: id });
expect(parseMcpToolError(await started).error.code).toBe("cancelled");
}, 60_000);
it("returns the bundle an export saved when the computer closes while it commits", async () => {
const { client } = await fakes();
const id = await startBuildComputer(client, BUILD_REQUEST);
await captureSeedAndEnd(client, id);
await callOk(client, "stop_build_capture", { computer_id: id });
const directory = join(await createTestTempDirectory("build-out-"), "bundle");
let closed: Promise<unknown> = Promise.resolve();
cancelDuringCommit(() => {
closed = call(client, "close_build_computer", { computer_id: id });
});
const exported = await call(client, "export_build_environment", {
computer_id: id,
task: BUILD_TASK,
directory,
});
expect(exported.isError, JSON.stringify(exported)).not.toBe(true);
expect(exported.structuredContent).toMatchObject({
computer_id: id,
directory,
});
expect((await readdir(directory)).length).toBeGreaterThan(0);
await closed;
}, 60_000);
/** Capture and export an app the request installs from a .deb at `url`; the stand-in's argv for capture and export. */
const captureDebRequest = async (
fakeSet: Awaited<ReturnType<typeof fakes>>,
url: string,
) => {
const { rea, client } = fakeSet;
const id = await startBuildComputer(client, {
kind: "request",
desktop_entry: "org.example.notes",
install: {
kind: "package_file",
url,
sha256: "0".repeat(64),
architecture: "amd64",
dependencies: [],
},
});
await captureSeedAndEnd(client, id);
await callOk(client, "stop_build_capture", { computer_id: id });
const exported = toolContract("export_build_environment").outputSchema.parse(
(
await callOk(client, "export_build_environment", {
computer_id: id,
task: BUILD_TASK,
directory: join(await createTestTempDirectory("build-out-"), "bundle"),
})
).structuredContent,
);
const calls = ((await rea.notes("calls.jsonl")) as { argv: string[] }[]).map(
({ argv }) => argv,
);
const argv = (command: string) =>
calls.find((candidate) => candidate[0] === command) ?? [];
expect(argv("capture-desktop-application")).toEqual(
expect.arrayContaining(["--package-file", "/opt/rea/package.deb"]),
);
return { exported, exportArgv: argv("export-desktop-application") };
};
it("exports an app from its .deb file, at the request's URL, when no apt source offers it", async () => {
const { exported, exportArgv } = await captureDebRequest(
await fakes({
install: { package: varicadPackage(), packageUrl: VARICAD_URL },
}),
VARICAD_URL,
);
expect(exportArgv).toEqual(
expect.arrayContaining(["--package-url", VARICAD_URL]),
);
expect(exported.files).toBeGreaterThan(0);
});
it("exports an app from apt, without the .deb's URL, when an apt source offers the captured version too", async () => {
// The capture found notes in noble/universe, so the export installs it by name.
const { exported, exportArgv } = await captureDebRequest(
await fakes(),
"https://downloads.example.com/notes_1.0_amd64.deb",
);
expect(exportArgv).not.toContain("--package-url");
expect(exported.files).toBeGreaterThan(0);
});
it("brings back a bundle whose file list outgrows a command's 1 MiB of output", async () => {
const fakeSet = await fakes();
const { outputs, client } = fakeSet;
// 1,500 files whose list, about 1.15 MB with their long paths, is more than the 1 MiB exec answers with.
const long = (letter: string) => letter.repeat(200);
const extra = join(outputs.bundle, "seed", long("a"), long("b"));
await mkdir(extra, { recursive: true });
for (let index = 0; index < 1500; index += 1)
await writeFile(join(extra, `${long("c")}-${String(index)}`), "x");
const id = await startBuildComputer(client, BUILD_REQUEST);
await captureSeedAndEnd(client, id);
await callOk(client, "stop_build_capture", { computer_id: id });
const directory = join(await createTestTempDirectory("build-out-"), "bundle");
const exported = toolContract("export_build_environment").outputSchema.parse(
(
await callOk(client, "export_build_environment", {
computer_id: id,
task: BUILD_TASK,
directory,
})
).structuredContent,
);
expect(exported.files).toBeGreaterThan(1500);
expect(
await readdir(join(directory, "seed", long("a"), long("b"))),
).toHaveLength(1500);
}, 120_000);
tests/boundary/mcp/buildComputerMcp.test.ts
+241 -69
View File
@@ -1,85 +1,38 @@
import { chmod, mkdir, readFile, rm, writeFile } from "node:fs/promises";
import {
chmod,
mkdir,
readFile,
readdir,
rm,
writeFile,
} from "node:fs/promises";
import { join } from "node:path";
import { setTimeout as delay } from "node:timers/promises";
import { Client, InMemoryTransport } from "@modelcontextprotocol/client";
import type { Client } from "@modelcontextprotocol/client";
import type { CallToolResult } from "@modelcontextprotocol/server";
import { expect, it, onTestFinished } from "vitest";
import { expect, it, vi } from "vitest";
import { z } from "zod";
import { toolContract } from "../../../src/contracts/toolContracts.js";
import { createServer } from "../../../src/server/createServer.js";
import { createTestBinarySession } from "../../fixtures/binarySession.js";
import {
buildComputerFakes as fakes,
bundleFolder,
callOk,
connectBuildServer as connect,
expectNoToken,
startBuildComputer,
} from "../../fixtures/buildComputerMcp.js";
import {
FAKE_FRAME_PNG,
fakeAgentEnvRunner,
startFakeComputer,
type FakeApp,
type FakeComputer,
type FakeComputerOptions,
} from "../../fixtures/fakeAgentEnvComputer.js";
import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
/** The host's environment without the settings the opt-in real test exports, which would reach past these fakes. */
const hostEnvironment = Object.fromEntries(
Object.entries(process.env).filter(
([name]) =>
![
"REA_AGENT_ENV_COMMAND",
"REA_DOCKER_COMMAND",
"REA_BUILD_COMPUTER_REA_ARCHIVE",
].includes(name),
),
);
const connect = async (environment: Readonly<Record<string, string>>) => {
const session = createTestBinarySession(() => {
throw new Error("build computers must not start a deep provider");
});
const server = createServer(
{ kind: "session", session },
{
environment: {
...hostEnvironment,
REA_MCP_INPUT_SCHEMA_PROFILE: "full",
...environment,
},
},
);
const client = new Client({ name: "build-computer-mcp", version: "1" });
onTestFinished(async () => {
await client.close();
await server.close();
await session.close();
});
const [clientTransport, serverTransport] =
InMemoryTransport.createLinkedPair();
await server.connect(serverTransport);
await client.connect(clientTransport);
return client;
};
const fakes = async (options: FakeComputerOptions = {}) => {
const computer = await startFakeComputer(options);
onTestFinished(() => computer.close());
const runner = await fakeAgentEnvRunner(computer);
return { computer, runner };
};
const bundleFolder = async (): Promise<string> => {
const bundle = await createTestTempDirectory("exported-bundle-");
await mkdir(join(bundle, "tasks"));
await writeFile(join(bundle, "tasks", "smoke.json"), "[]\n");
return bundle;
};
const images = (result: CallToolResult) =>
result.content.filter((block) => block.type === "image");
/** No delivered text, success or failure, may hold the control token. */
const expectNoToken = (computer: FakeComputer, result: CallToolResult) =>
expect(JSON.stringify(result)).not.toContain(computer.token);
const startFromBundle = async (client: Client, computer: FakeComputer) => {
const started = await client.callTool({
name: "start_build_computer",
@@ -319,10 +272,84 @@ it("takes the first screenshot with the caller's options", async () => {
},
});
expect(started.isError, JSON.stringify(started)).not.toBe(true);
expect(computer.calls[0]).toEqual({
name: "computer_screenshot",
arguments: { region: [0, 0, 640, 400], format: "jpeg", quality: 40 },
// The region crops the drawn frame REA waited for.
expect(computer.calls).toEqual([
{ name: "computer_screenshot", arguments: {} },
{
name: "computer_screenshot",
arguments: { region: [0, 0, 640, 400], format: "jpeg", quality: 40 },
},
]);
});
it("waits for the desktop to draw before it returns the first screenshot", async () => {
const { computer, runner } = await fakes({ blankFrames: 2 });
const client = await connect(runner.environment);
const { started } = await startFromBundle(client, computer);
expect(images(started)).toEqual([
{ type: "image", data: FAKE_FRAME_PNG, mimeType: "image/png" },
]);
expect(computer.calls).toEqual(
Array.from({ length: 3 }, () => ({
name: "computer_screenshot",
arguments: {},
})),
);
});
it("returns the latest frame when a retried screenshot outlasts the wait for the desktop, sending progress meanwhile", async () => {
// The third screenshot would answer a minute later, well past the 10 s wait.
const { computer, runner } = await fakes({
blankFrames: 10_000,
screenshotDelaysMs: [0, 0, 60_000],
});
const client = await connect(runner.environment);
const progress: { message?: string | undefined }[] = [];
const startedAt = Date.now();
const started = await client.callTool(
{
name: "start_build_computer",
arguments: { source: { kind: "bundle", path: await bundleFolder() } },
},
{ onprogress: (update) => progress.push(update) },
);
expect(started.isError, JSON.stringify(started)).not.toBe(true);
expect(Date.now() - startedAt).toBeLessThan(20_000);
expect(computer.calls).toHaveLength(3);
expect(images(started)).toHaveLength(1);
expect(progress.map(({ message }) => message)).toContainEqual(
expect.stringContaining("waiting for the desktop to draw"),
);
}, 40_000);
it("leaves the bundle folder's files as they were when it starts a computer from it", async () => {
const { runner } = await fakes();
// Python caches bytecode unless told not to; this host does not tell it.
const client = await connect({
...runner.environment,
PYTHONDONTWRITEBYTECODE: "",
});
const bundle = await bundleFolder();
await mkdir(join(bundle, "envs", "notes"), { recursive: true });
await writeFile(
join(bundle, "envs", "notes", "notes_app.py"),
"class Notes:\n pass\n",
);
const files = async () => (await readdir(bundle, { recursive: true })).sort();
const before = await files();
const id = await startBuildComputer(client, { kind: "bundle", path: bundle });
await callOk(client, "close_build_computer", { computer_id: id });
expect(
await readFile(
join(runner.state, "bundle", "envs", "notes", "notes_app.py"),
"utf8",
),
).toContain("class Notes");
expect(await files()).toEqual(before);
});
it("refuses to start without an absolute agentenv runner", async () => {
@@ -429,3 +456,148 @@ it("stops the helper when the caller cancels a start", async () => {
}
expect(await runner.notes("interrupted.jsonl")).toHaveLength(1);
});
/** An app whose reset keeps the computer unready for `polls` readiness polls, 500 ms apart. */
const slowApp = (polls: number): FakeApp => ({
name: "notes",
state: {},
seed: {},
methods: [],
unreadyAfterReset: polls,
});
it.each([
{
failure: "refuses",
fail: (computer: FakeComputer) =>
computer.refuseNext("computer_screenshot", "no display to capture"),
message: "no display to capture",
},
{
failure: "loses the reply to",
fail: (computer: FakeComputer) => computer.loseNext("computer_screenshot"),
message: expect.stringMatching(/\S/),
},
])(
"keeps a reset applied when the computer $failure the screenshot after it",
async ({ fail, message }) => {
const app = slowApp(0);
const { computer, runner } = await fakes({ app });
const client = await connect(runner.environment);
const id = await startBuildComputer(client, {
kind: "bundle",
path: await bundleFolder(),
});
fail(computer);
const reset = await callOk(client, "act_on_build_computer", {
computer_id: id,
action: "reset",
screenshot: true,
});
const { result } = toolContract("act_on_build_computer").outputSchema.parse(
reset.structuredContent,
);
expect(result).toEqual({
reset: true,
ready_ms: expect.any(Number),
screenshot_error: message,
});
expect(images(reset)).toEqual([]);
expect(app.methods).toEqual(["data/reset"]);
expect(computer.calls.map(({ name }) => name).at(-1)).toBe(
"computer_screenshot",
);
},
);
it("runs one computer's calls in the order they arrive and another computer's beside them", async () => {
const app = slowApp(4);
const { runner } = await fakes({ app });
const other = await fakes();
const client = await connect(runner.environment);
const first = await startBuildComputer(client, {
kind: "bundle",
path: await bundleFolder(),
});
const second = await startBuildComputer(client, {
kind: "bundle",
path: await bundleFolder(other.computer),
});
const act = (id: string, action: string) =>
callOk(client, "act_on_build_computer", { computer_id: id, action });
const order: string[] = [];
const reset = act(first, "reset").then(() => order.push("reset"));
await vi.waitFor(() => expect(app.methods).toEqual(["data/reset"]));
const behind = act(first, "screenshot").then(() =>
order.push("screenshot behind the reset"),
);
await act(second, "screenshot");
order.push("other computer's screenshot");
await Promise.all([reset, behind]);
expect(order).toEqual([
"other computer's screenshot",
"reset",
"screenshot behind the reset",
]);
}, 30_000);
it("sends progress while an action waits for the call before it on its computer", async () => {
const app = slowApp(14);
const { runner } = await fakes({ app });
const client = await connect(runner.environment);
const id = await startBuildComputer(client, {
kind: "bundle",
path: await bundleFolder(),
});
const reset = callOk(client, "act_on_build_computer", {
computer_id: id,
action: "reset",
});
await vi.waitFor(() => expect(app.methods).toEqual(["data/reset"]));
const progress: unknown[] = [];
const waited = await client.callTool(
{
name: "act_on_build_computer",
arguments: { computer_id: id, action: "screenshot" },
},
{ onprogress: (update) => progress.push(update) },
);
expect(waited.isError, JSON.stringify(waited)).not.toBe(true);
expect(progress.length).toBeGreaterThan(0);
await reset;
}, 30_000);
it("cancels a computer's call in progress on close and refuses the calls waiting behind it", async () => {
const app = slowApp(600);
const { computer, runner } = await fakes({ app });
const client = await connect(runner.environment);
const id = await startBuildComputer(client, {
kind: "bundle",
path: await bundleFolder(),
});
const calls = computer.calls.length;
const call = (name: string, fields: Record<string, unknown> = {}) =>
client.callTool({ name, arguments: { computer_id: id, ...fields } });
const reset = call("act_on_build_computer", { action: "reset" });
await vi.waitFor(() => expect(app.methods).toEqual(["data/reset"]));
const waiting = [
call("act_on_build_computer", { action: "screenshot" }),
call("stop_build_capture"),
];
const closed = await callOk(client, "close_build_computer", {
computer_id: id,
});
expect(closed.structuredContent).toEqual({
computer_id: id,
teardown: "agentenv",
});
for (const result of await Promise.all([reset, ...waiting]))
expect(parseMcpToolError(result).error.code).toBe("cancelled");
expect(computer.calls).toHaveLength(calls);
expect(app.methods).toEqual(["data/reset"]);
expect(await runner.notes("teardowns.jsonl")).toHaveLength(1);
expect(
parseMcpToolError(await call("act_on_build_computer", { action: "wait" }))
.error.message,
).toContain("start_build_computer");
});
tests/boundary/mcp/buildWitnessMcp.test.ts
+677
View File
@@ -0,0 +1,677 @@
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
import { join } from "node:path";
import type { CallToolResult } from "@modelcontextprotocol/server";
import { expect, it, vi } from "vitest";
import { isPortableBundleName } from "../../../src/application/environment/environmentBundle.js";
import { toolContract } from "../../../src/contracts/toolContracts.js";
import { parseDesktopTask } from "../../../src/domain/desktop/appTask.js";
import {
renderTaskSteps,
renderTaskVerifier,
taskVerifierArtifact,
} from "../../../src/domain/desktop/appTaskPython.js";
import { parseEvidence } from "../../../src/domain/evidence.js";
import {
buildComputerFakes,
bundleFolder,
callOk,
connectBuildServer,
expectNoToken,
startBuildComputer,
} from "../../fixtures/buildComputerMcp.js";
import {
FAKE_SEGMENTS,
type FakeApp,
type FakeComputerOptions,
} from "../../fixtures/fakeAgentEnvComputer.js";
import { parseMcpToolError } from "../../fixtures/mcpToolError.js";
import { cancelDuringCommit } from "../../fixtures/outputTreeCommit.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
const TASK = {
name: "add-eggs",
prompt: "Add eggs to today's list and save it.",
checks: [
{ kind: "lines_present", file: "Documents/today.txt", lines: ["eggs"] },
],
};
const WITNESS_ORDER = ["seed", "after_edit", "near_miss", "after_reset"];
const lines = (...text: string[]) => ({
"Documents/today.txt": {
format: "text",
view: { lines: text, line_count: text.length, truncated: false },
truncated: false,
},
});
/** A bundle folder holding `task` as the export writes it for the app. */
const taskBundle = async (task: unknown, app: string) => {
const parsed = parseDesktopTask(task, isPortableBundleName);
if (!parsed.ok) throw new Error(parsed.error);
const bundle = await bundleFolder();
const verifier = join(
bundle,
"artifacts",
taskVerifierArtifact(parsed.value),
"verify.py",
);
await mkdir(join(verifier, ".."), { recursive: true });
await writeFile(verifier, renderTaskVerifier(parsed.value, app));
await writeFile(
join(bundle, "tasks", `${parsed.value.name}.json`),
renderTaskSteps(parsed.value, app),
);
return bundle;
};
const witnessed = async (
slowReset = 0,
options: FakeComputerOptions = {},
task: unknown = TASK,
) => {
const app: FakeApp = {
name: "notes",
state: lines("milk"),
seed: lines("milk"),
methods: [],
unreadyAfterReset: slowReset,
};
const { computer, runner } = await buildComputerFakes({ ...options, app });
const client = await connectBuildServer(runner.environment);
const bundle = await taskBundle(task, app.name);
const id = await startBuildComputer(client, { kind: "bundle", path: bundle });
const directory = join(await createTestTempDirectory("witness-"), "add-eggs");
const check = (fields: Record<string, unknown>) =>
client.callTool({
name: "check_build_task",
arguments: { computer_id: id, task, ...fields },
});
const act = (action: string, fields: Record<string, unknown> = {}) =>
callOk(client, "act_on_build_computer", {
computer_id: id,
action,
...fields,
});
return { app, bundle, computer, client, id, directory, check, act };
};
/** Act on the app and set its state as that act would leave it. */
const edit = async (
witness: Awaited<ReturnType<typeof witnessed>>,
state: Record<string, unknown>,
) => {
await witness.act("type", { text: "x" });
witness.app.state = state;
};
/** Check the seed, the edit and the near miss, each after the act it needs. */
const throughNearMiss = async (
witness: Awaited<ReturnType<typeof witnessed>>,
) => {
await witness.check({ stage: "seed", directory: witness.directory });
await edit(witness, lines("milk", "eggs"));
await witness.check({ stage: "after_edit" });
await edit(witness, lines("milk", "egg"));
await witness.check({ stage: "near_miss" });
};
const parsed = (result: unknown) =>
toolContract("check_build_task").outputSchema.parse(
(result as { structuredContent: unknown }).structuredContent,
);
/** A stage its checks leave undecided: refused as a request naming the stage and what left each check undecided, in any wording. */
const expectUndecided = (
result: CallToolResult,
stage: string,
facts: readonly string[],
) => {
const { error } = parseMcpToolError(result);
expect(error.code).toBe("invalid_request");
for (const fact of [stage, ...facts]) expect(error.message).toContain(fact);
};
it("witnesses a task: the seed fails, the edit passes, a near miss fails and so does the reset seed", async () => {
const { app, computer, directory, check, act } = await witnessed();
// A plain check grades the live state, one result per check, and files nothing.
const plain = parsed(await check({}));
expect(plain).toMatchObject({
task: "add-eggs",
passed: 0,
failed: 1,
witness: null,
});
expect(
plain.results.map(({ id, passed, undecided }) => ({
id,
passed,
undecided,
})),
).toEqual([{ id: "check-1", passed: false }]);
for (const fact of ["Documents/today.txt", '"eggs"'])
expect(plain.results[0]?.description).toContain(fact);
const seed = parsed(await check({ stage: "seed", directory }));
expect(seed.witness).toMatchObject({
stage: "seed",
expectation: "some_fail",
held: true,
status: "open",
next_stage: "after_edit",
});
expect(
parseMcpToolError(await check({ stage: "after_edit" })).error.message,
).toContain("act on the app");
await act("screenshot");
await act("type", { text: "eggs\n" });
app.state = lines("milk", "eggs");
expect(parsed(await check({ stage: "after_edit" })).witness).toMatchObject({
held: true,
next_stage: "near_miss",
});
await act("key", { keys: ["BackSpace"] });
app.state = lines("milk", "egg");
expect(parsed(await check({ stage: "near_miss" })).witness).toMatchObject({
held: true,
next_stage: "after_reset",
});
expect(
parseMcpToolError(await check({ stage: "after_reset" })).error.message,
).toContain("reset");
const reset = await act("reset", { screenshot: true });
expect(reset.content.some((block) => block.type === "image")).toBe(true);
expect(app.state).toEqual(app.seed);
const last = await check({ stage: "after_reset" });
expectNoToken(computer, last);
const { witness } = parsed(last);
expect(witness).toMatchObject({
stage: "after_reset",
held: true,
status: "held",
next_stage: null,
});
const record = witness?.record;
expect(record).toMatchObject({
path: join(directory, "witness.json"),
verdict: "held",
failed_stage: null,
task: { name: "add-eggs" },
recording: {
recording_id: "rec-000000000001",
segments: FAKE_SEGMENTS.map((bytes, index) => ({
path: `recording/000${String(index)}.mp4`,
bytes: bytes.length,
})),
},
});
expect(record?.stages.map((stage) => [stage.stage, stage.held])).toEqual([
["seed", true],
["after_edit", true],
["near_miss", true],
["after_reset", true],
]);
expect(record?.stages.map((stage) => stage.results[0]?.passed)).toEqual([
false,
true,
false,
false,
]);
expect(
record?.actions.map(({ action, tool, outcome }) => [action, tool, outcome]),
).toEqual([
["screenshot", "computer_screenshot", "applied"],
["type", "computer_type", "applied"],
["key", "computer_key", "applied"],
["reset", "data/reset", "applied"],
]);
expect(app.methods).toEqual([
"data/get",
"data/get",
"data/get",
"data/get",
"data/reset",
"data/get",
]);
for (const [index, bytes] of FAKE_SEGMENTS.entries())
expect(
(
await readFile(join(directory, "recording", `000${String(index)}.mp4`))
).equals(bytes),
).toBe(true);
const evidence = parseEvidence(
JSON.parse(await readFile(join(directory, "witness.json"), "utf8")),
);
expect(evidence).toMatchObject({
evidence_id: record?.evidence_id,
predicate_type: "rea.desktop-build-witness",
});
expect(evidence.normalized_result).toMatchObject({ verdict: "held" });
expect(
parseMcpToolError(await check({ stage: "seed", directory })).error.message,
).toContain("already witnessed");
});
it("fails the witness at after_reset when the reset leaves the near miss instead of the seed", async () => {
const task = {
...TASK,
checks: [
{ id: "eggs", ...TASK.checks[0] },
{
id: "no-egg",
kind: "lines_absent",
file: "Documents/today.txt",
lines: ["egg"],
},
],
};
const { app, check, act, directory } = await witnessed(0, {}, task);
// The app's reset brings back the near miss rather than the seed.
Object.assign(app.seed, lines("milk", "egg"));
await check({ task, stage: "seed", directory });
await act("type", { text: "x" });
app.state = lines("milk", "eggs");
await check({ task, stage: "after_edit" });
await act("type", { text: "x" });
app.state = lines("milk", "egg");
await check({ task, stage: "near_miss" });
await act("reset");
const { witness } = parsed(await check({ task, stage: "after_reset" }));
expect(witness).toMatchObject({
held: false,
status: "failed",
record: { verdict: "failed", failed_stage: "after_reset" },
});
expect(witness?.record?.stages.at(-1)?.remediation).toContain("no-egg");
});
it("takes the after_reset stage after a reset whose screenshot the computer refused", async () => {
const witness = await witnessed();
const { app, check, client, computer, id } = witness;
await throughNearMiss(witness);
computer.refuseNext("computer_screenshot", "no display to capture");
await client.callTool({
name: "act_on_build_computer",
arguments: { computer_id: id, action: "reset", screenshot: true },
});
expect(app.state).toEqual(app.seed);
const checked = await check({ stage: "after_reset" });
expect(checked.isError, JSON.stringify(checked)).not.toBe(true);
const { witness: last } = parsed(checked);
expect(last).toMatchObject({
stage: "after_reset",
held: true,
status: "held",
});
expect(
last?.record?.actions.map(({ action, outcome }) => [action, outcome]),
).toContainEqual(["reset", "applied"]);
});
it("ends the witness when the seed already passes, and refuses its later stages", async () => {
const passing = {
...TASK,
checks: [
{ kind: "lines_present", file: "Documents/today.txt", lines: ["milk"] },
],
};
const { check, act, directory } = await witnessed(0, {}, passing);
const seed = parsed(await check({ task: passing, stage: "seed", directory }));
expect(seed.witness).toMatchObject({
held: false,
status: "failed",
record: { verdict: "failed", failed_stage: "seed" },
});
// The same seed check returns the record from its own folder, and refuses another.
expect(
parsed(await check({ task: passing, stage: "seed", directory })).witness,
).toEqual(seed.witness);
const elsewhere = `${directory}-elsewhere`;
expect(
parseMcpToolError(
await check({ task: passing, stage: "seed", directory: elsewhere }),
).error.message,
).toContain(`saved to ${directory}, not ${elsewhere}`);
await act("type", { text: "eggs\n" });
expect(
parseMcpToolError(await check({ task: passing, stage: "after_edit" })).error
.message,
).toContain("ended");
});
it("witnesses only the task the bundle holds, by its steps and its checks", async () => {
const passing = {
...TASK,
checks: [
{ kind: "lines_present", file: "Documents/today.txt", lines: ["milk"] },
],
};
const { check, computer, directory } = await witnessed(0, {}, passing);
for (const task of [TASK, { ...passing, prompt: "Keep milk listed." }])
expect(
parseMcpToolError(await check({ task, stage: "seed", directory })).error
.code,
).toBe("invalid_request");
expect(computer.recordingRequests).toEqual([]);
expect(
parsed(await check({ stage: "seed", directory })).witness,
).toMatchObject({ stage: "seed", held: false, status: "failed" });
});
it("refuses a witness without a folder for its record, or one already holding files", async () => {
const { check } = await witnessed();
expect(
parseMcpToolError(await check({ stage: "seed" })).error.message,
).toContain("directory");
const used = await createTestTempDirectory("witness-used-");
await writeFile(join(used, "keep"), "");
expect(
parseMcpToolError(await check({ stage: "seed", directory: used })).error
.message,
).toContain("not empty");
});
it("retries a seed whose state read failed, with the recording it began", async () => {
const witness = await witnessed();
witness.app.failGets = 1;
const failed = await witness.check({
stage: "seed",
directory: witness.directory,
});
expect(parseMcpToolError(failed).error.message).toContain("busy");
const seed = parsed(
await witness.check({ stage: "seed", directory: witness.directory }),
);
expect(seed.witness).toMatchObject({ held: true, next_stage: "after_edit" });
expect(
witness.computer.recordingRequests.filter(({ path }) =>
path.endsWith("/start"),
),
).toHaveLength(1);
});
it("asks for the recording a lost start reply began again under its request id when the seed is retried", async () => {
const witness = await witnessed(0, { lostRecordingStarts: 1 });
const { check, directory, computer } = witness;
const lost = await check({ stage: "seed", directory });
expect(parseMcpToolError(lost).error.message).toContain("recording");
const seed = parsed(await check({ stage: "seed", directory }));
expect(seed.witness).toMatchObject({ held: true, next_stage: "after_edit" });
const starts = computer.recordingRequests.filter(({ path }) =>
path.endsWith("/start"),
);
expect(starts).toHaveLength(2);
expect(starts[1]?.body).toEqual(starts[0]?.body);
});
it("files no seed or near miss whose only failing check could not read its file, and takes it once the file reads", async () => {
const witness = await witnessed();
const { app, check, directory, computer } = witness;
const locked = {
"Documents/today.txt": {
format: null,
view: null,
error: "database is locked",
},
};
app.state = locked;
const unread = ["Documents/today.txt", "database is locked"];
expectUndecided(await check({ stage: "seed", directory }), "seed", unread);
app.state = lines("milk");
expect(
parsed(await check({ stage: "seed", directory })).witness,
).toMatchObject({ stage: "seed", held: true });
await edit(witness, lines("milk", "eggs"));
await check({ stage: "after_edit" });
await edit(witness, locked);
expectUndecided(await check({ stage: "near_miss" }), "near_miss", unread);
app.state = lines("milk", "egg");
const nearMiss = parsed(await check({ stage: "near_miss" }));
expect(nearMiss.witness).toMatchObject({
stage: "near_miss",
held: true,
next_stage: "after_reset",
});
await witness.act("reset");
const record = parsed(await check({ stage: "after_reset" })).witness?.record;
expect(record?.stages.map(({ stage }) => stage)).toEqual(WITNESS_ORDER);
expect(
computer.recordingRequests.filter(({ path }) => path.endsWith("/start")),
).toHaveLength(1);
});
it("files no seed or near miss whose only failing check misses a line its cut view may have left out", async () => {
const witness = await witnessed();
const { app, check, directory } = witness;
const cut = {
"Documents/today.txt": {
format: "text",
view: {
lines: Array.from(
{ length: 5000 },
(_, index) => `note ${String(index + 1)}`,
),
line_count: 5001,
truncated: true,
},
truncated: true,
},
};
app.state = cut;
const missing = ["Documents/today.txt", '"eggs"'];
expectUndecided(await check({ stage: "seed", directory }), "seed", missing);
app.state = lines("milk");
expect(
parsed(await check({ stage: "seed", directory })).witness,
).toMatchObject({ stage: "seed", held: true });
await edit(witness, lines("milk", "eggs"));
await check({ stage: "after_edit" });
await edit(witness, cut);
expectUndecided(await check({ stage: "near_miss" }), "near_miss", missing);
app.state = lines("milk", "egg");
expect(parsed(await check({ stage: "near_miss" })).witness).toMatchObject({
stage: "near_miss",
held: true,
next_stage: "after_reset",
});
});
it("records a long witness from the bounded action log, naming what it dropped and what it cut", async () => {
const witness = await witnessed();
const { app, check, directory } = witness;
await check({ stage: "seed", directory });
for (let index = 0; index < 260; index += 1)
await witness.act("key", { keys: ["Right"] });
await witness.act("type", { text: "e".repeat(2048) });
app.state = lines("milk", "eggs");
expect(parsed(await check({ stage: "after_edit" })).witness).toMatchObject({
held: true,
});
await edit(witness, lines("milk", "egg"));
await check({ stage: "near_miss" });
await witness.act("reset");
const record = parsed(await check({ stage: "after_reset" })).witness?.record;
expect(record).toMatchObject({ verdict: "held", actions_dropped: 7 });
expect(record?.actions).toHaveLength(256);
expect(record?.actions[0]?.index).toBe(7);
expect(
record?.actions.find(({ action }) => action === "type")
?.truncated_arguments,
).toEqual(["text"]);
const { limitations } = parseEvidence(
JSON.parse(await readFile(join(directory, "witness.json"), "utf8")),
);
expect(limitations.join("\n")).toContain(
"the 7 earliest are not in the record",
);
expect(limitations.join("\n")).toContain("truncated_arguments");
});
it("saves a witness record again after a failed save, without checking again", async () => {
const witness = await witnessed();
const { check, directory, app } = witness;
await throughNearMiss(witness);
await witness.act("reset");
// Something else writes into the record's folder, so the save is refused.
await mkdir(directory, { recursive: true });
await writeFile(join(directory, "intruder"), "");
const refused = await check({ stage: "after_reset" });
expect(parseMcpToolError(refused).error.message).toContain("not empty");
await rm(join(directory, "intruder"));
const gets = app.methods.length;
const saved = parsed(await check({ stage: "after_reset" }));
expect(saved.witness).toMatchObject({
stage: "after_reset",
status: "held",
record: { verdict: "held" },
});
expect(saved.witness?.record?.stages).toHaveLength(4);
expect(app.methods.length).toBe(gets);
// Saved, the same check returns the record again without checking or writing; any other is refused.
expect(parsed(await check({ stage: "after_reset" })).witness).toEqual(
saved.witness,
);
expect(app.methods.length).toBe(gets);
expect(
parseMcpToolError(await check({ stage: "near_miss" })).error.message,
).toContain("ended");
});
it("returns the record a witness saved when the computer closes while the save commits", async () => {
const witness = await witnessed();
const { client, id, check, directory } = witness;
await throughNearMiss(witness);
await witness.act("reset");
let closed: Promise<unknown> = Promise.resolve();
cancelDuringCommit(() => {
closed = client.callTool({
name: "close_build_computer",
arguments: { computer_id: id },
});
});
const saved = parsed(await check({ stage: "after_reset" }));
expect(saved.witness?.record).toMatchObject({
path: join(directory, "witness.json"),
verdict: "held",
});
await closed;
});
it("returns a record saved as its check was cancelled when that check is asked again", async () => {
const witness = await witnessed();
const { client, id, directory } = witness;
await throughNearMiss(witness);
await witness.act("reset");
const caller = new AbortController();
cancelDuringCommit(() => {
caller.abort();
});
await expect(
client.callTool(
{
name: "check_build_task",
arguments: { computer_id: id, task: TASK, stage: "after_reset" },
},
{ signal: caller.signal },
),
).rejects.toThrow();
const again = parsed(await witness.check({ stage: "after_reset" }));
expect(again.witness?.record).toMatchObject({
path: join(directory, "witness.json"),
verdict: "held",
});
});
it("files one after_edit when two checks of it arrive together, so the record cannot hold without a near miss", async () => {
const witness = await witnessed();
const { check, directory } = witness;
await check({ stage: "seed", directory });
await edit(witness, lines("milk", "eggs"));
const both = await Promise.all([
check({ stage: "after_edit" }),
check({ stage: "after_edit" }),
]);
const filed = both.filter((result) => result.isError !== true);
expect(filed).toHaveLength(1);
expect(parsed(filed[0])).toMatchObject({
passed: 1,
witness: { stage: "after_edit", held: true, next_stage: "near_miss" },
});
const second = both.find((result) => result.isError === true);
expect(parseMcpToolError(second as CallToolResult).error.message).toContain(
"the next stage is near_miss",
);
await witness.act("reset");
expect(
parseMcpToolError(await check({ stage: "after_reset" })).error.message,
).toContain("the next stage is near_miss");
});
it("keeps a completed reset when the screenshot after it is cancelled", async () => {
const witness = await witnessed(0, { screenshotDelaysMs: [0, 60_000] });
await witness.check({ stage: "seed", directory: witness.directory });
await edit(witness, lines("milk", "eggs"));
await witness.check({ stage: "after_edit" });
await witness.act("type", { text: "near" });
const controller = new AbortController();
const resetting = witness.client.callTool(
{
name: "act_on_build_computer",
arguments: {
computer_id: witness.id,
action: "reset",
screenshot: true,
},
},
{ signal: controller.signal },
);
await vi.waitFor(() => expect(witness.app.methods).toContain("data/reset"));
controller.abort();
await expect(resetting).rejects.toThrow();
const near = parseMcpToolError(await witness.check({ stage: "near_miss" }));
expect(near.error.message).toContain("put the seed back");
expect(witness.app.state).toEqual(lines("milk"));
}, 30_000);
it("checks after_reset once the reset sent before it has relaunched the app", async () => {
const witness = await witnessed(4);
const { app, check } = witness;
await throughNearMiss(witness);
const order: string[] = [];
const reset = witness.act("reset").then(() => order.push("reset"));
await vi.waitFor(() => expect(app.methods.at(-1)).toBe("data/reset"));
const last = await check({ stage: "after_reset" });
order.push("check");
await reset;
expect(order).toEqual(["reset", "check"]);
const record = parsed(last).witness?.record;
expect(record).toMatchObject({ verdict: "held", failed_stage: null });
expect(record?.stages.at(-1)).toMatchObject({
stage: "after_reset",
actions_before: 3,
});
expect(record?.actions.at(-1)).toMatchObject({
action: "reset",
outcome: "applied",
});
expect(app.methods.slice(-2)).toEqual(["data/reset", "data/get"]);
}, 30_000);
it("sends progress while a reset waits for the relaunched app", async () => {
const witness = await witnessed(14);
const progress: unknown[] = [];
const reset = await witness.client.callTool(
{
name: "act_on_build_computer",
arguments: { computer_id: witness.id, action: "reset" },
},
{ onprogress: (update) => progress.push(update) },
);
expect(reset.isError, JSON.stringify(reset)).not.toBe(true);
expect(progress.length).toBeGreaterThan(0);
}, 30_000);
tests/boundary/providers/buildComputerLifecycle.test.ts
+54 -1
View File
@@ -2,7 +2,7 @@ import { chmod, rm, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { setTimeout as delay } from "node:timers/promises";
import { expect, it } from "vitest";
import { expect, it, onTestFinished } from "vitest";
import { fakeBuildComputers as setUp } from "../../fixtures/fakeAgentEnvComputer.js";
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
@@ -70,3 +70,56 @@ it("keeps a failed start whose teardown is unconfirmed, so close and shutdown ca
expect(await computers.close(id)).toMatchObject({ ok: false });
expect(await runner.notes("teardowns.jsonl")).toHaveLength(1);
});
it("waits for a command the computer answers after fetch's wait for headers ends", async () => {
const home = await createTestTempDirectory("fake-exec-home-");
const { computers, start } = await setUp({ exec: { home, bin: home } });
onTestFinished(() => computers.closeAll());
const started = await start();
if (!started.ok) throw started.error;
const steps = computers.steps(started.value.value.computer_id);
if (steps === undefined) throw new Error("the started computer is held");
// fetch's dispatcher with undici's 300 s wait for headers cut to 200 ms, which fires after about 1 s.
const key = Symbol.for("undici.globalDispatcher.1");
const global = globalThis as Record<symbol, unknown>;
const dispatcher = global[key] as {
readonly constructor: new (options: object) => unknown;
};
global[key] = new dispatcher.constructor({ headersTimeout: 200 });
try {
expect(await steps.computer.exec(["sleep", "3"], 10)).toMatchObject({
exit_code: 0,
timed_out: false,
});
} finally {
global[key] = dispatcher;
}
});
it("cancels a call in progress and the one waiting behind it when the owner closes", async () => {
const app = {
name: "notes",
state: {},
seed: {},
methods: [],
unreadyAfterReset: 600,
};
const { computers, runner, start } = await setUp({ app });
const started = await start();
if (!started.ok) throw started.error;
const act = (action: "reset" | "screenshot") =>
computers.act(
{ computer_id: started.value.value.computer_id, action },
new AbortController().signal,
);
const reset = act("reset");
while (app.methods.length === 0) await delay(50);
const waiting = act("screenshot");
await computers.closeAll();
for (const result of await Promise.all([reset, waiting]))
expect(result).toMatchObject({
ok: false,
error: { _tag: "AnalysisCancelledError" },
});
expect(await runner.notes("teardowns.jsonl")).toHaveLength(1);
});
tests/fixtures/buildCaptureFakes.ts
Vendored
+121
View File
@@ -0,0 +1,121 @@
import { writeFile } from "node:fs/promises";
import { join } from "node:path";
import { expect } from "vitest";
import {
EDITED_NOTE,
seedEndCapture,
writeCaptureFolder,
} from "../../src/application/desktop/desktopCapture.fixture.js";
import type { DesktopPackage } from "../../src/domain/desktop/desktopCapture.js";
import { serveCli } from "../boundary/cli/cliServeHarness.js";
import { buildComputerFakes, connectBuildServer } from "./buildComputerMcp.js";
import { fakeBuildComputerRea } from "./fakeBuildComputerRea.js";
import { createTestTempDirectory } from "./temporaryDirectory.js";
/** The task the agent writes for the seed-and-end capture. */
export const BUILD_TASK = {
name: "add-eggs",
prompt: "Add eggs to today's list and save it.",
checks: [
{ kind: "lines_present", file: "Documents/today.txt", lines: ["eggs"] },
],
};
/** A request for an app from the Ubuntu archive. */
export const BUILD_REQUEST = {
kind: "request",
desktop_entry: "org.example.notes",
install: { kind: "archive", package: "notes" },
};
const ENDS = ["--seed-checkpoint", "seed", "--end-checkpoint", "end"];
/** The app's package as the capture records it, and the URL the export downloads its file from, if it needs one. */
export interface CapturedInstall {
readonly package: DesktopPackage;
readonly packageUrl?: string;
}
/** What the real CLI writes for a capture made between seed and end: the draft, and the export with BUILD_TASK. */
const realCliOutputs = async (install: CapturedInstall | undefined) => {
const recorded = seedEndCapture();
const capture = await writeCaptureFolder(
install === undefined
? recorded
: {
...recorded,
application: { ...recorded.application, package: install.package },
},
undefined,
[EDITED_NOTE],
);
const work = await createTestTempDirectory("build-cli-");
const task = join(work, "task.json");
await writeFile(task, JSON.stringify(BUILD_TASK));
const bundle = join(work, "bundle");
const draft = await serveCli([
"describe-desktop-application-task",
capture,
...ENDS,
"--json",
]);
const exported = await serveCli([
"export-desktop-application",
capture,
bundle,
"--task",
task,
...ENDS,
...(install?.packageUrl === undefined
? []
: ["--package-url", install.packageUrl]),
"--json",
]);
expect(exported.exitCode, exported.stdout).toBe(0);
await writeFile(join(work, "draft.json"), draft.stdout);
await writeFile(join(work, "export.json"), exported.stdout);
return {
capture,
draft: join(work, "draft.json"),
export: join(work, "export.json"),
bundle,
};
};
/**
* A REA server whose build computers run REA's controller on this host
* against a REA stand-in printing the real CLI's outputs. `dropReplies`
* names controller commands, in order, whose reply is lost after they run;
* `env` adds to the stand-in's environment; `install` replaces the captured
* package, the archive's `notes`; `screenshotDelaysMs` delays the computer's
* screenshots in order.
*/
export const buildCaptureFakes = async (
options: {
readonly dropReplies?: string[];
readonly env?: Readonly<Record<string, string>>;
readonly install?: CapturedInstall;
readonly screenshotDelaysMs?: readonly number[];
} = {},
) => {
const outputs = await realCliOutputs(options.install);
const rea = await fakeBuildComputerRea(outputs);
const { computer, runner } = await buildComputerFakes({
...(options.screenshotDelaysMs === undefined
? {}
: { screenshotDelaysMs: options.screenshotDelaysMs }),
exec: {
home: rea.home,
bin: rea.bin,
env: { ...rea.env, ...options.env },
dropReplies: options.dropReplies ?? [],
},
});
const payload = join(runner.state, "rea-prod.tar.xz");
await writeFile(payload, "a packed REA build");
const client = await connectBuildServer({
...runner.environment,
REA_BUILD_COMPUTER_REA_ARCHIVE: payload,
});
return { rea, outputs, computer, client };
};
tests/fixtures/buildComputerMcp.ts
Vendored
+107
View File
@@ -0,0 +1,107 @@
import { mkdir, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { Client, InMemoryTransport } from "@modelcontextprotocol/client";
import type { CallToolResult } from "@modelcontextprotocol/server";
import { expect, onTestFinished } from "vitest";
import { toolContract } from "../../src/contracts/toolContracts.js";
import { createServer } from "../../src/server/createServer.js";
import { createTestBinarySession } from "./binarySession.js";
import {
fakeAgentEnvRunner,
startFakeComputer,
type FakeAgentEnvRunner,
type FakeComputer,
type FakeComputerOptions,
} from "./fakeAgentEnvComputer.js";
import { createTestTempDirectory } from "./temporaryDirectory.js";
/** The host's environment without the settings the opt-in real test exports, which would reach past the fakes. */
const hostEnvironment = Object.fromEntries(
Object.entries(process.env).filter(
([name]) =>
![
"REA_AGENT_ENV_COMMAND",
"REA_DOCKER_COMMAND",
"REA_BUILD_COMPUTER_REA_ARCHIVE",
].includes(name),
),
);
/** An MCP client of a REA server whose build computers reach the fakes through `environment`. */
export const connectBuildServer = async (
environment: Readonly<Record<string, string>>,
): Promise<Client> => {
const session = createTestBinarySession(() => {
throw new Error("build computers must not start a deep provider");
});
const server = createServer(
{ kind: "session", session },
{
environment: {
...hostEnvironment,
REA_MCP_INPUT_SCHEMA_PROFILE: "full",
...environment,
},
},
);
const client = new Client({ name: "build-computer-mcp", version: "1" });
onTestFinished(async () => {
await client.close();
await server.close();
await session.close();
});
const [clientTransport, serverTransport] =
InMemoryTransport.createLinkedPair();
await server.connect(serverTransport);
await client.connect(clientTransport);
return client;
};
/** A fake computer and the fake runner that hands it out. */
export const buildComputerFakes = async (
options: FakeComputerOptions = {},
): Promise<{ computer: FakeComputer; runner: FakeAgentEnvRunner }> => {
const computer = await startFakeComputer(options);
onTestFinished(() => computer.close());
const runner = await fakeAgentEnvRunner(computer);
return { computer, runner };
};
/** An exported bundle's folder as far as the fake runner reads it; given a fake computer, the runner hands that one out for it. */
export const bundleFolder = async (
computer?: Pick<FakeComputer, "url">,
): Promise<string> => {
const bundle = await createTestTempDirectory("exported-bundle-");
await mkdir(join(bundle, "tasks"));
await writeFile(join(bundle, "tasks", "smoke.json"), "[]\n");
if (computer !== undefined)
await writeFile(join(bundle, "fake-computer-url"), computer.url);
return bundle;
};
/** Call a tool and fail the test with its result when it is an error. */
export const callOk = async (
client: Client,
name: string,
args: Record<string, unknown>,
): Promise<CallToolResult> => {
const result = await client.callTool({ name, arguments: args });
expect(result.isError, JSON.stringify(result)).not.toBe(true);
return result as CallToolResult;
};
/** Start a build computer and return its id. */
export const startBuildComputer = async (
client: Client,
source: Record<string, unknown>,
): Promise<string> =>
toolContract("start_build_computer").outputSchema.parse(
(await callOk(client, "start_build_computer", { source }))
.structuredContent,
).computer_id;
/** No delivered text, success or failure, may hold the control token. */
export const expectNoToken = (computer: FakeComputer, result: unknown) =>
expect(JSON.stringify(result)).not.toContain(computer.token);
tests/fixtures/fakeAgentEnvComputer.ts
Vendored
+267 -13
View File
@@ -1,17 +1,27 @@
import { execFileSync } from "node:child_process";
import { execFileSync, spawn } from "node:child_process";
import { createHash } from "node:crypto";
import { access, chmod, mkdir, readFile, writeFile } from "node:fs/promises";
import {
access,
chmod,
mkdir,
readFile,
readdir,
stat,
writeFile,
} from "node:fs/promises";
import {
createServer,
type IncomingMessage,
type ServerResponse,
} from "node:http";
import type { AddressInfo } from "node:net";
import { join } from "node:path";
import { join, relative } from "node:path";
import { setTimeout as delay } from "node:timers/promises";
import {
McpServer,
WebStandardStreamableHTTPServerTransport,
type ServerContext,
} from "@modelcontextprotocol/server";
import { z } from "zod";
@@ -20,9 +30,12 @@ import { onTestFinished } from "vitest";
import { DesktopBuildComputers } from "../../src/application/desktop/DesktopBuildComputers.js";
import { createTestTempDirectory } from "./temporaryDirectory.js";
/** A 1x1 PNG, the fake driver's every frame. */
/** A 2x1 PNG of a black and a white pixel, the fake driver's drawn desktop. */
export const FAKE_FRAME_PNG =
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==";
"iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAIAAAB7QOjdAAAAD0lEQVR42mNgYGD4//8/AAYBAv67yYXpAAAAAElFTkSuQmCC";
/** A 2x1 PNG of two black pixels, a display the desktop has not drawn on yet. */
const FAKE_BLANK_PNG =
"iVBORw0KGgoAAAANSUhEUgAAAAIAAAABCAIAAAB7QOjdAAAAC0lEQVR42mNgAAMAAAcAASEi2xMAAAAASUVORK5CYII=";
/** The fake recording's segments, in order, as the driver keeps them under `rec-000000000001/`. */
export const FAKE_SEGMENTS = [
Buffer.from("not really an mp4 segment"),
@@ -37,10 +50,26 @@ export interface FakeComputerOptions {
readonly notReadyPolls?: number;
/** Readiness reports a final failed check instead. */
readonly readinessFails?: boolean;
/** Base64 characters in each screenshot's image, in place of a 1x1 PNG. */
/** Base64 characters in each screenshot's image, in place of a drawn PNG. */
readonly frameCharacters?: number;
/** Screenshots answered with one black frame before the desktop is drawn. */
readonly blankFrames?: number;
/** How long each screenshot, in order, waits before its reply, unless its request is cancelled. */
readonly screenshotDelaysMs?: readonly number[];
/** Bytes data/get appends to each segment beyond its manifest size. */
readonly segmentPadding?: number;
/** Recording starts, in order, whose recording begins but whose reply is lost. */
readonly lostRecordingStarts?: number;
/** Run computer-exec commands on this host in `home`, with `bin` first on PATH and `env` added. */
readonly exec?: {
readonly home: string;
readonly bin: string;
readonly env?: Readonly<Record<string, string>>;
/** Controller commands, in order, whose command runs but whose reply is lost. */
readonly dropReplies?: string[];
};
/** The one app the card lists. */
readonly app?: FakeApp;
}
const COMPUTER_TOOLS = [
@@ -53,6 +82,22 @@ const COMPUTER_TOOLS = [
"computer_wait",
] as const;
/** The one app a fake computer lists, serving data/get with its state views and data/reset to its seed. */
export interface FakeApp {
readonly name: string;
/** The views data/get answers now; a test sets them as the app's state changes. */
state: Record<string, unknown>;
readonly seed: Record<string, unknown>;
/** The data methods the app was asked, in order. */
readonly methods: string[];
/** data/get calls still to answer with an error, as a busy app would. */
failGets?: number;
/** Readiness polls answered not ready after each data/reset, as a slow relaunch would. */
readonly unreadyAfterReset?: number;
/** Readiness polls left before the relaunched app is ready. */
unready?: number;
}
/** A local stand-in for an agentenv computer's driver: readiness, `computer_*` MCP tools, recording and data/get. */
export interface FakeComputer {
readonly url: string;
@@ -66,6 +111,8 @@ export interface FakeComputer {
readonly dataRequests: unknown[];
/** Refuse the next call of a tool as the driver refuses one, with its message. */
refuseNext(name: string, message: string): void;
/** Run the next call of a tool but lose its reply, as a dropped connection would. */
loseNext(name: string): void;
close(): Promise<void>;
}
@@ -90,25 +137,38 @@ const startFakeDriver = async (
options.frameCharacters === undefined
? FAKE_FRAME_PNG
: "A".repeat(options.frameCharacters);
let blankFrames = options.blankFrames ?? 0;
let screenshots = 0;
const mcp = new McpServer({ name: "fake-reference-driver", version: "1" });
for (const name of COMPUTER_TOOLS)
mcp.registerTool(
name,
{ description: name, inputSchema: z.looseObject({}) },
(args: Record<string, unknown>) => {
async (args: Record<string, unknown>, context: ServerContext) => {
calls.push({ name, arguments: args });
if (name === "computer_screenshot") {
screenshots += 1;
const wait = options.screenshotDelaysMs?.[screenshots - 1] ?? 0;
if (wait > 0)
await delay(wait, undefined, {
signal: context.mcpReq.signal,
ref: false,
}).catch(() => undefined);
}
const refusal = refusals.get(name);
if (refusal !== undefined) {
refusals.delete(name);
return { content: [{ type: "text", text: refusal }], isError: true };
}
const frame = { frame_id: `f${String(calls.length)}`, width: 1280 };
const blank = name === "computer_screenshot" && blankFrames > 0;
if (blank) blankFrames -= 1;
const image =
name === "computer_screenshot" || args.screenshot === true
? [
{
type: "image" as const,
data: frameData,
data: blank ? FAKE_BLANK_PNG : frameData,
mimeType: "image/png",
},
]
@@ -130,11 +190,173 @@ const startFakeDriver = async (
return { mcp, transport };
};
/** Every file under home-relative paths, as the driver's data plane answers data/get. */
const homeFileParts = async (home: string, paths: readonly string[]) => {
const parts: unknown[] = [];
const visit = async (path: string): Promise<void> => {
const entry = await stat(path).catch(() => undefined);
if (entry?.isDirectory() === true)
for (const name of (await readdir(path)).sort())
await visit(join(path, name));
else if (entry?.isFile() === true)
parts.push({
kind: "file",
file: { bytes: (await readFile(path)).toString("base64") },
metadata: {
"urn:agentenv:data-objects/v1": { path: relative(home, path) },
},
});
};
for (const path of paths) await visit(join(home, path));
return parts;
};
/** The most of a command's stdout the computer's computer-exec answers with. */
const EXEC_STDOUT_BYTES = 1024 * 1024;
/** Run one computer-exec command on this host as the computer would: in the home, its stdin closed, its stdout cut at the cap. */
const runExec = (
exec: NonNullable<FakeComputerOptions["exec"]>,
token: string,
argv: readonly string[],
) =>
new Promise<Record<string, unknown>>((resolve, reject) => {
const [command = "", ...rest] = argv;
const child = spawn(command, rest, {
cwd: exec.home,
env: {
...process.env,
...exec.env,
HOME: exec.home,
PATH: `${exec.bin}:${process.env.PATH ?? ""}`,
AGENTENV_CONTROL_TOKEN: token,
},
stdio: ["ignore", "pipe", "pipe"],
});
const stdout: Buffer[] = [];
let stderr = "";
child.stdout.on("data", (chunk: Buffer) => stdout.push(chunk));
child.stderr.on("data", (chunk: Buffer) => (stderr += String(chunk)));
child.once("error", reject);
child.once("close", (code) => {
const output = Buffer.concat(stdout);
resolve({
exit_code: code,
stdout: output.subarray(0, EXEC_STDOUT_BYTES).toString("utf8"),
stderr,
truncated: output.length > EXEC_STDOUT_BYTES,
timed_out: false,
});
});
});
/** A reply the fake loses after doing the work, as a dropped connection would. */
const LOST = Symbol("lost reply");
/** A JSON-RPC request calling one of the driver's tools. */
const toolCall = z.object({
method: z.literal("tools/call"),
params: z.object({ name: z.string() }),
});
/** One of the app's data methods. */
const appRoute = (app: FakeApp, request: { id?: number; method?: string }) => {
const method = request.method ?? "";
app.methods.push(method);
if (method === "data/get" && (app.failGets ?? 0) > 0) {
app.failGets = (app.failGets ?? 0) - 1;
return {
jsonrpc: "2.0",
id: request.id,
error: { code: -32000, message: "the app is busy", data: {} },
};
}
if (method === "data/reset") {
app.state = structuredClone(app.seed);
app.unready = app.unreadyAfterReset ?? 0;
}
return {
jsonrpc: "2.0",
id: request.id,
result:
method === "data/get"
? {
parts: [
{ kind: "file", file: { bytes: "eA==" } },
{ kind: "data", data: { state: app.state } },
],
}
: {},
};
};
/** Not ready while the app relaunches after a reset; undefined once it is, for the usual readiness. */
const relaunching = (app: FakeApp | undefined) => {
if (app === undefined || (app.unready ?? 0) === 0) return undefined;
app.unready = (app.unready ?? 0) - 1;
return { ready: false, checks: [{ name: app.name, ok: false }] };
};
/** One command run as the computer would, its reply lost when the test asks. */
const execRoute = async (
exec: NonNullable<FakeComputerOptions["exec"]>,
token: string,
argv: readonly string[],
) => {
const answer = await runExec(exec, token, argv);
const { dropReplies } = exec;
if (dropReplies === undefined || dropReplies[0] !== argv[4]) return answer;
dropReplies.shift();
return LOST;
};
/** The card, the app's data methods, exec and home files, when the fake serves them; undefined for other routes. */
const buildRoute = async (
options: FakeComputerOptions,
token: string,
path: string,
parsed: unknown,
): Promise<unknown> => {
const { app, exec } = options;
const request = parsed as {
id?: number;
method?: string;
argv?: string[];
params?: { paths?: string[] };
};
if (path === "/.well-known/agent-env.json")
return {
name: "fake-computer",
children_environments:
app === undefined
? []
: [{ name: app.name, url: `/svc/${app.name}/agentenv` }],
};
if (app !== undefined && path === `/svc/${app.name}/agentenv`)
return appRoute(app, request);
if (path === "/agentenv/ext/computer-ready/status") return relaunching(app);
if (exec === undefined) return undefined;
if (path === "/agentenv/ext/computer-exec/run")
return execRoute(exec, token, request.argv ?? []);
const paths = request.params?.paths ?? [];
// Recordings keep the canned segment; other paths are the home's files.
if (
path !== "/agentenv" ||
(paths[0] ?? ".agentenv-recordings/").startsWith(".agentenv-recordings/")
)
return undefined;
return {
jsonrpc: "2.0",
id: request.id,
result: { parts: await homeFileParts(exec.home, paths) },
};
};
interface FakeState {
readonly options: FakeComputerOptions;
readonly recordingRequests: FakeComputer["recordingRequests"];
readonly dataRequests: unknown[];
polls: number;
lostStarts: number;
}
const segments = FAKE_SEGMENTS.map((bytes, index) => ({
@@ -169,6 +391,10 @@ const frameworkRoute = (
}
if (path.startsWith("/agentenv/ext/computer-recording/")) {
state.recordingRequests.push({ path, body: parsed });
if (path.endsWith("/start") && state.lostStarts > 0) {
state.lostStarts -= 1;
return LOST;
}
return path.endsWith("/start")
? {
recording_id: "rec-000000000001",
@@ -224,8 +450,10 @@ export const startFakeComputer = async (
recordingRequests: [],
dataRequests: [],
polls: 0,
lostStarts: options.lostRecordingStarts ?? 0,
};
const refusals = new Map<string, string>();
const lostReplies = new Set<string>();
const { mcp, transport } = await startFakeDriver(calls, refusals, options);
const server = createServer((request, response) => {
void (async () => {
@@ -247,6 +475,13 @@ export const startFakeComputer = async (
...(body.length === 0 ? {} : { body }),
}),
);
const called = toolCall.safeParse(
body.length === 0 ? null : JSON.parse(String(body)),
);
if (called.success && lostReplies.delete(called.data.params.name)) {
response.destroy();
return;
}
response.writeHead(answer.status, Object.fromEntries(answer.headers));
response.end(Buffer.from(await answer.arrayBuffer()));
return;
@@ -257,7 +492,11 @@ export const startFakeComputer = async (
return;
}
const parsed: unknown = body.length === 0 ? {} : JSON.parse(String(body));
sendJson(response, 200, frameworkRoute(state, url.pathname, parsed));
const answer =
(await buildRoute(options, token, url.pathname, parsed)) ??
frameworkRoute(state, url.pathname, parsed);
if (answer === LOST) response.destroy();
else sendJson(response, 200, answer);
})().catch((cause: unknown) => {
response.destroy(cause instanceof Error ? cause : undefined);
});
@@ -272,6 +511,7 @@ export const startFakeComputer = async (
recordingRequests: state.recordingRequests,
dataRequests: state.dataRequests,
refuseNext: (name, message) => refusals.set(name, message),
loseNext: (name) => lostReplies.add(name),
close: async () => {
await mcp.close();
server.closeAllConnections();
@@ -282,8 +522,11 @@ export const startFakeComputer = async (
/**
* A stand-in `agent_env.bundle.run_bundle`, which REA's real helper imports:
* it notes each call, checks the bundle's Python compiles, copies the bundle
* and writes to stdout as a build would, then hands out the fake computer.
* it notes each call, loads each of the bundle's Python files in a child
* Python as agentenv's `_declare` does, which compiles it and caches its
* bytecode beside it unless Python is told not to, copies the bundle and
* writes to stdout as a build would, then hands out the fake computer, or
* the one the bundle's `fake-computer-url` names.
*/
const FAKE_BUNDLE_MODULE = String.raw`
import glob
@@ -291,11 +534,19 @@ import json
import os
import shutil
import signal
import subprocess
import sys
import time
import types
STATE = os.environ["FAKE_STATE"]
DECLARE = """
import importlib.util, sys
folder, file = sys.argv[1], sys.argv[2]
sys.path.insert(0, folder)
spec = importlib.util.spec_from_file_location("computer_driver_module", folder + "/" + file)
spec.loader.get_code(spec.name)
"""
def _note(name, value):
@@ -324,8 +575,7 @@ def run_bundle(root, *, tasks=(), keep=False, emulate=True, sandbox=None, on_pro
_note("runs.jsonl", {"root": str(root), "tasks": list(tasks), "keep": keep, "emulate": emulate,
"sandbox": sandbox})
for path in glob.glob(os.path.join(root, "envs", "*", "*.py")):
with open(path) as handle:
compile(handle.read(), path, "exec")
subprocess.run([sys.executable, "-c", DECLARE, os.path.dirname(path), os.path.basename(path)], check=True)
shutil.copytree(root, os.path.join(STATE, "bundle"), dirs_exist_ok=True)
os.system("echo a build writing to stdout")
print("agentenv writing to stdout")
@@ -339,6 +589,10 @@ def run_bundle(root, *, tasks=(), keep=False, emulate=True, sandbox=None, on_pro
print(f"E: {os.environ['FAKE_FAIL']}", file=sys.stderr, flush=True)
raise RuntimeError(os.environ["FAKE_FAIL"])
url = os.environ["FAKE_COMPUTER_URL"]
named = os.path.join(root, "fake-computer-url")
if os.path.exists(named):
with open(named) as handle:
url = handle.read().strip()
env = types.SimpleNamespace(environment_url=url, mcp_url=url + "/mcp", sandbox_id="sbx-fake",
control_token=os.environ["FAKE_COMPUTER_TOKEN"])
run = types.SimpleNamespace(context=types.SimpleNamespace(deployed_envs=[env]), error=None, failed_steps=[])
tests/fixtures/fakeBuildComputerRea.ts
Vendored
+172
View File
@@ -0,0 +1,172 @@
import { execFileSync } from "node:child_process";
import { access, chmod, mkdir, readFile, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { createTestTempDirectory } from "./temporaryDirectory.js";
/**
* A stand-in for REA inside a build computer: `capture-desktop-application`
* reads its requests from stdin and acknowledges each as REA's capture does,
* then publishes a prepared capture folder on stop; `describe-...` and
* `export-...` print outputs the real CLI wrote for that folder, and the
* export copies the bundle it wrote. Each call notes its arguments, its
* working folder and which AGENTENV_ variables and APT_CONFIG it saw.
*/
const FAKE_REA = String.raw`
import json
import os
import shutil
import sys
NOTES = os.environ["FAKE_REA_NOTES"]
def note(name, value):
with open(os.path.join(NOTES, name), "a") as handle:
handle.write(json.dumps(value) + "\n")
def progress(message, event=None):
line = {"rea_progress": {"phase": "desktop_capture", "message": message}}
if event is not None:
line["rea_progress"]["event"] = event
sys.stderr.write(json.dumps(line) + "\n")
sys.stderr.flush()
def option(args, name):
return args[args.index(name) + 1] if name in args else None
command, args = sys.argv[1], sys.argv[2:]
note("calls.jsonl", {"argv": sys.argv[1:], "cwd": os.getcwd(), "apt_config": os.environ.get("APT_CONFIG"),
"agentenv": sorted(name for name in os.environ if name.startswith("AGENTENV_"))})
if command == "capture-desktop-application":
progress("Recorded checkpoint baseline.", {"kind": "checkpoint", "name": "baseline", "trigger": "baseline",
"at_ms": 0, "entries": 4, "changes": 0, "retained_files": 1,
"retained_bytes": 17})
progress("The application window is open.", {"kind": "window_open"})
taken = set()
for line in sys.stdin:
text = line.strip()
if text == "stop":
break
name = text[len("checkpoint "):].strip()
if name in taken:
progress("refused", {"kind": "checkpoint_refused", "name": name,
"reason": f"A checkpoint named {name} was already recorded."})
continue
taken.add(name)
progress(f"Recorded checkpoint {name}.", {"kind": "checkpoint", "name": name, "trigger": "request",
"at_ms": 1000 * len(taken), "entries": 5, "changes": 1,
"retained_files": 1, "retained_bytes": 5})
note("stopped.jsonl", {"pid": os.getpid()})
out = option(args, "--output")
shutil.copytree(os.environ["FAKE_REA_CAPTURE"], out)
with open(os.path.join(out, "capture.json")) as handle:
sys.stdout.write(handle.read())
elif command == "describe-desktop-application-task":
with open(os.environ["FAKE_REA_DRAFT"]) as handle:
sys.stdout.write(handle.read())
elif command == "export-desktop-application":
with open(option(args, "--task")) as handle:
note("tasks.jsonl", json.load(handle))
exported = os.environ["FAKE_REA_EXPORT"]
with open(exported) as handle:
output = handle.read()
if "code" not in json.loads(output):
shutil.copytree(os.environ["FAKE_REA_BUNDLE"], args[1])
sys.stdout.write(output)
sys.exit(0 if "code" not in json.loads(output) else 1)
else:
sys.exit(f"unexpected command {command}")
`;
/** apt-get update as an ordinary user under APT_CONFIG: one Packages list in the folder it names, before failing when FAKE_APT_PARTIAL is set. */
const FAKE_APT_GET = String.raw`
import os
import re
import sys
with open(os.environ["APT_CONFIG"]) as handle:
conf = handle.read()
lists = re.search(r'Dir::State::Lists "([^"]+)"', conf).group(1)
code = int(os.environ.get("FAKE_APT_EXIT", "0"))
if os.environ.get("FAKE_APT_EMPTY") and 'APT::Update::Error-Mode "any"' in conf and not os.environ.get("FAKE_APT_IGNORE_ERROR_MODE"):
print("Err:1 http://archive.ubuntu.com/ubuntu noble InRelease")
print("E: Failed to fetch http://archive.ubuntu.com/ubuntu/dists/noble/InRelease Temporary failure resolving 'archive.ubuntu.com'")
sys.exit(100)
if os.environ.get("FAKE_APT_EMPTY"):
sys.exit(0)
if code != 0 and os.environ.get("FAKE_APT_PARTIAL"):
open(os.path.join(lists, "archive.ubuntu.com_ubuntu_dists_noble_main_binary-amd64_Packages.lz4"), "w").close()
if code != 0:
print("Err:1 http://archive.ubuntu.com/ubuntu noble InRelease")
print("E: Failed to fetch http://archive.ubuntu.com/ubuntu/dists/noble/InRelease Temporary failure resolving 'archive.ubuntu.com'")
sys.exit(code)
open(os.path.join(lists, "archive.ubuntu.com_ubuntu_dists_noble_universe_binary-amd64_Packages.lz4"), "w").close()
open(os.path.join(lists, "archive.ubuntu.com_ubuntu_dists_noble_InRelease"), "w").close()
print("Reading package lists... Done")
sys.exit(0)
`;
/** Prepared outputs of the real CLI for the fake REA to print. */
export interface FakeReaOutputs {
readonly capture: string;
readonly draft: string;
readonly export: string;
readonly bundle: string;
}
export interface FakeBuildComputerRea {
readonly home: string;
readonly bin: string;
readonly env: Readonly<Record<string, string>>;
notes(name: string): Promise<unknown[]>;
}
/** A computer home and REA stand-in whose capture, draft and export answer with `outputs`. */
export const fakeBuildComputerRea = async (
outputs: FakeReaOutputs,
): Promise<FakeBuildComputerRea> => {
const root = await createTestTempDirectory("fake-build-rea-");
const home = join(root, "home");
const bin = join(root, "bin");
const notes = join(root, "notes");
await Promise.all([home, bin, notes].map((path) => mkdir(path)));
const python = execFileSync(
"python3",
["-c", "import sys; print(sys.executable)"],
{ encoding: "utf8" },
).trim();
for (const [name, source] of [
["rea", FAKE_REA],
["apt-get", FAKE_APT_GET],
] as const) {
await writeFile(join(bin, name), `#!${python}\n${source}`);
await chmod(join(bin, name), 0o755);
}
return {
home,
bin,
env: {
FAKE_REA_NOTES: notes,
FAKE_REA_CAPTURE: outputs.capture,
FAKE_REA_DRAFT: outputs.draft,
FAKE_REA_EXPORT: outputs.export,
FAKE_REA_BUNDLE: outputs.bundle,
},
notes: async (name) => {
const path = join(notes, name);
try {
await access(path);
} catch {
return [];
}
return (await readFile(path, "utf8"))
.trim()
.split("\n")
.map((line) => JSON.parse(line) as unknown);
},
};
};
tests/fixtures/outputTreeCommit.ts
Vendored
+25
View File
@@ -0,0 +1,25 @@
import { expect, onTestFinished, vi } from "vitest";
import { SafeOutputTree } from "../../src/artifacts/SafeOutputTree.js";
/**
* Call `cancel` once an output tree has published, inside the commit that
* published it, and hold the commit until its signal aborts: a cancellation
* that arrives while the commit still closes its handles.
*/
export const cancelDuringCommit = (cancel: () => void): void => {
const commit = SafeOutputTree.prototype.commit;
const spy = vi
.spyOn(SafeOutputTree.prototype, "commit")
.mockImplementation(async function (
this: SafeOutputTree,
signal?: AbortSignal,
) {
await commit.call(this, signal);
cancel();
await vi.waitFor(() => expect(signal?.aborted).toBe(true));
});
onTestFinished(() => {
spy.mockRestore();
});
};