mirror of
https://github.com/codewhale-hq/Codewhale.git
synced 2026-11-07 10:36:05 -10:00
plugins(a3): computer-use is one Node plugin — delete the Python server
Founder ruling R3: codewhale-cu is the canonical computer-use plugin, and it ships inside the product rather than as a sibling repository. This merges /Volumes/VIXinSSD/CW/codewhale-cu into plugins/computer-use in the engine's real bundle format (Agent Plugins v1 plugin.json + sibling mcp.json, net.codewhale extension) — the format the incoming tree already targeted. Deleted, with reasons: - plugins/computer-use/server/server.py + test_server.py (697 lines): a stdlib-only macOS/Linux MCP server offering screenshot, click, type_text, press_key, screen_size — and no scroll on macOS. The Node implementation supersedes it on every axis (38 tools, four platforms, accessibility-first observation with pixel fallback, recording, ssh/hdc remote computers), so keeping both would ship two computer-use servers behind one plugin name. - The invented codewhale-cu.json packaging format and install.mjs never come across: the engine discovers, reviews, and installs the bundle itself. - Smoke receipts stay out of the tree (gitignored) — they are per-run local evidence, not source. The bundle's own suites now run in CI beside the chat bridges, which already run dependency-free `node --test` bundles; they cover the manifest contract, the registry, the exec/ssh transport, the four backends, and the MCP stdio protocol. The in-repo Rust guard test moves with the bundle: node/server.mjs instead of python3/server.py, both skills present, and every mcp.json arg proven to resolve inside the plugin root. The codewhale-apps binding (packages/desktop-use, PR #181, 40033659) is repointed deliberately in the same slice, per R3: it imports <plugin>/src/transport.mjs, which the merge preserves, so only the default checkout path changes (CODEWHALE_CU_PLUGIN_PATH overrides it). That repo's suite is green at 7/7 against the new location. Storage and protocol identifiers are untouched on purpose: the ~/.codewhale-cu state directory, the remote agent path, CODEWHALE_CU_STATE_DIR, and the MCP serverInfo name all keep their spelling — those change only through an explicit migration. Gates: cargo fmt --all -- --check clean; cargo clippy --workspace --all-targets --locked -D warnings clean; actionlint on ci.yml reports the same 12 pre-existing shellcheck notes as HEAD (all in codes CI ignores) and none in the new step. Lanes: tui 11656 passed/13 skipped, tui-integration 280/0, tui-cucumber 16/0, cli 339/0, config 634 passed/1 skipped, lane 67/0. Plugin suite: 31 tests run, 31 passed, 0 failed. Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_013r9fgwt1HYymrB7iZGSkCm
This commit is contained in:
CodeWhale BotandClaude Opus 5 committed
1 parent
19f25eede3
commit
594fcc103c
29 files changed
+3984
-738
No files matched your search
@@ -245,6 +245,12 @@ jobs:
|
||||
(cd "integrations/${bridge}" && npm test)
|
||||
done
|
||||
|
||||
- name: Run computer-use plugin suites
|
||||
# The bundled plugin is dependency-free too; its suites cover the
|
||||
# manifest contract, the registry, the exec/ssh transport, the four
|
||||
# platform backends, and the MCP stdio protocol. No GUI input runs.
|
||||
run: (cd plugins/computer-use && npm test)
|
||||
|
||||
safety-gate:
|
||||
name: Safety gate
|
||||
needs: changes
|
||||
|
||||
@@ -159,3 +159,6 @@ CODEWHALE_0_9_0_CUTOVER.md
|
||||
|
||||
# Local playwright MCP artifacts
|
||||
.playwright-mcp/
|
||||
|
||||
# computer-use plugin smoke receipts (per-run local evidence)
|
||||
plugins/computer-use/receipts/
|
||||
@@ -2312,7 +2312,18 @@ args = ["server.js", "--mode=worker", "-e", "console.log('ready')"]
|
||||
let validated = PluginManifest::validate_from_path(&root.join("plugin.json"))
|
||||
.expect("in-repo computer-use bundle must validate");
|
||||
assert_eq!(validated.manifest.plugin.name, "computer-use");
|
||||
// One declared skills *root* (`skills/`), which holds both skills.
|
||||
assert_eq!(validated.inventory.skills, 1);
|
||||
let skills_root = validated.components.skills.first().expect("skills root");
|
||||
let mut skill_dirs: Vec<String> = fs::read_dir(skills_root)
|
||||
.unwrap()
|
||||
.map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned())
|
||||
.collect();
|
||||
skill_dirs.sort();
|
||||
assert_eq!(skill_dirs, ["computer-use", "recording"]);
|
||||
for skill in &skill_dirs {
|
||||
assert!(skills_root.join(skill).join("SKILL.md").is_file());
|
||||
}
|
||||
assert_eq!(validated.components.commands.len(), 1);
|
||||
assert!(validated.warnings.is_empty(), "{:?}", validated.warnings);
|
||||
|
||||
@@ -2320,11 +2331,20 @@ args = ["server.js", "--mode=worker", "-e", "console.log('ready')"]
|
||||
let servers =
|
||||
agent_plugin::parse_mcp_json(&mcp_text).expect("computer-use mcp.json must parse");
|
||||
let computer = servers.get("computer").expect("computer server");
|
||||
assert_eq!(computer.command.as_deref(), Some("python3"));
|
||||
assert_eq!(computer.command.as_deref(), Some("node"));
|
||||
assert!(
|
||||
computer.args.iter().any(|arg| arg.ends_with("server.py")),
|
||||
computer.args.iter().any(|arg| arg.ends_with("server.mjs")),
|
||||
"{:?}",
|
||||
computer.args
|
||||
);
|
||||
// The entrypoint must stay inside the bundle: the engine launches it
|
||||
// with `cwd` set to the plugin root.
|
||||
for arg in &computer.args {
|
||||
assert!(
|
||||
!Path::new(arg).is_absolute() && !arg.split('/').any(|part| part == ".."),
|
||||
"{arg} must stay inside the bundle"
|
||||
);
|
||||
assert!(root.join(arg).is_file(), "{arg} must exist in the bundle");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,105 @@
|
||||
# computer-use
|
||||
|
||||
Codewhale Computer Use — the `computer-use` plugin. One tool surface, four
|
||||
platforms, and **switching between registered computers as a default**: every
|
||||
tool accepts `computer`, and using a computer id sticks until you switch.
|
||||
|
||||
The bundle is an Agent Plugins v1 package (`plugin.json` + sibling `mcp.json`,
|
||||
`commands/`, `skills/`): the Codewhale Engine discovers, reviews, installs, and
|
||||
runs it. Nothing here writes to your Codewhale configuration.
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| Platforms | macOS, Windows, Linux (X11 + Wayland), HarmonyOS (hdc devices) |
|
||||
| Hosts | macOS, Windows, Linux (`when.os`); HarmonyOS is a target device, not a host |
|
||||
| Transports | local process, ssh + bundled remote agent, hdc |
|
||||
| Runtime deps | none (Node ≥ 20; platform tools probed at call time) |
|
||||
| Tools | 38: observe, pointer, keyboard/text, semantic, clipboard, recording, computer registry |
|
||||
|
||||
## Frontier ability set
|
||||
|
||||
- **Observe & resolve** — `list_apps`, `list_windows`, `list_displays`,
|
||||
`switch_display`, `get_app_state` (accessibility/UIA/uitest tree with
|
||||
element indices + `state_id`), `screenshot` (display/region, raster-bound
|
||||
coordinates), `zoom` (close-up crop of the last raster), `cursor_position`,
|
||||
`open_application` (exact-name rule), `request_access` (fail-closed
|
||||
permission/capability probe).
|
||||
- **Pointer** — left/double/triple/right/middle click, move, drag,
|
||||
down/up, scroll (4 directions).
|
||||
- **Keyboard & text** — `type` (unicode), `key` (chords + repeat),
|
||||
`hold_key`, `set_value` (semantic, background-safe), `select_text`,
|
||||
`perform_action` (element's own actions: AXPress / UIA Invoke / AT-SPI / uitest).
|
||||
- **Recording** — `recording_start/stop/status/list` (see below).
|
||||
- **Computers** — `computer_list`, `computer_switch`, `computer_register`
|
||||
(ssh agent auto-push), `computer_remove`.
|
||||
- **Safety** — `stop_computer_control` kill switch; permission probes that
|
||||
name the missing grant; receipts on every call naming the computer it
|
||||
happened on.
|
||||
|
||||
## Requirements
|
||||
|
||||
Tools and permissions are probed at call time; `request_access` reports what is
|
||||
missing and every capability **fails closed naming the missing tool or
|
||||
permission** — it never guesses and never half-acts.
|
||||
|
||||
- **macOS** — Accessibility + Screen Recording permission for the terminal
|
||||
app that hosts the Engine (System Settings → Privacy & Security).
|
||||
python3+pyobjc or cliclick improves cursor reads. ffmpeg optional (mp4 remux).
|
||||
- **Windows** — PowerShell (built in); ffmpeg for recording.
|
||||
- **Linux** — X11: xdotool, wmctrl, scrot or imagemagick, xclip; Wayland:
|
||||
grim, wtype, ydotool+ydotoold, wl-clipboard, wf-recorder; python3-pyatspi
|
||||
for the accessibility tree; ffmpeg for recording on X11.
|
||||
- **HarmonyOS** — `hdc` on PATH with the device connected
|
||||
(`hdc list targets`); ffmpeg on the host for snapshot-series recordings.
|
||||
|
||||
## How the four platforms map
|
||||
|
||||
| Ability | macOS | Windows | Linux | HarmonyOS |
|
||||
|---|---|---|---|---|
|
||||
| Accessibility tree | AX via System Events (JXA) | UIAutomation | AT-SPI (pyatspi) | `uitest dumpLayout` |
|
||||
| Raw input | CGEvent (JXA bridge) | user32 SendInput/mouse_event (PowerShell) | xdotool (X11) / ydotool+wtype (Wayland) | `uitest uiInput` |
|
||||
| Screenshots | `screencapture` | .NET CopyFromScreen | scrot/import (X11), grim (Wayland) | `snapshot_display` |
|
||||
| Recording | `screencapture -v` → .mov, ffmpeg remux to .mp4 | ffmpeg gdigrab | ffmpeg x11grab / wf-recorder | snapshot-series + ffmpeg mux |
|
||||
| Clipboard | pbcopy/pbpaste | Get/Set-Clipboard | xclip/xsel, wl-clipboard | fail-closed (not exposed by hdc) |
|
||||
|
||||
## Remote computers (ssh)
|
||||
|
||||
```json
|
||||
computer_register { "computer": "winbox", "transport": "ssh", "host": "winbox.lan", "user": "me" }
|
||||
```
|
||||
|
||||
Registration pushes the self-contained agent (`agent.mjs` + `src/`) to
|
||||
`~/.codewhale-cu/agent/` on the remote over scp, probes the remote platform
|
||||
through it, and pins the result. Remote calls run
|
||||
`node agent.mjs <base64 json>` — one JSON receipt line back. Only an
|
||||
allow-listed tool set executes remotely; arguments travel as data, never as
|
||||
shell. Requires publickey ssh (BatchMode) and Node ≥ 20 on the remote.
|
||||
|
||||
## HarmonyOS computers
|
||||
|
||||
```json
|
||||
computer_register { "computer": "pad", "transport": "hdc" }
|
||||
```
|
||||
|
||||
Drives the device over `hdc shell uitest ...` and `snapshot_display`. Element
|
||||
targets come from `dumpLayout`; input is touch-synthesis (click / swipe /
|
||||
inputText / keyEvent). Recording is honestly labeled `snapshot-series`
|
||||
(frame captures muxed on stop) because HarmonyOS exposes no CLI screen
|
||||
recorder.
|
||||
|
||||
## Development
|
||||
|
||||
The bundle lives at `plugins/computer-use` in the Codewhale repository and has
|
||||
no dependencies to install; run its suites from that directory.
|
||||
|
||||
```bash
|
||||
npm test # unit + protocol tests (no GUI input performed)
|
||||
npm run smoke # live end-to-end against this machine (isolated state dirs,
|
||||
# no clicks/typing into your session, no clipboard access)
|
||||
```
|
||||
|
||||
Smoke receipts land in `receipts/` with per-check pass/fail and artifact
|
||||
paths. Proven levels are separated: local live (this Mac: darwin) > mocked
|
||||
transport (ssh protocol, harmony backend logic) > code-complete (win32/linux
|
||||
paths, implemented to their documented tool interfaces but only verifiable on
|
||||
those platforms).
|
||||
@@ -0,0 +1,51 @@
|
||||
// codewhale-cu remote agent — runs on an ssh-registered computer.
|
||||
// One-shot: `node agent.mjs <base64(json)>` prints exactly one JSON receipt
|
||||
// line. The request is {"tool": "...", "args": {...}}; only tools in this
|
||||
// allow-list execute, so the transport can never become a generic shell.
|
||||
import { exec } from "./src/remote-runtime.mjs";
|
||||
|
||||
const ALLOWED = new Set([
|
||||
"platform", "probe", "list_displays", "switch_display", "list_apps", "list_windows",
|
||||
"open_application", "get_app_state", "screenshot", "zoom",
|
||||
"left_click", "double_click", "triple_click", "right_click", "middle_click",
|
||||
"mouse_move", "left_click_drag", "left_mouse_down", "left_mouse_up", "scroll",
|
||||
"type", "key", "hold_key", "set_value", "select_text", "perform_action",
|
||||
"read_clipboard", "write_clipboard", "cursor_position",
|
||||
"recordingStart", "recordingStop", "recordingStatus", "recordingList",
|
||||
]);
|
||||
|
||||
function reply(obj) {
|
||||
process.stdout.write(JSON.stringify(obj) + "\n");
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const argv = process.argv.slice(2);
|
||||
if (!argv[0]) reply({ ok: false, error: { code: "missing_payload", message: "usage: node agent.mjs <base64 payload>" } });
|
||||
|
||||
let req;
|
||||
try {
|
||||
req = JSON.parse(Buffer.from(argv[0], "base64").toString("utf8"));
|
||||
} catch {
|
||||
reply({ ok: false, error: { code: "bad_payload", message: "payload is not base64 JSON" } });
|
||||
}
|
||||
|
||||
const tool = req?.tool;
|
||||
if (!ALLOWED.has(tool)) {
|
||||
reply({ ok: false, error: { code: "tool_not_allowed", message: `tool "${tool}" is not in the remote allow-list` } });
|
||||
}
|
||||
|
||||
try {
|
||||
if (tool === "platform") {
|
||||
reply({ ok: true, platform: process.platform });
|
||||
}
|
||||
const mod = await import(`./src/backends/${process.platform}.mjs`);
|
||||
const backend = mod.create({ exec, computer: { id: "remote", transport: "local", platform: process.platform } });
|
||||
const fn = backend[tool];
|
||||
if (typeof fn !== "function") {
|
||||
reply({ ok: false, error: { code: "unsupported_on_platform", message: `"${tool}" is not implemented on ${process.platform}` } });
|
||||
}
|
||||
const data = await fn(req.args ?? {});
|
||||
reply({ ok: true, platform: process.platform, tool, data });
|
||||
} catch (err) {
|
||||
reply({ ok: false, platform: process.platform, tool, error: { code: err?.code ?? "tool_error", message: String(err?.message ?? err) } });
|
||||
}
|
||||
@@ -1,13 +1,19 @@
|
||||
---
|
||||
description: Drive this machine's screen, mouse, and keyboard
|
||||
usage: /computer [status|look]
|
||||
description: Drive a computer's screen, mouse, and keyboard
|
||||
usage: /computer [status|look|computers]
|
||||
---
|
||||
|
||||
$ARGUMENTS
|
||||
|
||||
- With no arguments or `status`: report whether the computer-use
|
||||
plugin is usable here — server reachable, platform supported,
|
||||
screen size — without taking any action.
|
||||
- With `look`: take one screenshot and describe what is on screen.
|
||||
- Anything else (clicking, typing, operating apps) goes through the
|
||||
computer-use skill's look-act-verify loop with per-action approval.
|
||||
- With no arguments or `status`: report whether computer use is usable on
|
||||
the active computer — server reachable, platform, screen size, and which
|
||||
permissions or platform tools are missing (`request_access`) — without
|
||||
taking any action.
|
||||
- With `look`: take one screenshot of the active computer and describe what
|
||||
is on screen.
|
||||
- With `computers`: list registered computers (`computer_list`) and say which
|
||||
one is active; every tool also takes `computer` to switch.
|
||||
- Anything else (clicking, typing, operating apps, recording) goes through
|
||||
the computer-use skill's observe-act-verify loop with per-action approval.
|
||||
A denied permission or a missing tool fails closed and is reported, never
|
||||
retried blindly.
|
||||
@@ -1 +1,16 @@
|
||||
{"mcpServers": {"computer": {"args": ["server/server.py"], "command": "python3", "cwd": ".", "type": "stdio"}}}
|
||||
{
|
||||
"$schema": "https://agent-plugins.org/schemas/mcp.json",
|
||||
"mcpServers": {
|
||||
"computer": {
|
||||
"type": "stdio",
|
||||
"command": "node",
|
||||
"args": ["mcp/server.mjs"],
|
||||
"cwd": ".",
|
||||
"extensions": {
|
||||
"net.codewhale": {
|
||||
"execute_timeout": 120
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,330 @@
|
||||
#!/usr/bin/env node
|
||||
// codewhale-cu MCP server — zero-dependency JSON-RPC 2.0 over stdio.
|
||||
// One tool surface, four platforms (darwin, win32, linux, harmonyos), with
|
||||
// computer switching as a default: every tool accepts `computer`, and using a
|
||||
// computer id switches the sticky active computer.
|
||||
import * as registry from "../src/registry.mjs";
|
||||
import { backendFor, installRemoteAgent, executorFor } from "../src/transport.mjs";
|
||||
import { TOOLS, TOOL_NAMES, READ_ONLY_TOOLS, REMOTE_TOOLS, BACKEND_METHOD } from "../src/tools.mjs";
|
||||
import { tryJson } from "../src/exec.mjs";
|
||||
|
||||
const VERSION = "0.1.0";
|
||||
const SERVER_NAME = "codewhale-cu";
|
||||
|
||||
// ---------- per-session runtime state ----------
|
||||
let controlStopped = false;
|
||||
let stateCounter = 0;
|
||||
/** state_id -> { computerId, app_ref, windowIndex, elements } */
|
||||
const appStates = new Map();
|
||||
/** computerId -> last raster metadata {file, scale, origin} */
|
||||
const lastRasters = new Map();
|
||||
/** computerId -> cached backend (local/hdc only) */
|
||||
const backendCache = new Map();
|
||||
|
||||
function receipt(computer, extra) {
|
||||
return {
|
||||
computer: computer ? { id: computer.id, transport: computer.transport, platform: computer.platform ?? computer.platformHint ?? null } : null,
|
||||
ts: new Date().toISOString(),
|
||||
...extra,
|
||||
};
|
||||
}
|
||||
|
||||
function fail(computer, code, message, extra = {}) {
|
||||
return receipt(computer, { ok: false, error: { code, message }, ...extra });
|
||||
}
|
||||
|
||||
async function getBackend(computer) {
|
||||
const key = computer.id;
|
||||
if (computer.transport === "local" || computer.transport === "hdc") {
|
||||
if (!backendCache.has(key)) {
|
||||
const { backend } = await backendFor(computer);
|
||||
backendCache.set(key, backend);
|
||||
}
|
||||
return backendCache.get(key);
|
||||
}
|
||||
const { backend } = await backendFor(computer);
|
||||
return backend;
|
||||
}
|
||||
|
||||
/** Element target -> enriched target with cached app identity and AX path. */
|
||||
function resolveElement(target) {
|
||||
const st = appStates.get(target.state_id);
|
||||
if (!st) throw new ServerError("unknown_state", `state_id "${target.state_id}" is unknown or expired — call get_app_state again`);
|
||||
const el = st.elements[target.index];
|
||||
if (!el) throw new ServerError("unknown_element", `element index ${target.index} is outside state ${target.state_id} (0..${st.elements.length - 1})`);
|
||||
return { state: st, element: el };
|
||||
}
|
||||
|
||||
class ServerError extends Error {
|
||||
constructor(code, message) { super(message); this.code = code; }
|
||||
}
|
||||
|
||||
/** Map raster-pixel coordinates to screen points using the bound raster. */
|
||||
function rasterToPoints(computerId, x, y) {
|
||||
const r = lastRasters.get(computerId);
|
||||
if (!r) throw new ServerError("no_raster", "no screenshot bound on this computer yet — call screenshot first so pixel targets have a frame");
|
||||
const scale = r.scale && r.scale > 0 ? r.scale : 1;
|
||||
return { x: (r.origin?.x ?? 0) + x / scale, y: (r.origin?.y ?? 0) + y / scale };
|
||||
}
|
||||
|
||||
/** Normalize a target into backend form: points for coordinates, resolved element for elements. */
|
||||
function normalizeTarget(computer, target, kind) {
|
||||
if (target?.type === "coordinate") {
|
||||
const pt = rasterToPoints(computer.id, target.x, target.y);
|
||||
return { x: Math.round(pt.x), y: Math.round(pt.y), strategy: "event" };
|
||||
}
|
||||
if (target?.type === "element") {
|
||||
const { state, element } = resolveElement(target);
|
||||
if (kind === "semantic") {
|
||||
return {
|
||||
app_ref: state.app_ref, windowIndex: element.windowIndex ?? 0, path: element.path,
|
||||
strategy: "a11y", role: element.role, label: element.label,
|
||||
};
|
||||
}
|
||||
// Pointer tools on element targets: aim at the element center (cached geometry).
|
||||
if (!element.position || !element.size) throw new ServerError("element_no_geometry", `element ${target.index} has no cached geometry — use a coordinate target`);
|
||||
const c = { x: Math.round(element.position.x + element.size.w / 2), y: Math.round(element.position.y + element.size.h / 2) };
|
||||
return { ...c, strategy: "a11y-center", role: element.role, label: element.label };
|
||||
}
|
||||
throw new ServerError("bad_target", "target must be {type:'coordinate',x,y} or {type:'element',state_id,index}");
|
||||
}
|
||||
|
||||
function bindRaster(computer, shot) {
|
||||
lastRasters.set(computer.id, {
|
||||
file: shot.file ?? shot.path,
|
||||
scale: shot.scale ?? 1,
|
||||
origin: shot.points ?? { x: 0, y: 0 },
|
||||
pixels: shot.pixels ?? null,
|
||||
capturedAt: shot.capturedAt ?? new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
|
||||
function rememberState(computer, app_ref, result) {
|
||||
const id = `s-${++stateCounter}`;
|
||||
appStates.set(id, { computerId: computer.id, app_ref, elements: result.elements ?? [], ts: Date.now() });
|
||||
if (appStates.size > 24) {
|
||||
for (const k of appStates.keys()) { appStates.delete(k); break; }
|
||||
}
|
||||
return id;
|
||||
}
|
||||
|
||||
// ---------- tool dispatch ----------
|
||||
async function callTool(params) {
|
||||
const name = params.name;
|
||||
if (!TOOL_NAMES.has(name)) {
|
||||
return { content: [{ type: "text", text: JSON.stringify({ ok: false, error: { code: "unknown_tool", message: `unknown tool "${name}"` } }) }], isError: true };
|
||||
}
|
||||
const args = params.arguments ?? {};
|
||||
|
||||
if (name === "stop_computer_control") {
|
||||
controlStopped = true;
|
||||
return { content: [{ type: "text", text: JSON.stringify(receipt(null, { ok: true, stopped: true, note: "Computer control refused for the rest of this session. Restart the session or the codewhale-cu server to continue." })) }] };
|
||||
}
|
||||
if (controlStopped && !READ_ONLY_TOOLS.has(name)) {
|
||||
return { content: [{ type: "text", text: JSON.stringify(fail(null, "control_stopped", "stop_computer_control is active; no further actions are permitted this session")) }], isError: true };
|
||||
}
|
||||
|
||||
if (name === "wait") {
|
||||
const s = Math.max(0, Math.min(30, Number(args.seconds) || 1));
|
||||
await new Promise((r) => setTimeout(r, s * 1000));
|
||||
return { content: [{ type: "text", text: JSON.stringify(receipt(null, { ok: true, waitedSec: s })) }] };
|
||||
}
|
||||
|
||||
if (name === "computer_list") {
|
||||
const reg = registry.list();
|
||||
return { content: [{ type: "text", text: JSON.stringify(receipt(null, {
|
||||
ok: true,
|
||||
active: reg.active,
|
||||
computers: Object.values(reg.computers).map((c) => ({ id: c.id, transport: c.transport, platform: c.platform ?? c.platformHint ?? null, label: c.label ?? null, host: c.host ?? null })),
|
||||
note: "Pass `computer` on any tool to switch (sticky), or computer_switch to switch explicitly.",
|
||||
})) }] };
|
||||
}
|
||||
|
||||
if (name === "computer_register") {
|
||||
try {
|
||||
const entry = registry.register({ id: args.computer, transport: args.transport, label: args.label, host: args.host, port: args.port, user: args.user, target: args.target });
|
||||
let installed = null;
|
||||
if (entry.transport === "ssh" && args.installAgent !== false) {
|
||||
installed = await installRemoteAgent(entry);
|
||||
registry.register({ id: entry.id, transport: "ssh", host: entry.host, port: entry.port, user: entry.user, platformHint: installed.remotePlatform, agentPath: installed.agentPath });
|
||||
}
|
||||
if (entry.transport === "ssh" && args.installAgent === false && !entry.platformHint) {
|
||||
// Probe cheaply through the agent; if it is missing, registration still succeeds.
|
||||
try {
|
||||
const ex = await executorFor(entry);
|
||||
const reply = await ex.remote({ tool: "platform" });
|
||||
registry.register({ id: entry.id, transport: "ssh", host: entry.host, port: entry.port, user: entry.user, platformHint: reply.platform });
|
||||
} catch {}
|
||||
}
|
||||
const fresh = registry.get(entry.id);
|
||||
return { content: [{ type: "text", text: JSON.stringify(receipt(null, { ok: true, registered: { ...fresh, platform: fresh.platform ?? fresh.platformHint ?? null }, agentInstall: installed })) }] };
|
||||
} catch (err) {
|
||||
// Registration problems (unreachable host, agent push failed) are
|
||||
// receipts, not protocol errors.
|
||||
return { content: [{ type: "text", text: JSON.stringify(fail(null, err.code ?? "register_failed", err.message ?? String(err))) }], isError: true };
|
||||
}
|
||||
}
|
||||
|
||||
if (name === "computer_remove") {
|
||||
const res = registry.remove(args.computer);
|
||||
backendCache.delete(args.computer);
|
||||
lastRasters.delete(args.computer);
|
||||
return { content: [{ type: "text", text: JSON.stringify(receipt(null, { ok: true, ...res })) }] };
|
||||
}
|
||||
|
||||
if (name === "computer_switch") {
|
||||
const c = registry.switchTo(args.computer);
|
||||
return { content: [{ type: "text", text: JSON.stringify(receipt(c, { ok: true, active: c.id })) }] };
|
||||
}
|
||||
|
||||
// Everything below acts on a computer.
|
||||
let computer;
|
||||
let switched = false;
|
||||
try {
|
||||
if (args.computer && args.computer !== registry.list().active) {
|
||||
computer = registry.switchTo(args.computer);
|
||||
switched = true;
|
||||
} else {
|
||||
computer = registry.active();
|
||||
}
|
||||
} catch (err) {
|
||||
return { content: [{ type: "text", text: JSON.stringify(fail(null, err.code ?? "registry_error", err.message)) }], isError: true };
|
||||
}
|
||||
|
||||
try {
|
||||
// ssh computers: dispatch over the wire to the remote agent.
|
||||
const backendMethod = BACKEND_METHOD[name] === "request_access" ? "probe" : BACKEND_METHOD[name];
|
||||
let data;
|
||||
|
||||
if (computer.transport === "ssh" && REMOTE_TOOLS.has(backendMethod)) {
|
||||
const ex = await executorFor(computer);
|
||||
const wireArgs = prepareWireArgs(computer, name, args);
|
||||
const reply = await ex.remote({ tool: backendMethod, args: wireArgs }, { timeoutMs: backendMethod.startsWith("recording") || backendMethod === "get_app_state" ? 60_000 : 30_000 });
|
||||
if (!reply.ok) throw new ServerError(reply.error?.code ?? "remote_error", reply.error?.message ?? "remote agent failed");
|
||||
data = reply.data;
|
||||
if (Array.isArray(data)) data = { items: data };
|
||||
if (backendMethod === "screenshot" && data?.file) {
|
||||
// Raster lives on the remote machine; bind geometry for coordinate mapping.
|
||||
bindRaster(computer, { ...data, file: null });
|
||||
data.note = "file lives on the remote computer; pull it with scp if you need the bytes locally";
|
||||
}
|
||||
} else {
|
||||
const backend = await getBackend(computer);
|
||||
if (typeof backend[backendMethod] !== "function") {
|
||||
throw new ServerError("unsupported_on_backend", `"${name}" is not implemented on the ${computer.platform ?? computer.transport} backend`);
|
||||
}
|
||||
const prepared = prepareLocalArgs(computer, name, args);
|
||||
data = await backend[backendMethod](prepared);
|
||||
if (Array.isArray(data)) data = { items: data }; // keep receipts objects
|
||||
if (name === "screenshot") bindRaster(computer, data);
|
||||
if (name === "get_app_state") {
|
||||
const stateId = rememberState(computer, prepared.app_ref, data);
|
||||
data.state_id = stateId;
|
||||
data.note = "Element targets are {type:'element', state_id, index}. State goes stale when the UI changes; observe again.";
|
||||
}
|
||||
}
|
||||
|
||||
return { content: [{ type: "text", text: JSON.stringify(receipt(computer, { ok: true, tool: name, switched, ...data })) }] };
|
||||
} catch (err) {
|
||||
return { content: [{ type: "text", text: JSON.stringify(fail(computer, err.code ?? "tool_error", err.message ?? String(err), { tool: name, switched })) }], isError: true };
|
||||
}
|
||||
}
|
||||
|
||||
/** Convert public tool args into local-backend args (targets normalized). */
|
||||
function prepareLocalArgs(computer, name, args) {
|
||||
const out = { ...args };
|
||||
delete out.computer;
|
||||
const semantic = new Set(["set_value", "select_text", "perform_action"]);
|
||||
if (out.target?.type) {
|
||||
out.target = normalizeTarget(computer, out.target, semantic.has(name) ? "semantic" : "pointer");
|
||||
}
|
||||
if (out.from_target?.type) out.from_target = normalizeTarget(computer, out.from_target, "pointer");
|
||||
if (out.to?.type) out.to = normalizeTarget(computer, out.to, "pointer");
|
||||
if (name === "get_app_state") {
|
||||
out.app_ref = out.app_ref ?? null;
|
||||
if (out.window_id != null) out.window_id = Number(out.window_id);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Convert public tool args into remote-agent args (self-contained, element targets resolved to paths). */
|
||||
function prepareWireArgs(computer, name, args) {
|
||||
const out = { ...args };
|
||||
delete out.computer;
|
||||
const semantic = new Set(["set_value", "select_text", "perform_action"]);
|
||||
if (out.target?.type === "element") {
|
||||
const t = normalizeTarget(computer, out.target, semantic.has(name) ? "semantic" : "pointer");
|
||||
out.target = { ...out.target, app_ref: t.app_ref, windowIndex: t.windowIndex, path: t.path, x: t.x, y: t.y };
|
||||
}
|
||||
if (out.from_target?.type === "element") {
|
||||
const t = normalizeTarget(computer, out.from_target, "pointer");
|
||||
out.from_target = { ...out.from_target, x: t.x, y: t.y };
|
||||
}
|
||||
if (out.to?.type === "element") {
|
||||
const t = normalizeTarget(computer, out.to, "pointer");
|
||||
out.to = { ...out.to, x: t.x, y: t.y };
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------- JSON-RPC loop ----------
|
||||
function respond(id, result) {
|
||||
process.stdout.write(JSON.stringify({ jsonrpc: "2.0", id, result }) + "\n");
|
||||
}
|
||||
function respondError(id, code, message) {
|
||||
process.stdout.write(JSON.stringify({ jsonrpc: "2.0", id, error: { code, message } }) + "\n");
|
||||
}
|
||||
|
||||
const HANDLERS = {
|
||||
initialize(params) {
|
||||
return {
|
||||
protocolVersion: params?.protocolVersion ?? "2025-06-18",
|
||||
capabilities: { tools: { listChanged: false } },
|
||||
serverInfo: { name: SERVER_NAME, version: VERSION, platforms: ["darwin", "win32", "linux", "harmonyos"], transports: ["local", "ssh", "hdc"] },
|
||||
};
|
||||
},
|
||||
"tools/list"() {
|
||||
return { tools: TOOLS };
|
||||
},
|
||||
async "tools/call"(params) {
|
||||
return await callTool(params ?? {});
|
||||
},
|
||||
ping() {
|
||||
return {};
|
||||
},
|
||||
};
|
||||
|
||||
let buffer = "";
|
||||
process.stdin.setEncoding("utf8");
|
||||
process.stdin.on("data", (chunk) => {
|
||||
buffer += chunk;
|
||||
let idx;
|
||||
while ((idx = buffer.indexOf("\n")) !== -1) {
|
||||
const line = buffer.slice(0, idx).trim();
|
||||
buffer = buffer.slice(idx + 1);
|
||||
if (!line) continue;
|
||||
handleLine(line);
|
||||
}
|
||||
});
|
||||
process.stdin.on("end", () => process.exit(0));
|
||||
|
||||
async function handleLine(line) {
|
||||
const msg = tryJson(line, null);
|
||||
if (!msg || typeof msg !== "object") return;
|
||||
const { id, method, params } = msg;
|
||||
if (!method) return; // response to a server request — we never issue any
|
||||
const handler = HANDLERS[method];
|
||||
if (!handler) {
|
||||
if (id != null) respondError(id, -32601, `method not found: ${method}`);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const result = await handler(params);
|
||||
if (id != null) respond(id, result);
|
||||
} catch (err) {
|
||||
if (id != null) respondError(id, -32603, err?.message ?? String(err));
|
||||
}
|
||||
}
|
||||
|
||||
// Notifications we must tolerate
|
||||
["notifications/initialized", "initialized", "notifications/cancelled"].forEach((m) => { if (!HANDLERS[m]) HANDLERS[m] = () => ({}); });
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"name": "codewhale-cu",
|
||||
"version": "0.1.0",
|
||||
"description": "Codewhale Computer Use plugin: frontier computer-use abilities (accessibility-first control, screenshots, zoom, recording) on macOS, Windows, Linux, and HarmonyOS, with multi-computer switching as a default. Zero runtime dependencies.",
|
||||
"license": "MIT",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"main": "./mcp/server.mjs",
|
||||
"scripts": {
|
||||
"test": "node --test tests/*.test.mjs",
|
||||
"smoke": "node scripts/smoke.mjs",
|
||||
"smoke:protocol": "node scripts/smoke-protocol.mjs"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
}
|
||||
@@ -1 +1,27 @@
|
||||
{"$schema": "https://agent-plugins.org/schemas/plugin.json", "author": {"name": "codewhale"}, "description": "See the screen, click, and type: local computer use over an MCP server plus the look-act-verify skill.", "extensions": {"net.codewhale": {"capabilities": {}, "commands": {"path": "commands"}, "when": {"binaries": ["python3"], "os": ["macos", "linux"]}}}, "keywords": ["computer-use", "screenshot", "automation"], "license": "MIT", "name": "computer-use", "version": "0.1.0"}
|
||||
{
|
||||
"$schema": "https://agent-plugins.org/schemas/plugin.json",
|
||||
"name": "computer-use",
|
||||
"version": "0.1.0",
|
||||
"description": "See the screen and operate it: accessibility-first control, screenshots, zoom, and screen recording on macOS, Windows, and Linux, plus registered remote computers over ssh and HarmonyOS devices over hdc.",
|
||||
"author": { "name": "Codewhale" },
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"computer-use",
|
||||
"automation",
|
||||
"screenshot",
|
||||
"recording",
|
||||
"accessibility",
|
||||
"harmonyos"
|
||||
],
|
||||
"extensions": {
|
||||
"net.codewhale": {
|
||||
"display_name": "Computer Use",
|
||||
"commands": { "path": "commands" },
|
||||
"skills": { "path": "skills" },
|
||||
"when": {
|
||||
"binaries": ["node"],
|
||||
"os": ["macos", "linux", "windows"]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
// Live smoke test: drives the real MCP server end-to-end on this machine.
|
||||
// Policy: no destructive input (no clicks/typing into the user's session),
|
||||
// no clipboard access, isolated state + recordings dirs.
|
||||
import { spawn } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import url from "node:url";
|
||||
|
||||
const __dirname = path.dirname(url.fileURLToPath(import.meta.url));
|
||||
const ROOT = path.resolve(__dirname, "..");
|
||||
const results = [];
|
||||
|
||||
function log(name, pass, detail = "") {
|
||||
results.push({ name, pass, detail: String(detail).slice(0, 500) });
|
||||
console.log(`${pass ? "PASS" : "FAIL"} ${name}${detail ? ` — ${detail}` : ""}`);
|
||||
}
|
||||
|
||||
const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "cu-smoke-state-"));
|
||||
const recDir = fs.mkdtempSync(path.join(os.tmpdir(), "cu-smoke-rec-"));
|
||||
|
||||
const server = spawn("node", [path.join(ROOT, "mcp", "server.mjs")], {
|
||||
env: { ...process.env, CODEWHALE_CU_STATE_DIR: stateDir, CODEWHALE_CU_RECORDINGS_DIR: recDir },
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
});
|
||||
let buf = "";
|
||||
const pending = new Map();
|
||||
let nextId = 1;
|
||||
server.stdout.setEncoding("utf8");
|
||||
server.stdout.on("data", (d) => {
|
||||
buf += d;
|
||||
let i;
|
||||
while ((i = buf.indexOf("\n")) !== -1) {
|
||||
const line = buf.slice(0, i).trim();
|
||||
buf = buf.slice(i + 1);
|
||||
if (!line) continue;
|
||||
try {
|
||||
const msg = JSON.parse(line);
|
||||
if (msg.id && pending.has(msg.id)) {
|
||||
pending.get(msg.id)(msg);
|
||||
pending.delete(msg.id);
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
});
|
||||
server.stderr.on("data", (d) => process.stderr.write(`[server] ${d}`));
|
||||
|
||||
function rpc(method, params, timeoutMs = 120_000) {
|
||||
const id = nextId++;
|
||||
return new Promise((resolve, reject) => {
|
||||
const t = setTimeout(() => { pending.delete(id); reject(new Error(`timeout waiting for ${method}`)); }, timeoutMs);
|
||||
pending.set(id, (msg) => { clearTimeout(t); resolve(msg); });
|
||||
server.stdin.write(JSON.stringify({ jsonrpc: "2.0", id, method, params }) + "\n");
|
||||
});
|
||||
}
|
||||
|
||||
async function tool(name, args = {}) {
|
||||
const res = await rpc("tools/call", { name, arguments: args });
|
||||
const text = res.result?.content?.[0]?.text ?? "{}";
|
||||
let parsed = null;
|
||||
try { parsed = JSON.parse(text); } catch {}
|
||||
return { raw: res.result, parsed, isError: res.result?.isError === true };
|
||||
}
|
||||
|
||||
try {
|
||||
// --- protocol ---
|
||||
const init = await rpc("initialize", { protocolVersion: "2025-06-18", capabilities: {} });
|
||||
log("initialize", !!init.result?.serverInfo?.name, `server=${init.result?.serverInfo?.name} v${init.result?.serverInfo?.version}`);
|
||||
await rpc("notifications/initialized", undefined, 5_000).catch(() => {});
|
||||
const tl = await rpc("tools/list", {});
|
||||
const tools = tl.result?.tools ?? [];
|
||||
log("tools/list", tools.length >= 38, `${tools.length} tools`);
|
||||
const names = new Set(tools.map((t) => t.name));
|
||||
for (const required of ["screenshot", "recording_start", "recording_stop", "computer_switch", "computer_list", "get_app_state", "left_click", "type", "key", "scroll", "zoom", "stop_computer_control"]) {
|
||||
if (!names.has(required)) log(`schema:${required}`, false, "missing");
|
||||
}
|
||||
log("schema:required-tools-present", true, "all key tools declared");
|
||||
|
||||
// --- registry / switching ---
|
||||
let r = await tool("computer_list");
|
||||
log("computer_list", r.parsed?.ok === true && r.parsed?.computers?.length >= 1, `active=${r.parsed?.active}`);
|
||||
|
||||
r = await tool("computer_register", { computer: "pad", transport: "hdc", label: "Harmony device" });
|
||||
log("computer_register(hdc)", r.parsed?.ok === true, JSON.stringify(r.parsed?.registered ?? r.parsed?.error));
|
||||
|
||||
r = await tool("computer_switch", { computer: "pad" });
|
||||
log("computer_switch", r.parsed?.ok === true && r.parsed?.active === "pad", `active=${r.parsed?.active}`);
|
||||
|
||||
r = await tool("list_apps");
|
||||
log("harmony fail-closed (no hdc device)", r.parsed?.ok === false, `code=${r.parsed?.error?.code}`);
|
||||
|
||||
r = await tool("screenshot", { computer: "local" });
|
||||
log("switch-by-use (computer:local on screenshot)", r.parsed?.ok === true && r.parsed?.switched === true && r.parsed?.computer?.id === "local", `file=${path.basename(r.parsed?.file ?? "")}`);
|
||||
|
||||
// --- local darwin live tools ---
|
||||
r = await tool("request_access");
|
||||
log("request_access/probe", r.parsed?.ok === true, `accessibility=${r.parsed?.permissions?.accessibility} capture=${r.parsed?.permissions?.screen_capture}`);
|
||||
|
||||
r = await tool("list_displays");
|
||||
log("list_displays", r.parsed?.ok === true && r.parsed?.items?.length >= 1, JSON.stringify(r.parsed?.items?.[0] ?? r.parsed?.error));
|
||||
|
||||
r = await tool("list_apps");
|
||||
log("list_apps", r.parsed?.ok === true && r.parsed?.apps?.length > 0, `${r.parsed?.apps?.length} apps`);
|
||||
|
||||
const someApp = r.parsed?.apps?.find((a) => a.windowCount > 0);
|
||||
if (someApp) {
|
||||
const st = await tool("get_app_state", { app_ref: { pid: someApp.pid } });
|
||||
log("get_app_state", st.parsed?.ok === true && st.parsed?.elements?.length > 0, `${someApp.name}: ${st.parsed?.elements?.length} elements, state_id=${st.parsed?.state_id}`);
|
||||
globalThis.__state = st.parsed;
|
||||
} else {
|
||||
log("get_app_state", false, "no windowed app to observe");
|
||||
}
|
||||
|
||||
r = await tool("cursor_position");
|
||||
log("cursor_position", r.parsed?.ok === true && Number.isFinite(r.parsed?.x), `x=${r.parsed?.x} y=${r.parsed?.y}`);
|
||||
|
||||
r = await tool("screenshot", {});
|
||||
const shotFile = r.parsed?.file;
|
||||
const shotOk = r.parsed?.ok === true && shotFile && fs.existsSync(shotFile) && fs.statSync(shotFile).size > 0;
|
||||
log("screenshot (real file)", shotOk, `${shotFile ? path.basename(shotFile) : "?"} ${r.parsed?.bytes ?? 0}B scale=${r.parsed?.scale}`);
|
||||
|
||||
if (shotOk) {
|
||||
const disp = r.parsed?.points;
|
||||
const w = Math.min(400, disp?.w ?? 400), h = Math.min(300, disp?.h ?? 300);
|
||||
r = await tool("zoom", { region: [0, 0, w, h] });
|
||||
log("zoom", r.parsed?.ok === true && r.parsed?.file && fs.existsSync(r.parsed.file), `${path.basename(r.parsed?.file ?? "")} ${r.parsed?.bytes ?? 0}B`);
|
||||
}
|
||||
|
||||
// --- recording ---
|
||||
r = await tool("recording_start", {});
|
||||
const recId = r.parsed?.id;
|
||||
const started = r.parsed?.ok === true && !!recId;
|
||||
log("recording_start", started, `id=${recId} mode=${r.parsed?.mode} pid=${r.parsed?.pid}`);
|
||||
if (started) {
|
||||
await new Promise((res) => setTimeout(res, 2500));
|
||||
r = await tool("recording_status", { id: recId });
|
||||
log("recording_status", r.parsed?.ok === true && r.parsed?.running === true, `running=${r.parsed?.running} bytes-so-far=${r.parsed?.bytes} (screencapture flushes at stop)`);
|
||||
r = await tool("recording_stop", { id: recId });
|
||||
const stopped = r.parsed?.ok === true && r.parsed?.file && fs.existsSync(r.parsed.file) && fs.statSync(r.parsed.file).size > 1000;
|
||||
log("recording_stop (real file)", stopped, `${path.basename(r.parsed?.file ?? "")} + mp4:${r.parsed?.mp4 ? "yes" : "no"} bytes=${r.parsed?.bytes}`);
|
||||
}
|
||||
r = await tool("recording_list");
|
||||
log("recording_list", r.parsed?.ok === true && r.parsed?.recordings?.length >= 1, `${r.parsed?.recordings?.length} artifacts in ${r.parsed?.dir}`);
|
||||
|
||||
// --- unknown tool + error shape ---
|
||||
r = await tool("no_such_tool");
|
||||
log("unknown tool fails closed", r.isError === true && r.parsed?.error?.code === "unknown_tool", `code=${r.parsed?.error?.code}`);
|
||||
|
||||
// --- kill switch ---
|
||||
r = await tool("stop_computer_control", { reason: "smoke" });
|
||||
log("stop_computer_control", r.parsed?.ok === true && r.parsed?.stopped === true);
|
||||
r = await tool("list_apps");
|
||||
log("actions refused after kill switch", r.isError === true && r.parsed?.error?.code === "control_stopped", `code=${r.parsed?.error?.code}`);
|
||||
r = await tool("computer_list");
|
||||
log("read-only still allowed after kill switch", r.parsed?.ok === true);
|
||||
} catch (err) {
|
||||
log("smoke-run", false, err.stack ?? err.message);
|
||||
} finally {
|
||||
server.kill("SIGTERM");
|
||||
const receipt = { at: new Date().toISOString(), host: `${process.platform} ${os.release()}`, results };
|
||||
const outDir = path.join(ROOT, "receipts");
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
const out = path.join(outDir, `smoke-${new Date().toISOString().replace(/[:.]/g, "-")}.json`);
|
||||
fs.writeFileSync(out, JSON.stringify(receipt, null, 2));
|
||||
const failed = results.filter((r) => !r.pass);
|
||||
console.log(`\n${results.length - failed.length}/${results.length} passed. Receipt: ${out}`);
|
||||
try { fs.rmSync(stateDir, { recursive: true, force: true }); } catch {}
|
||||
process.exit(failed.length ? 1 : 0);
|
||||
}
|
||||
@@ -1,600 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""codewhale computer-use MCP server (stdlib only, MCP 2024-11-05, stdio).
|
||||
|
||||
Tools: screen_size, screenshot, click, type_text, press_key, scroll.
|
||||
Platform backends: macOS via screencapture + osascript, Linux via
|
||||
grim/import + ydotool/wtype. Anything unavailable fails as an MCP
|
||||
isError result with a machine-readable `reason` so the model can relay
|
||||
it instead of retrying blind.
|
||||
|
||||
Protocol: newline-delimited JSON-RPC on stdin/stdout. Log to stderr;
|
||||
stdout is the wire.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import shutil
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
PROTOCOL_VERSION = "2024-11-05"
|
||||
SERVER_NAME = "computer-use"
|
||||
SERVER_VERSION = "0.1.0"
|
||||
TOOL_TIMEOUT_SECS = 30
|
||||
MAX_TYPE_CHARS = 2000
|
||||
|
||||
_ON_MAC = platform.system() == "Darwin"
|
||||
_ON_LINUX = platform.system() == "Linux"
|
||||
|
||||
_size_cache: list | None = None
|
||||
# full-display pixels per screenshot pixel. The model reads positions
|
||||
# from the (possibly downscaled) screenshot; click scales back up.
|
||||
_shot_scale: float = 1.0
|
||||
|
||||
|
||||
def log(msg: str) -> None:
|
||||
sys.stderr.write(f"[computer-use] {msg}\n")
|
||||
sys.stderr.flush()
|
||||
|
||||
|
||||
def err(reason: str, message: str) -> dict:
|
||||
return {
|
||||
"content": [{"type": "text", "text": message}],
|
||||
"isError": True,
|
||||
"_meta": {"reason": reason},
|
||||
}
|
||||
|
||||
|
||||
def ok_text(text: str) -> dict:
|
||||
return {"content": [{"type": "text", "text": text}]}
|
||||
|
||||
|
||||
def run(argv: list, **kwargs) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(
|
||||
argv, capture_output=True, timeout=TOOL_TIMEOUT_SECS, **kwargs
|
||||
)
|
||||
|
||||
|
||||
def need_tool(name: str) -> str | None:
|
||||
path = shutil.which(name)
|
||||
if path is None:
|
||||
return None
|
||||
return path
|
||||
|
||||
|
||||
def png_size(path: str) -> tuple | None:
|
||||
try:
|
||||
with open(path, "rb") as fh:
|
||||
header = fh.read(26)
|
||||
if header[:8] != b"\x89PNG\r\n\x1a\n":
|
||||
return None
|
||||
width, height = struct.unpack(">II", header[16:24])
|
||||
return [width, height]
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def mac_screenshot(dest: str) -> str | None:
|
||||
if need_tool("screencapture") is None:
|
||||
return "Screen capture helper `screencapture` is not on PATH."
|
||||
proc = run(["screencapture", "-x", "-t", "png", "-m", dest])
|
||||
if proc.returncode != 0:
|
||||
detail = proc.stderr.decode("utf-8", "replace").strip()
|
||||
if "not permitted" in detail or "permission" in detail.lower():
|
||||
return (
|
||||
"Screen Recording permission is missing. Open System "
|
||||
"Settings > Privacy & Security > Screen Recording and "
|
||||
"enable the terminal running codewhale, then try again."
|
||||
)
|
||||
return f"screencapture failed: {detail or proc.returncode}"
|
||||
return None
|
||||
|
||||
|
||||
def linux_screenshot(dest: str) -> str | None:
|
||||
for helper, argv in (
|
||||
("grim", ["grim", dest]),
|
||||
("import", ["import", "-window", "root", dest]),
|
||||
("scrot", ["scrot", "--overwrite", dest]),
|
||||
):
|
||||
if need_tool(helper) is None:
|
||||
continue
|
||||
proc = run(argv)
|
||||
if proc.returncode == 0:
|
||||
return None
|
||||
log(f"{helper} failed, trying next backend")
|
||||
return (
|
||||
"No working screenshot helper found. Install one of grim, "
|
||||
"ImageMagick import, or scrot."
|
||||
)
|
||||
|
||||
|
||||
MAX_SCREENSHOT_WIDTH = 1920
|
||||
|
||||
|
||||
def shrink_for_vision(png_path: str) -> tuple:
|
||||
"""Downscale to a vision-sized JPEG with native platform tools.
|
||||
|
||||
Returns (path, mime, size) of the best artifact, falling back to the
|
||||
original PNG when no resizer exists. Retina PNGs run to tens of
|
||||
megabytes; vision models read 1920px JPEGs fine at a tenth the cost.
|
||||
"""
|
||||
size = png_size(png_path)
|
||||
if _ON_MAC and need_tool("sips") is not None:
|
||||
with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tmp:
|
||||
small = tmp.name
|
||||
try:
|
||||
proc = run(
|
||||
[
|
||||
"sips",
|
||||
"-Z",
|
||||
str(MAX_SCREENSHOT_WIDTH),
|
||||
"-s",
|
||||
"format",
|
||||
"jpeg",
|
||||
"-s",
|
||||
"formatOptions",
|
||||
"80",
|
||||
png_path,
|
||||
"--out",
|
||||
small,
|
||||
]
|
||||
)
|
||||
if proc.returncode == 0:
|
||||
jpeg_size = png_size(small)
|
||||
if jpeg_size is None: # JPEG has no PNG header; trust sips
|
||||
width = min(size[0], MAX_SCREENSHOT_WIDTH) if size else 0
|
||||
height = (
|
||||
round(size[1] * width / size[0])
|
||||
if size and size[0]
|
||||
else 0
|
||||
)
|
||||
return small, "image/jpeg", [width, height]
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
os.unlink(small)
|
||||
except OSError:
|
||||
pass
|
||||
if _ON_LINUX and need_tool("convert") is not None:
|
||||
with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tmp:
|
||||
small = tmp.name
|
||||
try:
|
||||
proc = run(
|
||||
[
|
||||
"convert",
|
||||
png_path,
|
||||
"-resize",
|
||||
f"{MAX_SCREENSHOT_WIDTH}>",
|
||||
"-quality",
|
||||
"80",
|
||||
small,
|
||||
]
|
||||
)
|
||||
if proc.returncode == 0:
|
||||
width = min(size[0], MAX_SCREENSHOT_WIDTH) if size else 0
|
||||
height = (
|
||||
round(size[1] * width / size[0]) if size and size[0] else 0
|
||||
)
|
||||
return small, "image/jpeg", [width, height]
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
os.unlink(small)
|
||||
except OSError:
|
||||
pass
|
||||
return png_path, "image/png", size or [0, 0]
|
||||
|
||||
|
||||
def do_screenshot() -> dict:
|
||||
if not (_ON_MAC or _ON_LINUX):
|
||||
return err(
|
||||
"unsupported_platform",
|
||||
f"Computer use supports macOS and Linux; this host is {platform.system()}.",
|
||||
)
|
||||
with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmp:
|
||||
dest = tmp.name
|
||||
small: str | None = None
|
||||
try:
|
||||
if _ON_MAC:
|
||||
problem = mac_screenshot(dest)
|
||||
else:
|
||||
problem = linux_screenshot(dest)
|
||||
if problem is not None:
|
||||
reason = "missing_permission" if "permission" in problem else "missing_helper"
|
||||
return err(reason, problem)
|
||||
full = png_size(dest)
|
||||
if full is None:
|
||||
return err("capture_failed", "Screenshot helper produced no readable PNG.")
|
||||
small, mime, size = shrink_for_vision(dest)
|
||||
global _size_cache, _shot_scale
|
||||
_size_cache = size
|
||||
_shot_scale = (full[0] / size[0]) if size[0] else 1.0
|
||||
with open(small, "rb") as fh:
|
||||
data = base64.b64encode(fh.read()).decode("ascii")
|
||||
return {
|
||||
"content": [
|
||||
{
|
||||
"type": "image",
|
||||
"data": data,
|
||||
"mimeType": mime,
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"Screenshot {size[0]}x{size[1]}.",
|
||||
},
|
||||
]
|
||||
}
|
||||
finally:
|
||||
for path in (dest, small):
|
||||
if path is not None:
|
||||
try:
|
||||
os.unlink(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def do_screen_size() -> dict:
|
||||
global _size_cache
|
||||
if _size_cache is not None:
|
||||
width, height = _size_cache
|
||||
return ok_text(f'{{"width": {width}, "height": {height}}}')
|
||||
if _ON_MAC and need_tool("screencapture") is not None:
|
||||
with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmp:
|
||||
dest = tmp.name
|
||||
try:
|
||||
problem = mac_screenshot(dest)
|
||||
if problem is not None:
|
||||
return err("missing_helper", problem)
|
||||
size = png_size(dest)
|
||||
finally:
|
||||
try:
|
||||
os.unlink(dest)
|
||||
except OSError:
|
||||
pass
|
||||
if size is None:
|
||||
return err("capture_failed", "Could not determine the screen size.")
|
||||
_size_cache = size
|
||||
return ok_text(f'{{"width": {size[0]}, "height": {size[1]}}}')
|
||||
return err(
|
||||
"unknown_size" if (_ON_MAC or _ON_LINUX) else "unsupported_platform",
|
||||
"Screen size is unknown until the first screenshot on this platform. "
|
||||
"Call screenshot first.",
|
||||
)
|
||||
|
||||
|
||||
def osascript(script: str) -> tuple:
|
||||
with tempfile.NamedTemporaryFile(
|
||||
suffix=".scpt", mode="w", delete=False, encoding="utf-8"
|
||||
) as tmp:
|
||||
tmp.write(script)
|
||||
path = tmp.name
|
||||
try:
|
||||
proc = run(["osascript", path])
|
||||
finally:
|
||||
try:
|
||||
os.unlink(path)
|
||||
except OSError:
|
||||
pass
|
||||
return proc.returncode, proc.stderr.decode("utf-8", "replace").strip()
|
||||
|
||||
|
||||
def mac_click(x: int, y: int) -> str | None:
|
||||
if need_tool("osascript") is None:
|
||||
return "Missing helper: osascript is not on PATH."
|
||||
code, detail = osascript(
|
||||
f'tell application "System Events" to click at {{{x}, {y}}}'
|
||||
)
|
||||
if code != 0:
|
||||
if "not permitted" in detail or "1002" in detail:
|
||||
return (
|
||||
"Accessibility permission is missing. Open System Settings "
|
||||
"> Privacy & Security > Accessibility and enable the "
|
||||
"terminal running codewhale, then try again."
|
||||
)
|
||||
return f"Click failed: {detail or code}"
|
||||
return None
|
||||
|
||||
|
||||
def linux_type_backend() -> str | None:
|
||||
if need_tool("ydotool") is not None:
|
||||
return "ydotool"
|
||||
if need_tool("wtype") is not None:
|
||||
return "wtype"
|
||||
return None
|
||||
|
||||
|
||||
def do_click(args: dict) -> dict:
|
||||
try:
|
||||
x, y = int(args["x"]), int(args["y"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return err("bad_arguments", "click needs integer x and y pixel coordinates.")
|
||||
if x < 0 or y < 0:
|
||||
return err("bad_arguments", "click coordinates must be non-negative.")
|
||||
# Coordinates arrive in screenshot space; scale to display pixels.
|
||||
x, y = round(x * _shot_scale), round(y * _shot_scale)
|
||||
if _ON_MAC:
|
||||
problem = mac_click(x, y)
|
||||
if problem is not None:
|
||||
reason = "missing_permission" if "permission" in problem else "missing_helper"
|
||||
if problem.startswith("Click failed"):
|
||||
reason = "action_failed"
|
||||
return err(reason, problem)
|
||||
return ok_text(f"Clicked at {x},{y}.")
|
||||
if _ON_LINUX:
|
||||
backend = linux_type_backend()
|
||||
if backend == "ydotool":
|
||||
proc = run(["ydotool", "mousemove", str(x), str(y), "click", "1"])
|
||||
elif backend == "wtype":
|
||||
return err(
|
||||
"missing_helper",
|
||||
"wtype moves no mouse. Install ydotool for clicking on Linux.",
|
||||
)
|
||||
else:
|
||||
return err(
|
||||
"missing_helper",
|
||||
"No input helper found on Linux. Install ydotool (X11/Wayland) "
|
||||
"or wtype (Wayland typing only).",
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
return err("action_failed", f"Click failed: {proc.stderr.decode()[:200]}")
|
||||
return ok_text(f"Clicked at {x},{y}.")
|
||||
return err(
|
||||
"unsupported_platform",
|
||||
f"Clicking is supported on macOS and Linux; this host is {platform.system()}.",
|
||||
)
|
||||
|
||||
|
||||
def do_type_text(args: dict) -> dict:
|
||||
text = args.get("text", "")
|
||||
if not isinstance(text, str) or not text:
|
||||
return err("bad_arguments", "type_text needs a non-empty text string.")
|
||||
if len(text) > MAX_TYPE_CHARS:
|
||||
return err(
|
||||
"bad_arguments",
|
||||
f"type_text accepts at most {MAX_TYPE_CHARS} characters per call.",
|
||||
)
|
||||
if _ON_MAC:
|
||||
if need_tool("osascript") is None:
|
||||
return err("missing_helper", "Missing helper: osascript is not on PATH.")
|
||||
escaped = text.replace("\\", "\\\\").replace('"', '\\"')
|
||||
code, detail = osascript(
|
||||
f'tell application "System Events" to keystroke "{escaped}"'
|
||||
)
|
||||
if code != 0:
|
||||
if "not permitted" in detail or "1002" in detail:
|
||||
return err(
|
||||
"missing_permission",
|
||||
"Accessibility permission is missing. Open System Settings "
|
||||
"> Privacy & Security > Accessibility and enable the "
|
||||
"terminal running codewhale, then try again.",
|
||||
)
|
||||
return err("action_failed", f"Typing failed: {detail or code}")
|
||||
return ok_text(f"Typed {len(text)} characters.")
|
||||
if _ON_LINUX:
|
||||
backend = linux_type_backend()
|
||||
if backend == "ydotool":
|
||||
proc = run(["ydotool", "type", "--", text])
|
||||
elif backend == "wtype":
|
||||
proc = run(["wtype", "--", text])
|
||||
else:
|
||||
return err(
|
||||
"missing_helper",
|
||||
"No input helper found on Linux. Install ydotool or wtype.",
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
return err("action_failed", f"Typing failed: {proc.stderr.decode()[:200]}")
|
||||
return ok_text(f"Typed {len(text)} characters.")
|
||||
return err(
|
||||
"unsupported_platform",
|
||||
f"Typing is supported on macOS and Linux; this host is {platform.system()}.",
|
||||
)
|
||||
|
||||
|
||||
MAC_KEYS = {
|
||||
"return": "keystroke return",
|
||||
"enter": "keystroke return",
|
||||
"tab": "keystroke tab",
|
||||
"escape": "key code 53",
|
||||
"esc": "key code 53",
|
||||
"space": "keystroke space",
|
||||
"delete": "key code 51",
|
||||
"up": "key code 126",
|
||||
"down": "key code 125",
|
||||
"left": "key code 123",
|
||||
"right": "key code 124",
|
||||
}
|
||||
|
||||
|
||||
def do_press_key(args: dict) -> dict:
|
||||
key = str(args.get("key", "")).strip().lower()
|
||||
if not key:
|
||||
return err("bad_arguments", "press_key needs a key name such as return or tab.")
|
||||
if _ON_MAC:
|
||||
if key not in MAC_KEYS:
|
||||
return err(
|
||||
"bad_arguments",
|
||||
f"Unsupported key '{key}'. Supported: "
|
||||
+ ", ".join(sorted(MAC_KEYS)),
|
||||
)
|
||||
code, detail = osascript(
|
||||
f'tell application "System Events" to {MAC_KEYS[key]}'
|
||||
)
|
||||
if code != 0:
|
||||
return err("action_failed", f"Key press failed: {detail or code}")
|
||||
return ok_text(f"Pressed {key}.")
|
||||
if _ON_LINUX:
|
||||
if need_tool("ydotool") is None:
|
||||
return err("missing_helper", "press_key on Linux needs ydotool.")
|
||||
proc = run(["ydotool", "key", key])
|
||||
if proc.returncode != 0:
|
||||
return err("action_failed", f"Key press failed: {proc.stderr.decode()[:200]}")
|
||||
return ok_text(f"Pressed {key}.")
|
||||
return err(
|
||||
"unsupported_platform",
|
||||
f"Key presses are supported on macOS and Linux; this host is {platform.system()}.",
|
||||
)
|
||||
|
||||
|
||||
def do_scroll(args: dict) -> dict:
|
||||
if _ON_LINUX and need_tool("ydotool") is not None:
|
||||
try:
|
||||
dx, dy = int(args.get("dx", 0)), int(args.get("dy", 0))
|
||||
except (TypeError, ValueError):
|
||||
return err("bad_arguments", "scroll needs integer dx/dy.")
|
||||
proc = run(["ydotool", "mousemove", "--", str(dx), str(dy)])
|
||||
if proc.returncode != 0:
|
||||
return err("action_failed", "Scroll failed.")
|
||||
return ok_text(f"Scrolled by {dx},{dy}.")
|
||||
if _ON_MAC:
|
||||
return err(
|
||||
"unsupported_platform",
|
||||
"Scrolling is not implemented on macOS in computer-use 0.1.0. "
|
||||
"Use press_key (space, up, down) to move through scrollable content.",
|
||||
)
|
||||
return err(
|
||||
"unsupported_platform",
|
||||
f"Scrolling is supported on Linux with ydotool; this host is {platform.system()}.",
|
||||
)
|
||||
|
||||
|
||||
TOOLS = {
|
||||
"screen_size": {
|
||||
"description": "Report the main display size in pixels. Call before clicking.",
|
||||
"inputSchema": {"type": "object", "properties": {}},
|
||||
},
|
||||
"screenshot": {
|
||||
"description": "Capture the main display as a PNG vision block.",
|
||||
"inputSchema": {"type": "object", "properties": {}},
|
||||
},
|
||||
"click": {
|
||||
"description": "Click at screenshot pixels from the top-left (server scales to the display).",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"x": {"type": "integer"},
|
||||
"y": {"type": "integer"},
|
||||
},
|
||||
"required": ["x", "y"],
|
||||
},
|
||||
},
|
||||
"type_text": {
|
||||
"description": "Type text into the focused field. Never secrets.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {"text": {"type": "string"}},
|
||||
"required": ["text"],
|
||||
},
|
||||
},
|
||||
"press_key": {
|
||||
"description": "Press one named key: return, tab, escape, space, delete, up, down, left, right.",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {"key": {"type": "string"}},
|
||||
"required": ["key"],
|
||||
},
|
||||
},
|
||||
"scroll": {
|
||||
"description": "Scroll by dx/dy (Linux with ydotool only).",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"dx": {"type": "integer"},
|
||||
"dy": {"type": "integer"},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
HANDLERS = {
|
||||
"screen_size": lambda args: do_screen_size(),
|
||||
"screenshot": lambda args: do_screenshot(),
|
||||
"click": do_click,
|
||||
"type_text": do_type_text,
|
||||
"press_key": do_press_key,
|
||||
"scroll": do_scroll,
|
||||
}
|
||||
|
||||
|
||||
def handle(msg: dict):
|
||||
msg_id = msg.get("id")
|
||||
method = msg.get("method")
|
||||
|
||||
def reply(result: dict) -> dict:
|
||||
return {"jsonrpc": "2.0", "id": msg_id, "result": result}
|
||||
|
||||
def error(code: int, message: str) -> dict:
|
||||
return {
|
||||
"jsonrpc": "2.0",
|
||||
"id": msg_id,
|
||||
"error": {"code": code, "message": message},
|
||||
}
|
||||
|
||||
if method == "initialize":
|
||||
return reply(
|
||||
{
|
||||
"protocolVersion": PROTOCOL_VERSION,
|
||||
"serverInfo": {"name": SERVER_NAME, "version": SERVER_VERSION},
|
||||
"capabilities": {"tools": {}},
|
||||
}
|
||||
)
|
||||
if method in ("notifications/initialized", "notifications/cancelled"):
|
||||
return None
|
||||
if method == "ping":
|
||||
return reply({})
|
||||
if method == "tools/list":
|
||||
return reply(
|
||||
{
|
||||
"tools": [
|
||||
{
|
||||
"name": name,
|
||||
"description": spec["description"],
|
||||
"inputSchema": spec["inputSchema"],
|
||||
}
|
||||
for name, spec in TOOLS.items()
|
||||
]
|
||||
}
|
||||
)
|
||||
if method == "tools/call":
|
||||
params = msg.get("params", {}) or {}
|
||||
name = params.get("name", "")
|
||||
args = params.get("arguments", {}) or {}
|
||||
handler = HANDLERS.get(name)
|
||||
if handler is None:
|
||||
return error(-32602, f"Unknown tool '{name}'.")
|
||||
try:
|
||||
return reply(handler(args))
|
||||
except subprocess.TimeoutExpired:
|
||||
return reply(err("timed_out", f"Tool '{name}' timed out."))
|
||||
except Exception as exc: # never break the wire
|
||||
log(f"tool {name} crashed: {exc}")
|
||||
return reply(err("internal", f"Tool '{name}' failed unexpectedly."))
|
||||
return error(-32601, f"Unsupported method '{method}'.")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
msg = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
try:
|
||||
response = handle(msg if isinstance(msg, dict) else {})
|
||||
except Exception as exc: # never break the wire
|
||||
log(f"dispatch crashed: {exc}")
|
||||
continue
|
||||
if response is not None:
|
||||
sys.stdout.write(json.dumps(response) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,97 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Protocol tests for the computer-use MCP server. No display needed:
|
||||
failure paths and the JSON-RPC surface are pure; only screenshot and
|
||||
screen_size touch the host, and the suite skips those without one."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
import server
|
||||
|
||||
|
||||
def call(method, params=None, msg_id=1):
|
||||
msg = {"jsonrpc": "2.0", "id": msg_id, "method": method}
|
||||
if params is not None:
|
||||
msg["params"] = params
|
||||
return server.handle(msg)
|
||||
|
||||
|
||||
class HandshakeTest(unittest.TestCase):
|
||||
def test_initialize_advertises_tools(self):
|
||||
res = call("initialize", {"protocolVersion": "2024-11-05"})
|
||||
result = res["result"]
|
||||
self.assertEqual(result["protocolVersion"], "2024-11-05")
|
||||
self.assertIn("tools", result["capabilities"])
|
||||
|
||||
def test_initialized_notification_is_silent(self):
|
||||
self.assertIsNone(
|
||||
server.handle({"jsonrpc": "2.0", "method": "notifications/initialized"})
|
||||
)
|
||||
|
||||
def test_tools_list_names(self):
|
||||
res = call("tools/list")
|
||||
names = sorted(t["name"] for t in res["result"]["tools"])
|
||||
self.assertEqual(
|
||||
names,
|
||||
["click", "press_key", "screen_size", "screenshot", "scroll", "type_text"],
|
||||
)
|
||||
for tool in res["result"]["tools"]:
|
||||
self.assertIn("inputSchema", tool)
|
||||
|
||||
def test_unknown_method(self):
|
||||
res = call("resources/list")
|
||||
self.assertEqual(res["error"]["code"], -32601)
|
||||
|
||||
def test_unknown_tool(self):
|
||||
res = call("tools/call", {"name": "nope", "arguments": {}})
|
||||
self.assertEqual(res["error"]["code"], -32602)
|
||||
|
||||
|
||||
class ValidationTest(unittest.TestCase):
|
||||
def test_click_rejects_bad_coords(self):
|
||||
for args in ({"x": -1, "y": 5}, {"x": "a", "y": 1}, {}):
|
||||
res = call("tools/call", {"name": "click", "arguments": args})
|
||||
self.assertTrue(res["result"]["isError"], args)
|
||||
self.assertEqual(res["result"]["_meta"]["reason"], "bad_arguments")
|
||||
|
||||
def test_type_rejects_empty_and_huge(self):
|
||||
for args in ({"text": ""}, {}, {"text": "x" * 2001}):
|
||||
res = call("tools/call", {"name": "type_text", "arguments": args})
|
||||
self.assertTrue(res["result"]["isError"], str(args)[:20])
|
||||
|
||||
def test_press_key_allowlist(self):
|
||||
res = call("tools/call", {"name": "press_key", "arguments": {"key": ""}})
|
||||
self.assertTrue(res["result"]["isError"])
|
||||
if server._ON_MAC:
|
||||
res = call(
|
||||
"tools/call", {"name": "press_key", "arguments": {"key": "f13"}}
|
||||
)
|
||||
self.assertEqual(res["result"]["_meta"]["reason"], "bad_arguments")
|
||||
|
||||
|
||||
class LiveWireTest(unittest.TestCase):
|
||||
def test_stdio_round_trip(self):
|
||||
proc = subprocess.run(
|
||||
[sys.executable, os.path.join(os.path.dirname(__file__), "server.py")],
|
||||
input=(
|
||||
'{"jsonrpc":"2.0","id":1,"method":"initialize",'
|
||||
'"params":{"protocolVersion":"2024-11-05"}}\n'
|
||||
'{"jsonrpc":"2.0","id":2,"method":"tools/list"}\n'
|
||||
),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30,
|
||||
)
|
||||
lines = [json.loads(line) for line in proc.stdout.splitlines() if line.strip()]
|
||||
self.assertEqual(len(lines), 2)
|
||||
self.assertEqual(lines[0]["result"]["protocolVersion"], "2024-11-05")
|
||||
self.assertEqual(len(lines[1]["result"]["tools"]), 6)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,38 +1,98 @@
|
||||
---
|
||||
name: computer-use
|
||||
description: See the screen and operate it. Use when the user asks to click, type, or check something visually on this machine.
|
||||
description: Full desktop control on macOS, Windows, Linux, and HarmonyOS — accessibility-first observation and actions with pixel fallback, screenshots, zoom, screen recording, and switching between registered computers as a default.
|
||||
---
|
||||
|
||||
# Computer use
|
||||
# Codewhale Computer Use
|
||||
|
||||
You can see this machine's screen and operate its mouse and keyboard
|
||||
through the `computer` MCP server. Work in a tight look-act-verify
|
||||
loop and never act blind.
|
||||
## Computers first
|
||||
|
||||
## The loop
|
||||
The plugin controls **computers**, not "the screen". `computer_list` shows the
|
||||
registry; one computer is always **active**, and every tool acts on the active
|
||||
computer unless given `computer`.
|
||||
|
||||
1. **Look**: call `mcp_computer_screenshot` first. The image arrives as
|
||||
a vision block; read positions from it, never guess coordinates.
|
||||
2. **Act**: exactly one action per turn — `mcp_computer_click`,
|
||||
`mcp_computer_type_text`, `mcp_computer_press_key`, or
|
||||
`mcp_computer_scroll`. Coordinates are pixels in the latest
|
||||
screenshot from the top-left, matching `mcp_computer_screen_size`;
|
||||
the server scales them to the display.
|
||||
3. **Verify**: screenshot again after every action that should change
|
||||
something visible. If the screen did not change as expected, stop
|
||||
and report instead of retrying harder.
|
||||
- Pass `computer: "<id>"` on any tool to act on (and stickily switch to) that
|
||||
computer. `computer_switch` changes the active computer without acting.
|
||||
- `local` is the machine the plugin runs on. `ssh` computers run the bundled
|
||||
remote agent (pushed automatically at registration). `hdc` computers are
|
||||
HarmonyOS devices driven over hdc.
|
||||
- Every receipt names the computer it happened on. Read it before continuing —
|
||||
never assume the action landed on the machine you meant.
|
||||
|
||||
## Safety rules
|
||||
## Core loop
|
||||
|
||||
- Every action tool asks for approval unless the session runs in
|
||||
Always-approve posture. Say what you are about to click or type
|
||||
before the approval prompt appears.
|
||||
- Never type passwords, secrets, or one-time codes. If a login screen
|
||||
appears, stop and hand control back to the user.
|
||||
- Never dismiss system permission dialogs (Accessibility, Screen
|
||||
Recording) yourself. If the server reports a missing permission,
|
||||
tell the user exactly which toggle to flip and stop.
|
||||
- Prefer keyboard over mouse (Tab, arrows, Enter) — it is precise and
|
||||
does not depend on pixel coordinates.
|
||||
- If a tool reports `unsupported_platform` or `missing_helper`, stop
|
||||
and relay the server's message verbatim.
|
||||
Observe once, act once, then verify.
|
||||
|
||||
1. If readiness is unknown, call `request_access` once. It names missing
|
||||
permissions and missing tools per platform, and never pops dialogs.
|
||||
2. `list_apps` shows running apps only. If the user names an app that is
|
||||
absent, call `open_application` once with the original user-provided name,
|
||||
copied character-for-character — including case, spaces, punctuation, and
|
||||
suffixes such as `app` or `.exe`. Do not translate, localize, normalize,
|
||||
shorten, or retry with guesses.
|
||||
3. `get_app_state` returns a bounded accessibility tree (macOS AX / Windows
|
||||
UIA / Linux AT-SPI / HarmonyOS uitest) with element indices and a
|
||||
`state_id`. Start here, without a screenshot.
|
||||
4. If the tree contains the target, act on the element: `perform_action`
|
||||
(AXPress/Invoke/click…), `set_value` for editable fields, element click.
|
||||
The element path is background-safe on macOS and UIA platforms.
|
||||
5. Only when accessibility cannot express the target: `screenshot` (optionally
|
||||
`zoom` for small targets) and act with a coordinate target. Coordinates are
|
||||
pixels **in the latest returned raster** for that computer; the server maps
|
||||
them to screen points. After a new screenshot, old pixels are stale.
|
||||
6. Verify with a fresh observation or a task oracle before claiming success.
|
||||
`action_sent: true` means it may already have happened — never replay.
|
||||
|
||||
## Choosing targets
|
||||
|
||||
- Element: `{"type":"element","state_id":"s-1","index":4}` — prefer this.
|
||||
- Coordinate: `{"type":"coordinate","x":496,"y":331}` — pixels from the latest
|
||||
raster only; submit `x`/`y` unchanged, never transform them yourself.
|
||||
- Never translate pixels into an element target; never invent `state_id`s.
|
||||
|
||||
## Raw input reality (read before clicking)
|
||||
|
||||
- macOS: element actions and `set_value` are background-safe. Raw pointer and
|
||||
keyboard events are posted at the target point and land on whatever is
|
||||
frontmost there — activate the app first for click/type flows, then act.
|
||||
- Windows/Linux: raw input is foreground by nature; UIA/AT-SPI element actions
|
||||
are the precise path.
|
||||
- HarmonyOS: `uitest` synthesizes touches; there is no hover or cursor.
|
||||
|
||||
## Keyboard
|
||||
|
||||
- macOS uses `cmd` (`cmd+c`), Linux/Windows use `ctrl` (`ctrl+c`).
|
||||
- `key` for chords and repeats, `hold_key` for a duration, `type` for text.
|
||||
- Prefer `set_value` on editable elements over typing.
|
||||
|
||||
## Recording
|
||||
|
||||
`recording_start` → work → `recording_stop` returns the finalized file path.
|
||||
macOS uses `screencapture -v` (a receipt warning about Screen Recording
|
||||
permission means the user must grant it once). Linux uses x11grab/wf-recorder,
|
||||
Windows ffmpeg gdigrab, HarmonyOS snapshot-series (no native CLI recorder —
|
||||
the receipt says so). `recording_status` / `recording_list` report bytes and
|
||||
paths. Screenshots land in the same directory.
|
||||
|
||||
## Safety
|
||||
|
||||
- `stop_computer_control` is the kill switch; after it, actions fail closed
|
||||
for the session. Do not continue after it or after a denied permission.
|
||||
- Never retry a refused action unchanged. Re-observe, choose a fresh target.
|
||||
- If a permission is explicitly denied, tell the user which permission in
|
||||
which Settings pane, and end the turn. Do not promise later retries.
|
||||
|
||||
## Recipes
|
||||
|
||||
- **Screenshot** — optionally a computer id, display index, or `[x,y,w,h]`
|
||||
region; call `screenshot`; report path, size, computer/display. Black or
|
||||
empty capture means the host lacks Screen Recording permission (macOS):
|
||||
say so and stop.
|
||||
- **Record** — `recording_start` (parse computer id, fps, display, duration
|
||||
or "record for 30s" → `durationSec` on macOS), then report id, path, mode.
|
||||
To stop, find the running id via `recording_list` and call `recording_stop`.
|
||||
- **Switch computers** — `computer_list`; if asked to add: ssh `user@host`
|
||||
(agent is pushed automatically) or `hdc [target]` for a HarmonyOS device;
|
||||
otherwise show the registry and remind that any tool accepts `computer`.
|
||||
- **Status** — `computer_list`, then `request_access` per computer; call out
|
||||
anything that will fail closed with the exact install hint from the receipt.
|
||||
@@ -0,0 +1,30 @@
|
||||
---
|
||||
name: recording
|
||||
description: Capture screen recordings and screenshots on any registered computer (macOS, Windows, Linux, HarmonyOS) and manage the recording library.
|
||||
---
|
||||
|
||||
# Recording and screenshots
|
||||
|
||||
1. Pick the computer (`computer_list`, or pass `computer` — it switches).
|
||||
2. `recording_start` with optional `display` (macOS/Linux), `region` (Linux/
|
||||
Windows), `fps` (Linux/Windows), `durationSec` (macOS auto-stop), and
|
||||
`intervalMs` (HarmonyOS snapshot cadence).
|
||||
3. Do the work (or let the user do it).
|
||||
4. `recording_stop` with the returned `id` → finalized file path + bytes.
|
||||
5. `recording_list` shows everything saved; `recording_status` checks one.
|
||||
|
||||
Platform truths:
|
||||
|
||||
- **macOS**: `screencapture -v` per display; stop finalizes a .mov and, when
|
||||
ffmpeg is installed, a stream-copied .mp4 alongside. If a recording dies
|
||||
instantly, the terminal host is missing Screen Recording permission — tell
|
||||
the user, do not retry.
|
||||
- **Windows**: ffmpeg gdigrab of the desktop or a region.
|
||||
- **Linux X11**: ffmpeg x11grab (honors $DISPLAY, region, fps). Wayland:
|
||||
wf-recorder (per-output via CU_WAYLAND_OUTPUT).
|
||||
- **HarmonyOS**: no native CLI recorder; the backend captures
|
||||
`snapshot_display` frames at `intervalMs` and muxes with ffmpeg on stop.
|
||||
The receipt labels the mode `snapshot-series` — never call it real-time.
|
||||
|
||||
Screenshots: `screenshot` returns the saved path and raster geometry; `zoom`
|
||||
crops the latest raster when a target is too small to read.
|
||||
@@ -0,0 +1,621 @@
|
||||
// macOS backend. Zero third-party dependencies:
|
||||
// - observation: osascript JXA over System Events (accessibility tree)
|
||||
// - raw input: CGEvent posted through JXA's CoreGraphics bridge
|
||||
// - stills/video: /usr/sbin/screencapture (video requires macOS 13+)
|
||||
// - crop: sips - clipboard: pbcopy/pbpaste
|
||||
// All scripts travel as temp files + one base64 payload argument, so tool
|
||||
// arguments never become AppleScript syntax.
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import crypto from "node:crypto";
|
||||
import { spawn } from "node:child_process";
|
||||
import { run, runOk, ExecError, tryJson, have } from "../exec.mjs";
|
||||
|
||||
const KEY_CODES = {
|
||||
return: 36, enter: 36, tab: 48, space: 49, escape: 53, esc: 53, delete: 51,
|
||||
backspace: 51, forwarddelete: 117, home: 115, end: 119, pageup: 116, pagedown: 121,
|
||||
left: 123, right: 124, down: 125, up: 126, clear: 71, capslock: 57, f1: 122,
|
||||
f2: 120, f3: 99, f4: 118, f5: 96, f6: 97, f7: 98, f8: 100, f9: 101, f10: 109,
|
||||
f11: 103, f12: 111, volumeup: 72, volumedown: 73, mute: 74, help: 114,
|
||||
a: 0, s: 1, d: 2, f: 3, h: 4, g: 5, z: 6, x: 7, c: 8, v: 9, b: 11, q: 12,
|
||||
w: 13, e: 14, r: 15, y: 16, t: 17, "1": 18, "2": 19, "3": 20, "4": 21,
|
||||
"5": 23, "6": 22, "7": 26, "8": 28, "9": 25, "0": 29, "-": 27, "=": 24,
|
||||
"[": 33, "]": 30, "\\": 42, ";": 41, "'": 39, ",": 43, ".": 47, "/": 44,
|
||||
o: 31, u: 32, i: 34, p: 35, l: 37, j: 38, k: 40, n: 45, m: 46,
|
||||
};
|
||||
const MODIFIERS = {
|
||||
cmd: 1 << 20, command: 1 << 20, win: 1 << 20, meta: 1 << 20,
|
||||
shift: 1 << 17, ctrl: 1 << 18, control: 1 << 18, alt: 1 << 19, opt: 1 << 19, option: 1 << 19,
|
||||
fn: 1 << 23, function: 1 << 23,
|
||||
};
|
||||
const MOUSE = {
|
||||
left: { down: 1, up: 2, dragged: 7 },
|
||||
right: { down: 3, up: 4, dragged: 8 },
|
||||
middle: { down: 25, up: 26, dragged: 27 },
|
||||
};
|
||||
|
||||
export function create({ exec }) {
|
||||
const runL = (cmd, args, opts) => exec.run(cmd, args, opts);
|
||||
const state = { activeDisplay: 1, lastRaster: null };
|
||||
|
||||
// ---------- JXA helper ----------
|
||||
async function jxa(script, payload = {}, timeoutMs = 20_000) {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "cu-jxa-"));
|
||||
try {
|
||||
const file = path.join(dir, "s.js");
|
||||
fs.writeFileSync(file, script);
|
||||
// Payload travels as a plain argv string: osascript is spawned without a
|
||||
// shell, so JSON content can never become script syntax.
|
||||
const r = await runL("osascript", ["-l", "JavaScript", file, JSON.stringify(payload)], { timeoutMs });
|
||||
if (r.timedOut) throw new ExecError("osascript timed out", r);
|
||||
if (r.code !== 0) {
|
||||
const msg = (r.stderr || r.stdout).trim().split("\n")[0] || "osascript failed";
|
||||
throw new ExecError(/(not allowed assistive|assistive access|250)/i.test(r.stderr || "") || /(-25211|-1719|not allowed)/i.test(msg)
|
||||
? `${msg} (accessibility permission for the host terminal is required: System Settings → Privacy & Security → Accessibility)`
|
||||
: msg, r);
|
||||
}
|
||||
return tryJson(r.stdout.trim(), r.stdout.trim());
|
||||
} finally {
|
||||
try { fs.rmSync(dir, { recursive: true, force: true }); } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
const JXA_PRELUDE = `
|
||||
function run(argv){
|
||||
var P = JSON.parse(argv[0]);
|
||||
function g(f){ try { return f(); } catch(e){ return null; } }
|
||||
function num(v){ if (typeof v==='function') v = g(v); if (v===null||v===undefined) return null; if (typeof v==='object'){ try { v = ObjC.unwrap(v); } catch(e){} } var n = Number(v); return isFinite(n)? n : null; }
|
||||
function pt(p){ p = g(function(){return p();}); if(!p) return null; try { return { x: num(p[0]!==undefined?p[0]:p.x), y: num(p[1]!==undefined?p[1]:p.y) }; } catch(e){ return null; } }
|
||||
function sz(s){ s = g(function(){return s();}); if(!s) return null; try { return { w: num(s[0]!==undefined?s[0]:s.width), h: num(s[1]!==undefined?s[1]:s.height) }; } catch(e){ return null; } }
|
||||
function elInfo(el, idx, winIdx, path){
|
||||
return {
|
||||
index: idx, path: path, windowIndex: winIdx,
|
||||
role: g(function(){ return String(el.role()); }),
|
||||
subrole: g(function(){ var s = el.subrole(); return s ? String(s) : null; }),
|
||||
label: g(function(){ var t = el.title(); if(t) return String(t); var n = el.name(); if(n) return String(n); var h = el.help(); return h ? String(h) : null; }),
|
||||
value: g(function(){ var v = el.value(); if (v===null||v===undefined) return null; var s = String(v); return s.length>120? s.slice(0,120)+'…' : s; }),
|
||||
enabled: g(function(){ return !!el.enabled(); }),
|
||||
focused: g(function(){ return !!el.focused(); }),
|
||||
position: pt(function(){ return el.position(); }),
|
||||
size: sz(function(){ return el.size(); }),
|
||||
actions: g(function(){ return el.actions().map(function(a){ return String(a.name()); }); }) || []
|
||||
};
|
||||
}
|
||||
`;
|
||||
|
||||
async function findProcess(ref) {
|
||||
return jxa(`${JXA_PRELUDE}
|
||||
var se = Application('System Events');
|
||||
var procs = se.applicationProcesses.js ? se.applicationProcesses : se.applicationProcesses;
|
||||
var found = null;
|
||||
var list = se.applicationProcesses();
|
||||
for (var i=0;i<list.length;i++){
|
||||
var p = list[i];
|
||||
if (P.pid != null && num(function(){ return p.unixId(); }) === P.pid){ found = p; break; }
|
||||
if (P.bundle_id){ var b = g(function(){ return String(p.bundleIdentifier()); }); if (b && b.toLowerCase()===P.bundle_id.toLowerCase()){ found = p; break; } }
|
||||
if (P.name){ var n = g(function(){ return String(p.name()); }); if (n && n.toLowerCase()===P.name.toLowerCase()){ found = p; break; } }
|
||||
}
|
||||
if (!found) { return JSON.stringify({found:false}); }
|
||||
else {
|
||||
return JSON.stringify({ found: true, name: g(function(){return String(found.name());}), pid: num(function(){return found.unixId();}),
|
||||
bundle_id: g(function(){ var b = found.bundleIdentifier(); return b? String(b): null; }),
|
||||
frontmost: g(function(){ return !!found.frontmost(); }),
|
||||
windows: (function(){ var out=[]; var ws = g(function(){ return found.windows(); }) || [];
|
||||
for (var i=0;i<ws.length;i++){ out.push({ index:i, title: g(function(){ var t = ws[i].name(); return t? String(t): null; })(),
|
||||
subrole: g(function(){ var s = ws[i].subrole(); return s? String(s): null; })(),
|
||||
position: pt(function(){ return ws[i].position(); }), size: sz(function(){ return ws[i].size(); }) }); }
|
||||
return out; })() });
|
||||
}
|
||||
}`, ref);
|
||||
}
|
||||
|
||||
async function walkTree(appRef, depth = 8, maxElements = 400) {
|
||||
return jxa(`${JXA_PRELUDE}
|
||||
var se = Application('System Events');
|
||||
var procs = se.applicationProcesses();
|
||||
var found = null;
|
||||
for (var i=0;i<procs.length;i++){
|
||||
var p = procs[i];
|
||||
if (P.pid != null && num(function(){ return p.unixId(); }) === P.pid){ found = p; break; }
|
||||
if (P.bundle_id){ var b = g(function(){ return String(p.bundleIdentifier()); }); if (b && b.toLowerCase()===P.bundle_id.toLowerCase()){ found = p; break; } }
|
||||
if (P.name){ var n = g(function(){ return String(p.name()); }); if (n && n.toLowerCase()===P.name.toLowerCase()){ found = p; break; } }
|
||||
}
|
||||
if (!found) { return JSON.stringify({found:false}); }
|
||||
else {
|
||||
var elements = [];
|
||||
var truncated = false;
|
||||
function descend(el, winIdx, path, d){
|
||||
if (elements.length >= P.maxElements || d > P.depth){ if (d > P.depth) truncated = true; return; }
|
||||
var info = elInfo(el, elements.length, winIdx, path);
|
||||
elements.push(info);
|
||||
var kids = g(function(){ return el.uiElements(); }) || [];
|
||||
for (var k=0;k<kids.length;k++) descend(kids[k], winIdx, path.concat(k), d+1);
|
||||
}
|
||||
var ws = g(function(){ return found.windows(); }) || [];
|
||||
var winLimit = (P.window_id != null) ? [P.window_id] : null;
|
||||
for (var w=0; w<ws.length; w++){
|
||||
if (winLimit && winLimit.indexOf(w) === -1) continue;
|
||||
descend(ws[w], w, [], 0);
|
||||
}
|
||||
return JSON.stringify({ found: true,
|
||||
name: g(function(){return String(found.name());}), pid: num(function(){return found.unixId();}),
|
||||
bundle_id: g(function(){ var b=found.bundleIdentifier(); return b? String(b): null; }),
|
||||
frontmost: g(function(){ return !!found.frontmost(); }),
|
||||
truncated: truncated, elements: elements });
|
||||
}
|
||||
}`, { ...appRef, depth, maxElements }, 30_000);
|
||||
}
|
||||
|
||||
async function resolveElementPath(appRef, windowIndex, pathArr) {
|
||||
return jxa(`${JXA_PRELUDE}
|
||||
var se = Application('System Events');
|
||||
var procs = se.applicationProcesses();
|
||||
var found = null;
|
||||
for (var i=0;i<procs.length;i++){
|
||||
var p = procs[i];
|
||||
if (P.pid != null && num(function(){ return p.unixId(); }) === P.pid){ found = p; break; }
|
||||
if (P.bundle_id){ var b = g(function(){ return String(p.bundleIdentifier()); }); if (b && b.toLowerCase()===P.bundle_id.toLowerCase()){ found = p; break; } }
|
||||
if (P.name){ var n = g(function(){ return String(p.name()); }); if (n && n.toLowerCase()===P.name.toLowerCase()){ found = p; break; } }
|
||||
}
|
||||
if (!found) { return JSON.stringify({found:false}); }
|
||||
else {
|
||||
var ws = g(function(){ return found.windows(); }) || [];
|
||||
if (!(P.windowIndex >= 0) || P.windowIndex >= ws.length){ return JSON.stringify({found:true, element:false, reason:"window_index_missing"}); }
|
||||
else {
|
||||
var el = ws[P.windowIndex]; var ok = true;
|
||||
for (var k=0;k<P.path.length;k++){
|
||||
var kids = g(function(){ return el.uiElements(); }) || [];
|
||||
if (P.path[k] >= kids.length){ ok = false; break; }
|
||||
el = kids[P.path[k]];
|
||||
}
|
||||
if (!ok) return JSON.stringify({found:true, element:false, reason:"element_stale"});
|
||||
else {
|
||||
var info = elInfo(el, 0, P.windowIndex, P.path);
|
||||
info.app = { name: g(function(){return String(found.name());}), pid: num(function(){return found.unixId();}) };
|
||||
return JSON.stringify({ found:true, element:true, element: info });
|
||||
}
|
||||
}
|
||||
}
|
||||
}`, { ...appRef, windowIndex, path: pathArr }, 30_000);
|
||||
}
|
||||
|
||||
async function elementAction(appRef, windowIndex, pathArr, action) {
|
||||
return jxa(`${JXA_PRELUDE}
|
||||
var se = Application('System Events');
|
||||
var procs = se.applicationProcesses();
|
||||
var found = null;
|
||||
for (var i=0;i<procs.length;i++){
|
||||
var p = procs[i];
|
||||
if (P.pid != null && num(function(){ return p.unixId(); }) === P.pid){ found = p; break; }
|
||||
if (P.bundle_id){ var b = g(function(){ return String(p.bundleIdentifier()); }); if (b && b.toLowerCase()===P.bundle_id.toLowerCase()){ found = p; break; } }
|
||||
if (P.name){ var n = g(function(){ return String(p.name()); }); if (n && n.toLowerCase()===P.name.toLowerCase()){ found = p; break; } }
|
||||
}
|
||||
if (!found) { return JSON.stringify({ok:false, code:"app_not_found"}); }
|
||||
else {
|
||||
var ws = g(function(){ return found.windows(); }) || [];
|
||||
if (!(P.windowIndex >= 0) || P.windowIndex >= ws.length){ return JSON.stringify({ok:false, code:"element_stale"}); }
|
||||
else {
|
||||
var el = ws[P.windowIndex]; var ok = true;
|
||||
for (var k=0;k<P.path.length;k++){
|
||||
var kids = g(function(){ return el.uiElements(); }) || [];
|
||||
if (P.path[k] >= kids.length){ ok = false; break; }
|
||||
el = kids[P.path[k]];
|
||||
}
|
||||
if (!ok){ return JSON.stringify({ok:false, code:"element_stale"}); }
|
||||
else {
|
||||
var done = false, err = null;
|
||||
try {
|
||||
if (P.kind === 'action'){ el.actions.byName(P.action).perform(); done = true; }
|
||||
else if (P.kind === 'set_value'){ el.value = P.value; done = true; }
|
||||
else if (P.kind === 'select_text'){
|
||||
el.attributes.byName('AXSelectedTextRange').value = { loc: P.range[0], len: P.range[1] };
|
||||
done = true;
|
||||
}
|
||||
else { err = 'unknown_kind'; }
|
||||
} catch(e){ err = String(e); }
|
||||
return JSON.stringify({ ok: done && !err, code: err || 'sent', sent: done,
|
||||
element: { role: g(function(){ return String(el.role()); })(), label: g(function(){ var t=el.title(); return t?String(t):(el.name()?String(el.name()):null); })() } });
|
||||
}
|
||||
}
|
||||
}
|
||||
}`, { ...appRef, windowIndex, path: pathArr, kind: action.kind, action: action.action, value: action.value, range: action.range }, 30_000);
|
||||
}
|
||||
|
||||
// ---------- raw input via CGEvent ----------
|
||||
async function cg(script, timeoutMs = 10_000) {
|
||||
return jxa(`ObjC.import('CoreGraphics');
|
||||
function run(argv){ var P = JSON.parse(argv[0]);
|
||||
${script}
|
||||
}`, {}, timeoutMs);
|
||||
}
|
||||
|
||||
async function postMouseEvent(type, x, y, button, clickState) {
|
||||
return cg(`var pt = { x: P.x, y: P.y };
|
||||
var ev = $.CGEventCreateMouseEvent($(), P.type, pt, P.button);
|
||||
if (P.clickState > 1) $.CGEventSetIntegerValueField(ev, $.kCGMouseEventClickState, P.clickState);
|
||||
$.CGEventPost($.kCGHIDEventTap, ev);
|
||||
return JSON.stringify({ ok: true, x: P.x, y: P.y, type: P.type, button: P.button });`,
|
||||
{ type, x, y, button, clickState });
|
||||
}
|
||||
|
||||
function mouseName(button) { return { left: "left", right: "right", middle: "middle" }[button] ?? "left"; }
|
||||
|
||||
function assertInScreen(x, y) {
|
||||
if (!Number.isFinite(x) || !Number.isFinite(y)) throw new ExecError("coordinates must be finite numbers");
|
||||
}
|
||||
|
||||
async function pointerClick(button, x, y, clicks) {
|
||||
assertInScreen(x, y);
|
||||
const m = MOUSE[button] ?? MOUSE.left;
|
||||
await postMouseEvent(m.dragged, x, y, button === "middle" ? 2 : button === "right" ? 1 : 0, clicks);
|
||||
await postMouseEvent(m.down, x, y, button === "middle" ? 2 : button === "right" ? 1 : 0, clicks);
|
||||
await postMouseEvent(m.up, x, y, button === "middle" ? 2 : button === "right" ? 1 : 0, clicks);
|
||||
return { action_sent: true, strategy: "event", at: { x, y }, button, clicks };
|
||||
}
|
||||
|
||||
async function keyEvent(code, flags, down) {
|
||||
return cg(`var ev = $.CGEventCreateKeyboardEvent($(), P.code, P.down);
|
||||
if (P.flags) $.CGEventSetFlags(ev, P.flags);
|
||||
$.CGEventPost($.kCGHIDEventTap, ev);
|
||||
return JSON.stringify({ ok: true, code: P.code, down: P.down });`, { code, flags, down });
|
||||
}
|
||||
|
||||
function parseChord(text) {
|
||||
const parts = String(text).split("+").map((s) => s.trim().toLowerCase()).filter(Boolean);
|
||||
if (!parts.length) throw new ExecError("empty key text");
|
||||
let flags = 0;
|
||||
let key = null;
|
||||
for (const p of parts) {
|
||||
if (MODIFIERS[p] != null) flags |= MODIFIERS[p];
|
||||
else if (KEY_CODES[p] != null) { if (key) throw new ExecError(`multiple non-modifier keys in "${text}"`); key = p; }
|
||||
else throw new ExecError(`unknown key "${p}" (supported: ${Object.keys(KEY_CODES).join(", ")} + modifiers cmd/ctrl/alt/shift/fn)`);
|
||||
}
|
||||
if (key == null) throw new ExecError(`no non-modifier key in "${text}" — use hold_key for modifier-only holds`);
|
||||
return { flags, code: KEY_CODES[key], key };
|
||||
}
|
||||
|
||||
// ---------- displays ----------
|
||||
async function displayInfo() {
|
||||
const r = await runL("system_profiler", ["SPDisplaysDataType", "-json"], { timeoutMs: 25_000 });
|
||||
const j = tryJson(r.stdout, null);
|
||||
const items = j?.SPDisplaysDataType?.flatMap?.((g) => g.spdisplays_ndrvs ?? []) ?? [];
|
||||
// Main-display point geometry via Finder (pure AppleScript — JXA cannot
|
||||
// bridge C functions that return structs like CGRect).
|
||||
const bounds = await runL("osascript", ["-e", 'tell application "Finder" to get bounds of window of desktop'], { timeoutMs: 12_000 }).then((x) =>
|
||||
x.code === 0 ? x.stdout.trim().split(",").map((n) => Number(n.trim())) : null).catch(() => null);
|
||||
return items.map((d, i) => {
|
||||
const res = (d._spdisplays_resolution ?? "").match(/(\d+)\s*x\s*(\d+)/) ?? [null, null, null];
|
||||
return {
|
||||
index: i + 1,
|
||||
id: d._spdisplays_display_id ?? null,
|
||||
name: d._name ?? `Display ${i + 1}`,
|
||||
pixels: { w: Number(res[1]) || null, h: Number(res[2]) || null },
|
||||
main: String(d.spdisplays_main ?? "n").toLowerCase() === "y" || i === 0,
|
||||
};
|
||||
}).map((d, i) => ({
|
||||
...d,
|
||||
// Point geometry is only precisely known for the main display; other
|
||||
// displays get best-effort placement to the right of the main display.
|
||||
points: i === 0 && bounds && bounds.length === 4
|
||||
? { x: bounds[0], y: bounds[1], w: bounds[2] - bounds[0], h: bounds[3] - bounds[1] }
|
||||
: { x: null, y: null, w: null, h: null },
|
||||
scale: d.pixels.w && i === 0 && bounds?.length === 4 && bounds[2] - bounds[0] > 0
|
||||
? +(d.pixels.w / (bounds[2] - bounds[0])).toFixed(3) : 1,
|
||||
}));
|
||||
}
|
||||
|
||||
// ---------- screenshots ----------
|
||||
function recordingsDir() {
|
||||
return process.env.CODEWHALE_CU_RECORDINGS_DIR || path.join(os.homedir(), ".codewhale-cu", "recordings");
|
||||
}
|
||||
|
||||
async function screenshot({ display, region, path: outPath } = {}) {
|
||||
const dir = recordingsDir();
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
const file = outPath || path.join(dir, `shot-${new Date().toISOString().replace(/[:.]/g, "-")}-${crypto.randomBytes(3).toString("hex")}.png`);
|
||||
if (!/\.png$/.test(file)) throw new ExecError("screenshot path must end in .png");
|
||||
const args = ["-x", "-t", "png"];
|
||||
const disp = display ?? state.activeDisplay;
|
||||
if (disp && disp !== "all") args.push("-D", String(disp));
|
||||
if (region) {
|
||||
if (!region.every((n) => Number.isFinite(n) && n >= 0) || region.length !== 4) {
|
||||
throw new ExecError("region must be [x, y, w, h] in screen points");
|
||||
}
|
||||
args.push("-R", region.join(","));
|
||||
}
|
||||
args.push(file);
|
||||
const r = await runL("screencapture", args, { timeoutMs: 20_000 });
|
||||
if (r.code !== 0) throw new ExecError(`screencapture exited ${r.code}: ${r.stderr.trim().slice(0, 300)}`, r);
|
||||
const stat = fs.statSync(file);
|
||||
const displays = await displayInfo();
|
||||
const d = displays.find((x) => x.index === (disp === "all" ? 1 : disp)) ?? displays[0];
|
||||
state.lastRaster = {
|
||||
file,
|
||||
bytes: stat.size,
|
||||
display: disp ?? 1,
|
||||
points: d?.points ?? null,
|
||||
pixels: d?.pixels ?? null,
|
||||
scale: d?.scale ?? 1,
|
||||
capturedAt: new Date().toISOString(),
|
||||
};
|
||||
return { ...state.lastRaster, path: file };
|
||||
}
|
||||
|
||||
async function zoom({ source, region, path: outPath }) {
|
||||
if (!source && !state.lastRaster) throw new ExecError("no screenshot taken yet on this computer — call screenshot first");
|
||||
const [x, y, w, h] = region;
|
||||
if (![x, y, w, h].every((n) => Number.isFinite(n) && n >= 0)) throw new ExecError("region must be [x, y, w, h] in last-raster pixels");
|
||||
const src = source ?? state.lastRaster.file;
|
||||
const dir = recordingsDir();
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
const out = outPath || path.join(dir, `zoom-${crypto.randomBytes(4).toString("hex")}.png`);
|
||||
await runOk("sips", ["-s", "format", "png", "-c", String(Math.round(h)), String(Math.round(w)), "--cropOffset", String(Math.round(y)), String(Math.round(x)), src, "--out", out], { timeoutMs: 15_000 });
|
||||
return { file: out, bytes: fs.statSync(out).size, source: src, region, scale: state.lastRaster.scale };
|
||||
}
|
||||
|
||||
// ---------- recording ----------
|
||||
const rec = new Map(); // id -> {pid, file, startedAt, mode}
|
||||
|
||||
async function recordingStart({ display, durationSec, region } = {}) {
|
||||
const dir = recordingsDir();
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
const id = crypto.randomBytes(4).toString("hex");
|
||||
const file = path.join(dir, `rec-${id}.mov`);
|
||||
const args = ["-v", "-x"];
|
||||
const disp = display ?? state.activeDisplay;
|
||||
if (disp && disp !== "all") args.push("-D", String(disp));
|
||||
if (durationSec) args.push("-V", String(Math.max(1, Math.round(durationSec))));
|
||||
if (region) args.unshift("-R", region.join(","));
|
||||
args.push(file);
|
||||
const child = spawn("screencapture", args, { stdio: "ignore", detached: true });
|
||||
child.unref();
|
||||
const startedAt = new Date().toISOString();
|
||||
rec.set(id, { pid: child.pid, file, startedAt, mode: "screencapture", display: disp ?? 1 });
|
||||
if (durationSec) {
|
||||
const timer = setTimeout(() => rec.delete(id), (Math.round(durationSec) + 10) * 1000);
|
||||
timer.unref?.();
|
||||
}
|
||||
await new Promise((res) => setTimeout(res, 400));
|
||||
try { process.kill(child.pid, 0); } catch {
|
||||
rec.delete(id);
|
||||
throw new ExecError("screencapture -v exited immediately — screen recording permission (Screen & System Recording) is likely missing for the host terminal");
|
||||
}
|
||||
return { id, pid: child.pid, file, display: disp ?? 1, durationSec: durationSec ?? null, region: region ?? null, fps: "device-default", startedAt };
|
||||
}
|
||||
|
||||
async function recordingStop({ id }) {
|
||||
const r = rec.get(id);
|
||||
if (!r) throw new ExecError(`unknown or already-finished recording "${id}" (recording_status/recording_list shows current state)`);
|
||||
let mp4 = null;
|
||||
try { process.kill(r.pid, "SIGINT"); } catch {}
|
||||
await new Promise((res) => setTimeout(res, 1500));
|
||||
const ffmpeg = await import("../exec.mjs").then((m) => m.have("ffmpeg"));
|
||||
if (ffmpeg && fs.existsSync(r.file)) {
|
||||
const out = r.file.replace(/\.mov$/, ".mp4");
|
||||
const rr = await runL("ffmpeg", ["-y", "-loglevel", "error", "-i", r.file, "-c", "copy", out], { timeoutMs: 120_000 });
|
||||
if (rr.code === 0) mp4 = out;
|
||||
}
|
||||
const size = fs.existsSync(r.file) ? fs.statSync(r.file).size : 0;
|
||||
rec.delete(id);
|
||||
return { id, file: r.file, mp4, bytes: size, startedAt: r.startedAt, stoppedAt: new Date().toISOString() };
|
||||
}
|
||||
|
||||
async function recordingStatus({ id }) {
|
||||
const r = rec.get(id);
|
||||
if (!r) return { id, running: false };
|
||||
let alive = true;
|
||||
try { process.kill(r.pid, 0); } catch { alive = false; }
|
||||
return { id, running: alive, pid: r.pid, file: r.file, bytes: fs.existsSync(r.file) ? fs.statSync(r.file).size : 0, startedAt: r.startedAt };
|
||||
}
|
||||
|
||||
async function recordingList() {
|
||||
const dir = recordingsDir();
|
||||
const out = [];
|
||||
for (const f of fs.existsSync(dir) ? fs.readdirSync(dir) : []) {
|
||||
const full = path.join(dir, f);
|
||||
const st = fs.statSync(full);
|
||||
if (st.isFile() && /\.(mov|mp4|png)$/i.test(f)) out.push({ file: full, bytes: st.size, modifiedAt: st.mtime.toISOString() });
|
||||
}
|
||||
out.sort((a, b) => b.modifiedAt.localeCompare(a.modifiedAt));
|
||||
return { dir, recordings: out.slice(0, 50), running: [...rec.keys()] };
|
||||
}
|
||||
|
||||
// ---------- apps / windows ----------
|
||||
async function listApps() {
|
||||
return jxa(`${JXA_PRELUDE}
|
||||
var se = Application('System Events');
|
||||
var procs = se.applicationProcesses();
|
||||
var out = [];
|
||||
for (var i=0;i<procs.length;i++){
|
||||
var p = procs[i];
|
||||
var bg = g(function(){ return !!p.backgroundOnly(); });
|
||||
var name = g(function(){ return String(p.name()); });
|
||||
var wins = g(function(){ return p.windows(); });
|
||||
out.push({ name: name, pid: num(function(){ return p.unixId(); }),
|
||||
bundle_id: g(function(){ var b = p.bundleIdentifier(); return b? String(b): null; }),
|
||||
frontmost: g(function(){ return !!p.frontmost(); }),
|
||||
hidden: g(function(){ return !!p.hidden(); }),
|
||||
windowCount: wins ? wins.length : 0 });
|
||||
}
|
||||
return JSON.stringify({ apps: out });
|
||||
}`, {}, 25_000);
|
||||
}
|
||||
|
||||
async function listWindows(appRef) {
|
||||
const p = await findProcess(appRef ?? {});
|
||||
if (!p.found) throw new ExecError("application not found — call list_apps for exact names/pids");
|
||||
return { app: { name: p.name, pid: p.pid, bundle_id: p.bundle_id }, windows: p.windows };
|
||||
}
|
||||
|
||||
async function openApplication({ name, bundle_id: bid, pid, url: urlArg, activate = false } = {}) {
|
||||
if (!name && !bid) throw new ExecError("open_application needs name or bundle_id");
|
||||
const args = [];
|
||||
if (urlArg) args.push(urlArg);
|
||||
if (bid) args.unshift("-b", bid); else args.unshift("-a", name);
|
||||
if (activate) args.unshift("-F");
|
||||
const r = await runL("open", args, { timeoutMs: 25_000 });
|
||||
if (r.code !== 0) throw new ExecError(`open failed: ${r.stderr.trim().slice(0, 200)}`, r);
|
||||
await new Promise((res) => setTimeout(res, 600));
|
||||
const find = {};
|
||||
if (bid) find.bundle_id = bid; else if (pid) find.pid = pid; else find.name = String(name).replace(/\.app$/, "");
|
||||
const p = await findProcess(find).catch(() => null);
|
||||
return { launched: true, activate, url: urlArg ?? null, resolved: p?.found ? { name: p.name, pid: p.pid, bundle_id: p.bundle_id, frontmost: p.frontmost } : null };
|
||||
}
|
||||
|
||||
// ---------- clipboard / cursor / waits ----------
|
||||
async function readClipboard() {
|
||||
const r = await runL("pbpaste", [], { timeoutMs: 5_000, maxBuffer: 4 * 1024 * 1024 });
|
||||
return { text: r.stdout, encoding: "utf8" };
|
||||
}
|
||||
async function writeClipboard({ text }) {
|
||||
const child = spawn("pbcopy", [], { stdio: ["pipe", "ignore", "ignore"] });
|
||||
child.stdin.end(String(text ?? ""));
|
||||
await new Promise((res, rej) => { child.on("close", res); child.on("error", rej); });
|
||||
return { written: String(text ?? "").length };
|
||||
}
|
||||
async function cursorPosition() {
|
||||
// Quartz (pyobjc) first — JXA cannot bridge CGEventGetLocation's CGPoint.
|
||||
const py = `from Quartz import CGEventCreate
|
||||
l = CGEventCreate(None).location
|
||||
print('{\"x\": %d, \"y\": %d}' % (l.x, l.y))`;
|
||||
const r = await runL("python3", ["-c", py], { timeoutMs: 8_000 });
|
||||
if (r.code === 0) {
|
||||
const j = tryJson(r.stdout.trim(), null);
|
||||
if (j && Number.isFinite(j.x)) return { x: j.x, y: j.y };
|
||||
}
|
||||
const cc = await runL("cliclick", ["p"], { timeoutMs: 8_000 });
|
||||
if (cc.code === 0) {
|
||||
const m = /(-?\d+)\s*,\s*(-?\d+)/.exec(cc.stdout.trim());
|
||||
if (m) return { x: Number(m[1]), y: Number(m[2]) };
|
||||
}
|
||||
throw new ExecError("cursor position needs python3 with pyobjc (Quartz) or cliclick on PATH");
|
||||
}
|
||||
|
||||
// ---------- probe ----------
|
||||
async function probe() {
|
||||
const caps = { screenshot: true, recording: true, accessibility_tree: true, clipboard: true, displays: true };
|
||||
const perms = {};
|
||||
try { await jxa(`function run(argv){ return JSON.stringify({n: Application('System Events').applicationProcesses.length}); }`, {}, 8_000); perms.accessibility = "granted"; }
|
||||
catch (e) { perms.accessibility = "denied_or_unavailable"; caps.accessibility_tree = false; caps.raw_input = "unreliable"; }
|
||||
try {
|
||||
const t = os.tmpdir() + `/cu-probe-${crypto.randomBytes(3).toString("hex")}.png`;
|
||||
const r = await runL("screencapture", ["-x", "-R0,0,2,2", "-t", "png", t], { timeoutMs: 8_000 });
|
||||
perms.screen_capture = r.code === 0 ? "ok" : "failed";
|
||||
try { fs.rmSync(t, { force: true }); } catch {}
|
||||
} catch { perms.screen_capture = "failed"; }
|
||||
const hasRecording = fs.existsSync("/usr/sbin/screencapture");
|
||||
return { platform: "darwin", capabilities: caps, permissions: perms, note: "macOS does not expose Screen-Recording TCC state to CLI; a black/empty screenshot means Screen Recording permission is missing. Raw pointer/keyboard events go to whatever is frontmost at the target point — activate the app first for click-type actions." };
|
||||
}
|
||||
|
||||
return {
|
||||
platform: "darwin",
|
||||
probe,
|
||||
list_displays: displayInfo,
|
||||
async switch_display({ index }) {
|
||||
const ds = await displayInfo();
|
||||
if (!ds.some((d) => d.index === index)) throw new ExecError(`no display ${index}; have [${ds.map((d) => d.index).join(", ")}]`);
|
||||
state.activeDisplay = index;
|
||||
return { activeDisplay: index };
|
||||
},
|
||||
list_apps: listApps,
|
||||
list_windows: listWindows,
|
||||
open_application: openApplication,
|
||||
get_app_state: async ({ app_ref, detail, depth }) => {
|
||||
const t = await walkTree(app_ref ?? {}, detail === "full" ? 12 : 8, detail === "full" ? 800 : 400);
|
||||
if (!t.found) throw new ExecError("application not found — call list_apps for exact names/pids");
|
||||
return t;
|
||||
},
|
||||
screenshot,
|
||||
zoom,
|
||||
left_click: ({ target }) => pointerClick("left", target.x, target.y, 1),
|
||||
double_click: ({ target }) => pointerClick("left", target.x, target.y, 2),
|
||||
triple_click: ({ target }) => pointerClick("left", target.x, target.y, 3),
|
||||
right_click: ({ target }) => pointerClick("right", target.x, target.y, 1),
|
||||
middle_click: ({ target }) => pointerClick("middle", target.x, target.y, 1),
|
||||
mouse_move: async ({ target }) => {
|
||||
assertInScreen(target.x, target.y);
|
||||
await postMouseEvent(5, target.x, target.y, 0, 0);
|
||||
return { action_sent: true, at: { x: target.x, y: target.y } };
|
||||
},
|
||||
left_mouse_down: async ({ target }) => { assertInScreen(target.x, target.y); await postMouseEvent(MOUSE.left.down, target.x, target.y, 0, 1); return { action_sent: true }; },
|
||||
left_mouse_up: async ({ target }) => {
|
||||
const loc = await cursorPosition();
|
||||
await postMouseEvent(MOUSE.left.up, loc.x, loc.y, 0, 1);
|
||||
return { action_sent: true };
|
||||
},
|
||||
left_click_drag: async ({ from_target: from, to }) => {
|
||||
assertInScreen(from.x, from.y); assertInScreen(to.x, to.y);
|
||||
await postMouseEvent(MOUSE.left.down, from.x, from.y, 0, 1);
|
||||
const steps = 12;
|
||||
for (let i = 1; i <= steps; i++) {
|
||||
await new Promise((r) => setTimeout(r, 24));
|
||||
await postMouseEvent(MOUSE.left.dragged, from.x + ((to.x - from.x) * i) / steps, from.y + ((to.y - from.y) * i) / steps, 0, 1);
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 60));
|
||||
await postMouseEvent(MOUSE.left.up, to.x, to.y, 0, 1);
|
||||
return { action_sent: true, from, to };
|
||||
},
|
||||
scroll: async ({ target, direction = "down", amount = 5 }) => {
|
||||
assertInScreen(target.x, target.y);
|
||||
await postMouseEvent(5, target.x, target.y, 0, 0);
|
||||
const dx = direction === "left" ? -amount : direction === "right" ? amount : 0;
|
||||
const dy = direction === "up" ? amount : direction === "down" ? -amount : 0;
|
||||
return cg(`var ev = $.CGEventCreateScrollWheelEvent($(), 1, 2, P.dy, P.dx);
|
||||
$.CGEventPost($.kCGHIDEventTap, ev);
|
||||
return JSON.stringify({ ok: true });`, { dx, dy }).then(() => ({ action_sent: true, direction, amount }));
|
||||
},
|
||||
type: async ({ text }) => {
|
||||
if (!text) return { action_sent: false, note: "empty text" };
|
||||
const r = await cg(`var ev = $.CGEventCreateKeyboardEvent($(), 0, true);
|
||||
$.CGEventKeyboardSetUnicodeString(ev, P.text.length, P.text);
|
||||
$.CGEventPost($.kCGHIDEventTap, ev);
|
||||
var ev2 = $.CGEventCreateKeyboardEvent($(), 0, false);
|
||||
$.CGEventKeyboardSetUnicodeString(ev2, P.text.length, P.text);
|
||||
$.CGEventPost($.kCGHIDEventTap, ev2);
|
||||
return JSON.stringify({ ok: true, chars: P.text.length });`, { text }, 15_000);
|
||||
return { action_sent: true, chars: text.length, strategy: "unicode-events" };
|
||||
},
|
||||
key: async ({ text, repeat = 1 }) => {
|
||||
const { flags, code, key } = parseChord(text);
|
||||
for (let i = 0; i < Math.max(1, Math.min(100, repeat)); i++) {
|
||||
await keyEvent(code, flags, true);
|
||||
await keyEvent(code, flags, false);
|
||||
if (i < repeat - 1) await new Promise((r) => setTimeout(r, 30));
|
||||
}
|
||||
return { action_sent: true, key, code, repeat: Math.max(1, Math.min(100, repeat)) };
|
||||
},
|
||||
hold_key: async ({ text, duration }) => {
|
||||
const { flags, code, key } = parseChord(text);
|
||||
const d = Math.max(0.05, Math.min(30, Number(duration) || 1));
|
||||
await keyEvent(code, flags, true);
|
||||
await new Promise((r) => setTimeout(r, d * 1000));
|
||||
await keyEvent(code, flags, false);
|
||||
return { action_sent: true, key, heldSec: d };
|
||||
},
|
||||
set_value: async ({ target, value }) => {
|
||||
const el = await elementAction(target.app_ref, target.windowIndex, target.path, { kind: "set_value", value });
|
||||
if (!el.ok) throw new ExecError(`set_value failed: ${el.code}`);
|
||||
return { action_sent: true, strategy: "a11y", element: el.element };
|
||||
},
|
||||
select_text: async ({ target, text_range }) => {
|
||||
const el = await elementAction(target.app_ref, target.windowIndex, target.path, { kind: "select_text", range: text_range });
|
||||
if (!el.ok) throw new ExecError(`select_text failed: ${el.code}`);
|
||||
return { action_sent: true, strategy: "a11y", element: el.element };
|
||||
},
|
||||
perform_action: async ({ target, action }) => {
|
||||
const el = await elementAction(target.app_ref, target.windowIndex, target.path, { kind: "action", action });
|
||||
if (!el.ok) throw new ExecError(`perform_action failed: ${el.code} — check the element's actions list from get_app_state`);
|
||||
return { action_sent: el.sent, strategy: "a11y", action, element: el.element };
|
||||
},
|
||||
read_clipboard: readClipboard,
|
||||
write_clipboard: writeClipboard,
|
||||
cursor_position: cursorPosition,
|
||||
recordingStart,
|
||||
recordingStop,
|
||||
recordingStatus,
|
||||
recordingList,
|
||||
};
|
||||
}
|
||||
|
||||
export default { create };
|
||||
@@ -0,0 +1,283 @@
|
||||
// HarmonyOS backend — drives a device/emulator through `hdc` (HarmonyOS
|
||||
// Device Connector) plus the on-device `uitest` and `snapshot_display` tools.
|
||||
// Observation: `uitest dumpLayout` (the accessibility-tree equivalent).
|
||||
// Input: `uitest uiInput` (click / swipe / inputText / keyEvent).
|
||||
// Stills: `snapshot_display`. Video: no CLI screen recorder exists on
|
||||
// current HarmonyOS shells, so recording is an honest
|
||||
// snapshot-series mode muxed with ffmpeg on the host.
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import crypto from "node:crypto";
|
||||
import { run, runOk, ExecError, tryJson, have } from "../exec.mjs";
|
||||
|
||||
const DEVICE_TMP = "/data/local/tmp/cu";
|
||||
|
||||
function escDeviceText(s) {
|
||||
if (/[^\x20-\x7E]/.test(s)) throw new ExecError("harmony uiInput text must be printable ASCII on this backend");
|
||||
return `'${String(s).replace(/'/g, `'\\''`)}'`;
|
||||
}
|
||||
|
||||
function jpegSize(buf) {
|
||||
// Minimal JPEG SOF parser — enough to learn the panel size of a snapshot.
|
||||
let i = 2;
|
||||
while (i + 9 < buf.length) {
|
||||
if (buf[i] !== 0xff) { i++; continue; }
|
||||
const marker = buf[i + 1];
|
||||
if (marker >= 0xc0 && marker <= 0xcf && ![0xc4, 0xc8, 0xcc].includes(marker)) {
|
||||
return { h: buf.readUInt16BE(i + 5), w: buf.readUInt16BE(i + 7) };
|
||||
}
|
||||
i += 2 + buf.readUInt16BE(i + 2);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function parseBounds(b) {
|
||||
const m = /\[(\d+),(\d+)\]\[(\d+),(\d+)\]/.exec(String(b ?? ""));
|
||||
if (!m) return null;
|
||||
const [x1, y1, x2, y2] = [Number(m[1]), Number(m[2]), Number(m[3]), Number(m[4])];
|
||||
return { x: x1, y: y1, w: x2 - x1, h: y2 - y1, cx: Math.round((x1 + x2) / 2), cy: Math.round((y1 + y2) / 2) };
|
||||
}
|
||||
|
||||
export function flatten(node, pathArr = [], out = []) {
|
||||
if (!node || out.length >= 600) return out;
|
||||
const a = node.attributes ?? {};
|
||||
out.push({
|
||||
index: out.length,
|
||||
path: pathArr,
|
||||
role: a.type ?? "node",
|
||||
label: a.text || a.id || a.description || a.name || null,
|
||||
value: a.text ?? null,
|
||||
bounds: parseBounds(a.bounds),
|
||||
attributes: a,
|
||||
actions: ["click", "longClick", "inputText"],
|
||||
});
|
||||
for (let i = 0; i < (node.children?.length ?? 0); i++) flatten(node.children[i], [...pathArr, i], out);
|
||||
return out;
|
||||
}
|
||||
|
||||
export function create({ exec }) {
|
||||
const shell = (args, opts = {}) => exec.shell(args, { timeoutMs: 20_000, ...opts });
|
||||
|
||||
async function deviceOut(args, opts = {}) {
|
||||
const r = await shell(args, opts);
|
||||
if (r.code !== 0) throw new ExecError(`hdc shell ${args[0]} exited ${r.code}: ${(r.stderr || r.stdout).trim().slice(0, 300)}`, r);
|
||||
return r.stdout;
|
||||
}
|
||||
|
||||
async function snapshot(localPath) {
|
||||
const remote = `${DEVICE_TMP}-${crypto.randomBytes(3).toString("hex")}.jpeg`;
|
||||
await deviceOut(["snapshot_display", "-f", remote], { timeoutMs: 25_000 });
|
||||
try {
|
||||
await exec.pullFile(remote, localPath, { timeoutMs: 30_000 });
|
||||
} finally {
|
||||
await shell(["rm", "-f", remote]).catch(() => {});
|
||||
}
|
||||
return localPath;
|
||||
}
|
||||
|
||||
async function dumpLayout() {
|
||||
const remote = `${DEVICE_TMP}-layout.json`;
|
||||
await deviceOut(["uitest", "dumpLayout", "-p", remote], { timeoutMs: 40_000 });
|
||||
let data;
|
||||
try {
|
||||
data = await exec.readFile(remote, { timeoutMs: 30_000 });
|
||||
} finally {
|
||||
await shell(["rm", "-f", remote]).catch(() => {});
|
||||
}
|
||||
return JSON.parse(data.toString("utf8"));
|
||||
}
|
||||
|
||||
async function uiInput(args, opts = {}) {
|
||||
await deviceOut(["uitest", "uiInput", ...args], opts);
|
||||
return { action_sent: true, strategy: "event", backend: "uitest" };
|
||||
}
|
||||
|
||||
async function centerOf(target) {
|
||||
const tree = await dumpLayout();
|
||||
const els = flatten(tree);
|
||||
const el = els[target.index];
|
||||
if (!el || !el.bounds) throw new ExecError("element_stale — re-run get_app_state; uitest indexes change with the UI");
|
||||
return el.bounds;
|
||||
}
|
||||
|
||||
let frameSeq = 0;
|
||||
let recording = null; // {id, dir, startedAt, intervalMs, timer, display}
|
||||
let displayPixels = null;
|
||||
|
||||
return {
|
||||
platform: "harmonyos",
|
||||
probe: async () => {
|
||||
const r = await exec.run("hdc", [...exec.targetArgs, "list", "targets"], { timeoutMs: 10_000 });
|
||||
const targets = r.stdout.trim().split("\n").filter(Boolean);
|
||||
const connected = r.code === 0 && targets.length > 0 && !targets.includes("[Empty]");
|
||||
return {
|
||||
platform: "harmonyos",
|
||||
connected,
|
||||
targets,
|
||||
capabilities: { screenshot: connected, accessibility_tree: connected, clipboard: false, recording: "snapshot-series" },
|
||||
note: "HarmonyOS drives the device over hdc. Clipboard read/write is not exposed by hdc and fails closed. Recording muxes snapshot_display frames with ffmpeg.",
|
||||
};
|
||||
},
|
||||
list_displays: async () => {
|
||||
if (!displayPixels) {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "cu-hm-"));
|
||||
try {
|
||||
const shot = path.join(dir, "probe.jpeg");
|
||||
await snapshot(shot);
|
||||
displayPixels = jpegSize(fs.readFileSync(shot)) ?? { w: null, h: null };
|
||||
} finally { fs.rmSync(dir, { recursive: true, force: true }).catch(() => {}); }
|
||||
}
|
||||
return [{ index: 1, name: "device", pixels: displayPixels, points: displayPixels, scale: 1, main: true }];
|
||||
},
|
||||
async switch_display({ index }) {
|
||||
if (index !== 1) throw new ExecError("harmony backend exposes display 1 only");
|
||||
return { activeDisplay: 1 };
|
||||
},
|
||||
list_apps: async () => {
|
||||
const out = await deviceOut(["bm", "dump", "-a"], { timeoutMs: 25_000 });
|
||||
const bundles = out.split("\n").map((s) => s.trim()).filter((s) => /^[a-zA-Z][\w.]*$/.test(s));
|
||||
return { apps: bundles.map((b) => ({ name: b, bundle_id: b, kind: "bundle" })) };
|
||||
},
|
||||
list_windows: async () => {
|
||||
const out = await deviceOut(["hidumper", "-s", "WindowManagerService", "-a", "-a"], { timeoutMs: 25_000 }).catch(() => "");
|
||||
const windows = out.split("\n").filter((l) => /Window Name|bundleName/i.test(l)).slice(0, 40).map((l) => ({ title: l.trim().slice(0, 160) }));
|
||||
return { windows: windows.length ? windows : [{ title: "(window list unavailable on this HarmonyOS build)" }] };
|
||||
},
|
||||
open_application: async ({ bundle_id: bid, ability, name } = {}) => {
|
||||
const bundle = bid ?? name;
|
||||
if (!bundle) throw new ExecError("open_application needs bundle_id (harmony bundle name)");
|
||||
const candidates = ability ? [ability] : ["EntryAbility", "MainAbility"];
|
||||
let last = null;
|
||||
for (const a of candidates) {
|
||||
const r = await shell(["aa", "start", "-b", bundle, "-a", a]);
|
||||
if (r.code === 0 && !/Error|error/.test(r.stdout + r.stderr)) {
|
||||
return { launched: true, bundle, ability: a };
|
||||
}
|
||||
last = (r.stderr || r.stdout).trim().slice(0, 200);
|
||||
}
|
||||
throw new ExecError(`aa start failed: ${last}`);
|
||||
},
|
||||
get_app_state: async () => {
|
||||
const tree = await dumpLayout();
|
||||
const els = flatten(tree);
|
||||
return { bundle_id: tree.attributes?.bundleName ?? null, elements: els, truncated: els.length >= 600 };
|
||||
},
|
||||
screenshot: async ({ path: outPath } = {}) => {
|
||||
const dir = process.env.CODEWHALE_CU_RECORDINGS_DIR || path.join(os.homedir(), ".codewhale-cu", "recordings");
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
const file = outPath || path.join(dir, `shot-${new Date().toISOString().replace(/[:.]/g, "-")}-${crypto.randomBytes(3).toString("hex")}.jpeg`);
|
||||
await snapshot(file);
|
||||
const buf = fs.readFileSync(file);
|
||||
displayPixels = jpegSize(buf) ?? displayPixels;
|
||||
return { file, bytes: buf.length, pixels: jpegSize(buf), scale: 1, points: jpegSize(buf) };
|
||||
},
|
||||
zoom: async ({ region, path: outPath }) => {
|
||||
throw new ExecError("zoom is not supported on the harmony backend yet — screenshot + region on the host is the workaround");
|
||||
},
|
||||
left_click: ({ target }) => uiInput(["click", String(Math.round(target.x)), String(Math.round(target.y))]),
|
||||
double_click: ({ target }) => uiInput(["doubleClick", String(Math.round(target.x)), String(Math.round(target.y))]),
|
||||
triple_click: async ({ target }) => {
|
||||
await uiInput(["doubleClick", String(Math.round(target.x)), String(Math.round(target.y))]);
|
||||
return uiInput(["click", String(Math.round(target.x)), String(Math.round(target.y))]);
|
||||
},
|
||||
right_click: async () => { throw new ExecError("uitest uiInput has no right-click; use longClick semantics via hold or left_click"); },
|
||||
middle_click: async () => { throw new ExecError("middle click is not exposed by uitest uiInput"); },
|
||||
mouse_move: async () => ({ action_sent: false, note: "hover without press is not exposed by uitest uiInput" }),
|
||||
left_click_drag: ({ from_target: from, to }) =>
|
||||
uiInput(["swipe", String(Math.round(from.x)), String(Math.round(from.y)), String(Math.round(to.x)), String(Math.round(to.y)), "200"], { timeoutMs: 30_000 }),
|
||||
left_mouse_down: async () => { throw new ExecError("low-level press/release is not exposed by uitest uiInput; use left_click_drag"); },
|
||||
left_mouse_up: async () => { throw new ExecError("low-level press/release is not exposed by uitest uiInput; use left_click_drag"); },
|
||||
scroll: ({ target, direction = "down", amount = 300 }) => {
|
||||
const dist = Math.max(60, Math.min(1200, amount * 24));
|
||||
const dx = direction === "left" ? dist : direction === "right" ? -dist : 0;
|
||||
const dy = direction === "up" ? dist : direction === "down" ? -dist : 0;
|
||||
return uiInput(["swipe", String(Math.round(target.x)), String(Math.round(target.y)), String(Math.round(target.x + dx)), String(Math.round(target.y + dy)), "400"]);
|
||||
},
|
||||
type: async ({ text }) => {
|
||||
if (!text) return { action_sent: false, note: "empty text" };
|
||||
await uiInput(["inputText", "300", "300", escDeviceText(text)]).catch(async (e) => {
|
||||
// Some builds require coordinates of the focused field; retry with a click-first pattern.
|
||||
throw e;
|
||||
});
|
||||
return { action_sent: true, chars: text.length, note: "inputText at 300,300 — click the field first for focused input" };
|
||||
},
|
||||
key: ({ text }) => {
|
||||
const KEYMAP = { enter: "Enter", return: "Enter", escape: "Esc", esc: "Esc", back: "Back", home: "Home", backspace: "Back", delete: "Del", tab: "Tab", left: "DPAD_LEFT", right: "DPAD_RIGHT", up: "DPAD_UP", down: "DPAD_DOWN", power: "Power", menu: "Menu" };
|
||||
const k = KEYMAP[String(text).toLowerCase()] ?? String(text);
|
||||
if (!/^[A-Za-z0-9_]+$/.test(k)) throw new ExecError(`unsupported key "${text}" on harmony backend`);
|
||||
return uiInput(["keyEvent", k]);
|
||||
},
|
||||
hold_key: async ({ text, duration }) => {
|
||||
if (String(text).toLowerCase() !== "click") throw new ExecError('hold_key on harmony supports only hold_key({"text":"click"}) = longClick');
|
||||
const d = Math.max(1, Math.min(5, Number(duration) || 1));
|
||||
return uiInput(["longClick", "300", "300"]);
|
||||
},
|
||||
set_value: async ({ target, value }) => {
|
||||
const b = await centerOf(target);
|
||||
await uiInput(["click", String(b.cx), String(b.cy)]);
|
||||
await new Promise((r) => setTimeout(r, 300));
|
||||
await uiInput(["inputText", String(b.cx), String(b.cy), escDeviceText(String(value))]);
|
||||
return { action_sent: true, strategy: "uitest-element" };
|
||||
},
|
||||
select_text: async () => { throw new ExecError("select_text is not exposed by uitest dumpLayout/uiInput on the harmony backend"); },
|
||||
perform_action: async ({ target, action }) => {
|
||||
const b = await centerOf(target);
|
||||
if (action === "longClick") return uiInput(["longClick", String(b.cx), String(b.cy)]);
|
||||
return uiInput(["click", String(b.cx), String(b.cy)]);
|
||||
},
|
||||
read_clipboard: async () => { throw new ExecError("clipboard read is not exposed by hdc on current HarmonyOS builds"); },
|
||||
write_clipboard: async () => { throw new ExecError("clipboard write is not exposed by hdc on current HarmonyOS builds"); },
|
||||
cursor_position: async () => { throw new ExecError("cursor position does not exist on touch devices"); },
|
||||
recordingStart: async ({ intervalMs = 400 } = {}) => {
|
||||
if (recording) throw new ExecError(`recording ${recording.id} already running`);
|
||||
if (!(await have("ffmpeg"))) throw new ExecError("ffmpeg is required on the host to mux harmony snapshot-series recordings");
|
||||
const id = crypto.randomBytes(4).toString("hex");
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), `cu-rec-${id}-`));
|
||||
const startedAt = new Date().toISOString();
|
||||
recording = { id, dir, startedAt, intervalMs, seq: 0 };
|
||||
const tick = async () => {
|
||||
if (!recording || recording.id !== id) return;
|
||||
try {
|
||||
const remote = `${DEVICE_TMP}-rec-${String(recording.seq).padStart(5, "0")}.jpeg`;
|
||||
await deviceOut(["snapshot_display", "-f", remote], { timeoutMs: 15_000 });
|
||||
await exec.pullFile(remote, path.join(dir, `f${String(recording.seq).padStart(5, "0")}.jpeg`), { timeoutMs: 15_000 });
|
||||
await shell(["rm", "-f", remote]).catch(() => {});
|
||||
recording.seq++;
|
||||
} catch {}
|
||||
};
|
||||
await tick();
|
||||
const timer = setInterval(tick, Math.max(150, intervalMs));
|
||||
recording.timer = timer;
|
||||
return { id, mode: "snapshot-series", intervalMs, startedAt, note: "no HarmonyOS CLI screen recorder; frames are muxed into mp4 on stop" };
|
||||
},
|
||||
recordingStop: async ({ id }) => {
|
||||
if (!recording || recording.id !== id) throw new ExecError(`unknown recording "${id}"`);
|
||||
clearInterval(recording.timer);
|
||||
const { dir, seq, startedAt, intervalMs } = recording;
|
||||
recording = null;
|
||||
const dirOut = process.env.CODEWHALE_CU_RECORDINGS_DIR || path.join(os.homedir(), ".codewhale-cu", "recordings");
|
||||
fs.mkdirSync(dirOut, { recursive: true });
|
||||
const out = path.join(dirOut, `rec-${id}.mp4`);
|
||||
const fps = Math.max(1, Math.min(15, Math.round(1000 / Math.max(150, intervalMs))));
|
||||
const r = await run("ffmpeg", ["-y", "-loglevel", "error", "-framerate", String(fps), "-i", path.join(dir, "f%05d.jpeg"), "-c:v", "libx264", "-pix_fmt", "yuv420p", out], { timeoutMs: 180_000 });
|
||||
const bytes = fs.existsSync(out) ? fs.statSync(out).size : 0;
|
||||
try { fs.rmSync(dir, { recursive: true, force: true }); } catch {}
|
||||
if (r.code !== 0) throw new ExecError(`ffmpeg mux failed: ${(r.stderr || "").slice(0, 300)}`, r);
|
||||
return { id, mode: "snapshot-series", frames: seq, fps, file: out, bytes, startedAt, stoppedAt: new Date().toISOString() };
|
||||
},
|
||||
recordingStatus: ({ id }) => recording && recording.id === id
|
||||
? { id, running: true, mode: "snapshot-series", frames: recording.seq, startedAt: recording.startedAt }
|
||||
: { id, running: false },
|
||||
recordingList: async () => {
|
||||
const dir = process.env.CODEWHALE_CU_RECORDINGS_DIR || path.join(os.homedir(), ".codewhale-cu", "recordings");
|
||||
const out = fs.existsSync(dir) ? fs.readdirSync(dir).filter((f) => /\.(mp4|mov|jpeg|png)$/i.test(f)).map((f) => {
|
||||
const st = fs.statSync(path.join(dir, f));
|
||||
return { file: path.join(dir, f), bytes: st.size, modifiedAt: st.mtime.toISOString() };
|
||||
}).sort((a, b) => b.modifiedAt.localeCompare(a.modifiedAt)).slice(0, 50) : [];
|
||||
return { dir, recordings: out, running: recording ? [recording.id] : [] };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export default { create };
|
||||
@@ -0,0 +1,558 @@
|
||||
// Linux backend — X11 first (xdotool/wmctrl/scrot/xclip), Wayland where the
|
||||
// right tools exist (grim/wtype/ydotool/wf-recorder/wl-clipboard). The
|
||||
// accessibility tree comes from AT-SPI via python3+pyatspi when installed.
|
||||
// Everything probes at call time and fails closed with the missing tool named.
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import crypto from "node:crypto";
|
||||
import { spawn } from "node:child_process";
|
||||
import { run, runOk, ExecError, tryJson, have } from "../exec.mjs";
|
||||
|
||||
const XKEYS = {
|
||||
return: "Return", enter: "Return", tab: "Tab", escape: "Escape", esc: "Escape",
|
||||
space: "space", backspace: "BackSpace", delete: "Delete", home: "Home", end: "End",
|
||||
pageup: "Page_Up", pagedown: "Page_Down", left: "Left", right: "Right", up: "Up",
|
||||
down: "Down", capslock: "Caps_Lock", menu: "Menu", print: "Print",
|
||||
};
|
||||
|
||||
function spawnDetached(cmd, args, stdinText = "", quiet = true) {
|
||||
const child = spawn(cmd, args, {
|
||||
stdio: stdinText ? ["pipe", "ignore", quiet ? "ignore" : "pipe"] : ["ignore", "ignore", quiet ? "ignore" : "pipe"],
|
||||
detached: true,
|
||||
});
|
||||
if (stdinText) child.stdin.end(stdinText);
|
||||
child.unref();
|
||||
return child;
|
||||
}
|
||||
|
||||
export function create({ exec }) {
|
||||
const tools = {};
|
||||
let session = null; // "x11" | "wayland"
|
||||
let probed = false;
|
||||
let lastRaster = null;
|
||||
let recording = null; // {id, pid, file, startedAt, mode}
|
||||
|
||||
async function probeSession() {
|
||||
if (probed) return session;
|
||||
probed = true;
|
||||
const wayland = !!(process.env.WAYLAND_DISPLAY || process.env.XDG_SESSION_TYPE === "wayland");
|
||||
const x11 = !!(process.env.DISPLAY || process.env.XDG_SESSION_TYPE === "x11");
|
||||
session = wayland && !x11 ? "wayland" : x11 ? "x11" : null;
|
||||
for (const t of ["xdotool", "wmctrl", "scrot", "import", "grim", "slurp", "wtype", "ydotool", "wf-recorder", "ffmpeg", "xclip", "xsel", "wl-copy", "wl-paste", "python3", "xrandr", "swaymsg", "hyprctl"]) {
|
||||
tools[t] = await have(t);
|
||||
}
|
||||
tools.pyatspi = tools.python3 && (await run("python3", ["-c", "import pyatspi"], { timeoutMs: 10_000 })).code === 0;
|
||||
return session;
|
||||
}
|
||||
|
||||
function need(tool, purpose) {
|
||||
if (!tools[tool]) throw new ExecError(`linux backend needs "${tool}" for ${purpose} — install it and retry`);
|
||||
}
|
||||
|
||||
async function shotTool() {
|
||||
if (session === "wayland") { need("grim", "screenshots on Wayland"); return { cmd: "grim", base: [] }; }
|
||||
if (session === "x11") {
|
||||
if (tools.scrot) return { cmd: "scrot", base: ["-z"] };
|
||||
need("import", "screenshots on X11 (imagemagick)");
|
||||
return { cmd: "import", base: ["-window", "root"] };
|
||||
}
|
||||
throw new ExecError("no X11 ($DISPLAY) or Wayland ($WAYLAND_DISPLAY) session visible to this process");
|
||||
}
|
||||
|
||||
function recordingsDir() {
|
||||
return process.env.CODEWHALE_CU_RECORDINGS_DIR || path.join(os.homedir(), ".codewhale-cu", "recordings");
|
||||
}
|
||||
|
||||
async function xdotool(args, opts = {}) {
|
||||
need("xdotool", "input on X11");
|
||||
const r = await run("xdotool", args, opts);
|
||||
if (r.code !== 0) throw new ExecError(`xdotool ${args[0]} exited ${r.code}: ${r.stderr.trim().slice(0, 200)}`, r);
|
||||
return r.stdout.trim();
|
||||
}
|
||||
|
||||
async function ydotool(args, opts = {}) {
|
||||
need("ydotool", "input on Wayland (ydotool needs its daemon running: sudo ydotoold)");
|
||||
const r = await run("ydotool", args, opts);
|
||||
if (r.code !== 0) throw new ExecError(`ydotool exited ${r.code}: ${r.stderr.trim().slice(0, 200)}`, r);
|
||||
return r.stdout.trim();
|
||||
}
|
||||
|
||||
function xdotoolKey(text) {
|
||||
return String(text).split("+").map((p) => {
|
||||
const k = p.trim().toLowerCase();
|
||||
if (XKEYS[k]) return XKEYS[k];
|
||||
if (/^f\d{1,2}$/.test(k)) return k.toUpperCase();
|
||||
return p.trim(); // pass through names already in xdotool form
|
||||
}).join("+");
|
||||
}
|
||||
|
||||
function assertNum(v, name) {
|
||||
const n = Number(v);
|
||||
if (!Number.isFinite(n)) throw new ExecError(`${name} must be a finite number`);
|
||||
return n;
|
||||
}
|
||||
|
||||
// ---------- AT-SPI tree ----------
|
||||
const PYATSPI_WALK = `import json, sys, pyatspi
|
||||
app_name = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
depth_max = int(sys.argv[2]) if len(sys.argv) > 2 else 8
|
||||
max_el = int(sys.argv[3]) if len(sys.argv) > 3 else 400
|
||||
desktop = pyatspi.Registry.getDesktop(0)
|
||||
root = None
|
||||
if app_name:
|
||||
for i in range(desktop.childCount):
|
||||
app = desktop.getChildAtIndex(i)
|
||||
if app and app_name.lower() in (app.name or "").lower():
|
||||
root = app
|
||||
break
|
||||
else:
|
||||
for i in range(desktop.childCount):
|
||||
a = desktop.getChildAtIndex(i)
|
||||
if a and a.childCount:
|
||||
root = a
|
||||
break
|
||||
if root is None:
|
||||
print(json.dumps({"found": False}))
|
||||
sys.exit(0)
|
||||
els = []
|
||||
truncated = False
|
||||
def info(e, path):
|
||||
ext = None
|
||||
try: ext = e.getExtents(pyatspi.DESKTOP_COORDS)
|
||||
except Exception: pass
|
||||
txt = None
|
||||
try:
|
||||
q = e.queryText()
|
||||
n = min(120, q.characterCount)
|
||||
if n > 0: txt = q.getText(0, n)
|
||||
except Exception: pass
|
||||
acts = []
|
||||
try:
|
||||
a = e.queryAction()
|
||||
acts = [a.getName(i) for i in range(a.nActions)]
|
||||
except Exception: pass
|
||||
els.append({"index": len(els), "path": path, "role": e.getRoleName() if e.getRoleName() else None,
|
||||
"label": e.name or None, "value": txt,
|
||||
"position": {"x": ext.x, "y": ext.y} if ext else None,
|
||||
"size": {"w": ext.width, "h": ext.height} if ext else None,
|
||||
"actions": acts})
|
||||
def walk(e, path, d):
|
||||
global truncated
|
||||
if len(els) >= max_el or d > depth_max:
|
||||
truncated = True
|
||||
return
|
||||
try: info(e, path)
|
||||
except Exception: return
|
||||
for i in range(e.childCount):
|
||||
try: c = e.getChildAtIndex(i)
|
||||
except Exception: continue
|
||||
if c: walk(c, path + [i], d + 1)
|
||||
walk(root, [], 0)
|
||||
print(json.dumps({"found": True, "name": root.name, "elements": els, "truncated": truncated}))`;
|
||||
|
||||
async function atspiResolve(pathArr, pythonBody, extraArg = null) {
|
||||
need("python3", "semantic element actions (AT-SPI)");
|
||||
const script = `import json, sys, pyatspi
|
||||
desktop = pyatspi.Registry.getDesktop(0)
|
||||
target_path = json.loads(sys.argv[1])
|
||||
extra = sys.argv[2] if len(sys.argv) > 2 else None
|
||||
node = None
|
||||
for i in range(desktop.childCount):
|
||||
a = desktop.getChildAtIndex(i)
|
||||
if a and a.childCount:
|
||||
node = a
|
||||
break
|
||||
if node is None:
|
||||
print(json.dumps({"ok": False, "code": "app_not_found"}))
|
||||
sys.exit(0)
|
||||
found = None
|
||||
stack = [(node, [])]
|
||||
while stack:
|
||||
n, p = stack.pop(0)
|
||||
if p == target_path:
|
||||
found = n
|
||||
break
|
||||
if len(p) > 12: continue
|
||||
try:
|
||||
for i in range(n.childCount):
|
||||
c = n.getChildAtIndex(i)
|
||||
if c: stack.append((c, p + [i]))
|
||||
except Exception: pass
|
||||
if found is None:
|
||||
print(json.dumps({"ok": False, "code": "element_stale"}))
|
||||
sys.exit(0)
|
||||
try:
|
||||
${pythonBody}
|
||||
except Exception as e:
|
||||
print(json.dumps({"ok": False, "code": str(e)}))`;
|
||||
const argv = ["-c", script, JSON.stringify(pathArr ?? [])];
|
||||
if (extraArg != null) argv.push(String(extraArg));
|
||||
const r = await run("python3", argv, { timeoutMs: 30_000 });
|
||||
const out = tryJson((r.stdout.trim().split("\n").pop() ?? ""), null);
|
||||
if (!out) throw new ExecError(`AT-SPI action failed: ${(r.stderr || r.stdout).slice(0, 250)}`, r);
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------- input helpers ----------
|
||||
function clickButton(button, clicks) {
|
||||
if (session === "x11") {
|
||||
const args = ["click"];
|
||||
if (clicks > 1) args.push("--repeat", String(clicks), "--delay", "80");
|
||||
args.push(String(button));
|
||||
return xdotool(args);
|
||||
}
|
||||
// ydotool click mask: down|up|count nibble (0xC0 = left click once, +1 per extra click;
|
||||
// 0x04 bit selects right button, 0x02 middle).
|
||||
const count = Math.max(1, Math.min(3, clicks));
|
||||
const code = button === 3 ? 0xc0 + count + 0x04 : button === 2 ? 0xc0 + count + 0x02 : 0xc0 + count - 1;
|
||||
return ydotool(["click", "0x" + code.toString(16)]);
|
||||
}
|
||||
|
||||
return {
|
||||
platform: "linux",
|
||||
probe: async () => {
|
||||
const s = await probeSession();
|
||||
const caps = {
|
||||
screenshot: !!((session === "wayland" && tools.grim) || (session === "x11" && (tools.scrot || tools.import))),
|
||||
clipboard: !!(tools.xclip || tools.xsel || (tools["wl-copy"] && tools["wl-paste"])),
|
||||
recording: !!(tools.ffmpeg || tools["wf-recorder"]),
|
||||
accessibility_tree: tools.pyatspi,
|
||||
};
|
||||
const missing = [];
|
||||
if (session === "x11" && !tools.xdotool) missing.push("xdotool (input)");
|
||||
if (session === "wayland" && !tools.ydotool) missing.push("ydotool+ydotoold (mouse input)");
|
||||
if (session === "wayland" && !tools.grim) missing.push("grim (screenshots)");
|
||||
if (session === "x11" && !tools.scrot && !tools.import) missing.push("scrot or imagemagick (screenshots)");
|
||||
if (!caps.recording) missing.push("ffmpeg (X11) or wf-recorder (Wayland)");
|
||||
if (!tools.pyatspi) missing.push("python3-pyatspi (accessibility tree)");
|
||||
return { platform: "linux", session: s, capabilities: caps, missing, note: "Every capability probes at call time and fails closed naming the missing tool." };
|
||||
},
|
||||
list_displays: async () => {
|
||||
await probeSession();
|
||||
if (session === "x11" && tools.xrandr) {
|
||||
const r = await runOk("xrandr", ["--query"], { timeoutMs: 15_000 });
|
||||
const displays = [];
|
||||
let i = 1;
|
||||
for (const m of r.stdout.matchAll(/^(\S+) connected (?:primary )?(\d+)x(\d+)\+(\d+)\+(\d+)/gm)) {
|
||||
displays.push({ index: i++, name: m[1], points: { x: Number(m[4]), y: Number(m[5]), w: Number(m[2]), h: Number(m[3]) }, pixels: { w: Number(m[2]), h: Number(m[3]) }, scale: 1, main: /primary/.test(m[0]) || i === 1 });
|
||||
}
|
||||
if (displays.length) return displays;
|
||||
}
|
||||
if (session === "wayland" && tools.swaymsg) {
|
||||
const r = await run("swaymsg", ["-t", "get_outputs", "-r"], { timeoutMs: 15_000 });
|
||||
const outs = tryJson(r.stdout, []);
|
||||
if (Array.isArray(outs) && outs.length) {
|
||||
return outs.map((o, i) => ({ index: i + 1, name: o.name, points: { x: o.rect?.x, y: o.rect?.y, w: o.rect?.width, h: o.rect?.height }, pixels: { w: o.current_mode?.width, h: o.current_mode?.height }, scale: o.scale ?? 1, main: i === 0 }));
|
||||
}
|
||||
}
|
||||
if (session === "wayland" && tools.hyprctl) {
|
||||
const r = await run("hyprctl", ["-j", "monitors"], { timeoutMs: 15_000 });
|
||||
const ms = tryJson(r.stdout, []);
|
||||
if (Array.isArray(ms) && ms.length) {
|
||||
return ms.map((o, i) => ({ index: i + 1, name: o.name, points: { x: o.x, y: o.y, w: o.width, h: o.height }, pixels: { w: o.width, h: o.height }, scale: o.scale ?? 1, main: !!o.main || i === 0 }));
|
||||
}
|
||||
}
|
||||
throw new ExecError("display enumeration needs xrandr (X11) or swaymsg/hyprctl (Wayland) — install one and retry");
|
||||
},
|
||||
switch_display: async ({ index }) => ({ activeDisplay: index ?? 1, note: "linux screenshots grab the compositor's virtual screen; per-display selection applies only where the shot tool supports it" }),
|
||||
list_apps: async () => {
|
||||
await probeSession();
|
||||
if (session === "x11" && tools.wmctrl) {
|
||||
const r = await runOk("wmctrl", ["-lx"], { timeoutMs: 15_000 });
|
||||
const seen = new Map();
|
||||
for (const line of r.stdout.split("\n")) {
|
||||
const parts = line.split(/\s+/);
|
||||
const wmClass = parts[2];
|
||||
if (wmClass) seen.set(wmClass, { name: wmClass.split(".")[0], wm_class: wmClass });
|
||||
}
|
||||
return { apps: [...seen.values()] };
|
||||
}
|
||||
if (session === "wayland" && (tools.swaymsg || tools.hyprctl)) {
|
||||
const w = await this.list_windows();
|
||||
const seen = new Map();
|
||||
for (const win of w.windows) {
|
||||
const cls = win.wm_class || win.app_id;
|
||||
if (cls) seen.set(cls, { name: cls, wm_class: cls });
|
||||
}
|
||||
return { apps: [...seen.values()] };
|
||||
}
|
||||
throw new ExecError("list_apps needs wmctrl (X11) or swaymsg/hyprctl (Wayland)");
|
||||
},
|
||||
list_windows: async () => {
|
||||
await probeSession();
|
||||
if (session === "x11" && tools.wmctrl) {
|
||||
const r = await runOk("wmctrl", ["-lGx"], { timeoutMs: 15_000 });
|
||||
const windows = [];
|
||||
for (const line of r.stdout.split("\n")) {
|
||||
const m = /^(\S+)\s+(-?\d+)\s+(-?\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(.*)$/.exec(line);
|
||||
if (m) windows.push({ id: m[1], desktop: m[2], position: { x: Number(m[3]), y: Number(m[4]) }, size: { w: Number(m[5]), h: Number(m[6]) }, wm_class: m[7], title: m[8] });
|
||||
}
|
||||
return { windows };
|
||||
}
|
||||
if (session === "wayland" && tools.swaymsg) {
|
||||
const r = await run("swaymsg", ["-t", "get_tree", "-r"], { timeoutMs: 15_000 });
|
||||
const windows = [];
|
||||
const walk = (n) => {
|
||||
if (n.type === "con" && n.name) windows.push({ id: String(n.id), title: n.name, wm_class: n.app_id ?? null, position: { x: n.rect?.x, y: n.rect?.y }, size: { w: n.rect?.width, h: n.rect?.height }, focused: !!n.focused });
|
||||
(n.nodes ?? []).forEach(walk);
|
||||
(n.floating_nodes ?? []).forEach(walk);
|
||||
};
|
||||
walk(tryJson(r.stdout, {}));
|
||||
return { windows };
|
||||
}
|
||||
if (session === "wayland" && tools.hyprctl) {
|
||||
const r = await run("hyprctl", ["-j", "clients"], { timeoutMs: 15_000 });
|
||||
const clients = tryJson(r.stdout, []);
|
||||
return { windows: clients.map((c) => ({ id: String(c.address), title: c.title, wm_class: c.class, position: { x: c.at?.[0], y: c.at?.[1] }, size: { w: c.size?.[0], h: c.size?.[1] }, focused: !!c.focused })) };
|
||||
}
|
||||
throw new ExecError("list_windows needs wmctrl (X11), swaymsg (sway) or hyprctl (hyprland)");
|
||||
},
|
||||
open_application: async ({ name, bundle_id: bid, url: urlArg } = {}) => {
|
||||
const target = name ?? bid;
|
||||
if (!target || !/^[A-Za-z0-9][A-Za-z0-9 ._-]*$/.test(target)) throw new ExecError("open_application needs a plain executable/desktop name");
|
||||
spawnDetached(target, urlArg ? [urlArg] : [], "", true);
|
||||
await new Promise((r) => setTimeout(r, 500));
|
||||
return { launched: true, name: target, url: urlArg ?? null };
|
||||
},
|
||||
get_app_state: async ({ app_ref } = {}) => {
|
||||
const t = await run("python3", ["-c", PYATSPI_WALK, app_ref?.name ?? "", "10", "500"], { timeoutMs: 45_000 }).then((r) =>
|
||||
tryJson((r.stdout.trim().split("\n").pop() ?? ""), null));
|
||||
if (!t) throw new ExecError("AT-SPI walk failed — is python3-pyatspi installed and the desktop running an accessibility bus (AT_SPI_BUS)?");
|
||||
if (!t.found) throw new ExecError("application not found in the AT-SPI tree — pass app_ref.name from list_apps");
|
||||
return t;
|
||||
},
|
||||
screenshot: async ({ display, region, path: outPath } = {}) => {
|
||||
await probeSession();
|
||||
const { cmd, base } = await shotTool();
|
||||
const dir = recordingsDir();
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
const file = outPath || path.join(dir, `shot-${new Date().toISOString().replace(/[:.]/g, "-")}-${crypto.randomBytes(3).toString("hex")}.png`);
|
||||
let args = [...base];
|
||||
if (cmd === "grim") {
|
||||
if (region) args.push("-g", `${Math.round(region[0])},${Math.round(region[1])} ${Math.round(region[2])}x${Math.round(region[3])}`);
|
||||
args.push(file);
|
||||
} else if (cmd === "scrot") {
|
||||
if (region) args.push("-a", `${Math.round(region[0])},${Math.round(region[1])},${Math.round(region[2])},${Math.round(region[3])}`);
|
||||
args.push(file);
|
||||
} else {
|
||||
if (region) args.push("-crop", `${Math.round(region[2])}x${Math.round(region[3])}+${Math.round(region[0])}+${Math.round(region[1])}`);
|
||||
args.push(file);
|
||||
}
|
||||
const r = await run(cmd, args, { timeoutMs: 20_000 });
|
||||
if (r.code !== 0) throw new ExecError(`${cmd} exited ${r.code}: ${r.stderr.trim().slice(0, 300)}`, r);
|
||||
lastRaster = { file, bytes: fs.statSync(file).size, capturedAt: new Date().toISOString() };
|
||||
return { ...lastRaster };
|
||||
},
|
||||
zoom: async ({ source, region, path: outPath }) => {
|
||||
need("ffmpeg", "zoom/crop");
|
||||
const src = source ?? lastRaster?.file;
|
||||
if (!src) throw new ExecError("no screenshot taken yet on this computer — call screenshot first");
|
||||
const out = outPath || path.join(recordingsDir(), `zoom-${crypto.randomBytes(4).toString("hex")}.png`);
|
||||
await runOk("ffmpeg", ["-y", "-loglevel", "error", "-i", src, "-vf", `crop=${Math.round(region[2])}:${Math.round(region[3])}:${Math.round(region[0])}:${Math.round(region[1])}`, out], { timeoutMs: 20_000 });
|
||||
return { file: out, bytes: fs.statSync(out).size, region, source: src };
|
||||
},
|
||||
left_click: ({ target }) => { assertNum(target.x, "x"); assertNum(target.y, "y"); return inputChain(target.x, target.y, () => clickButton(1, 1)); },
|
||||
double_click: ({ target }) => inputChain(target.x, target.y, () => clickButton(1, 2)),
|
||||
triple_click: ({ target }) => inputChain(target.x, target.y, () => clickButton(1, 3)),
|
||||
right_click: ({ target }) => inputChain(target.x, target.y, () => clickButton(3, 1)),
|
||||
middle_click: ({ target }) => inputChain(target.x, target.y, () => clickButton(2, 1)),
|
||||
mouse_move: ({ target }) => inputMove(target.x, target.y),
|
||||
left_click_drag: async ({ from_target: from, to }) => {
|
||||
await inputMove(from.x, from.y);
|
||||
if (session === "x11") await xdotool(["mousedown", "1"]);
|
||||
else await ydotool(["click", "0x40"]);
|
||||
for (let i = 1; i <= 10; i++) {
|
||||
await new Promise((r) => setTimeout(r, 20));
|
||||
await inputMove(from.x + ((to.x - from.x) * i) / 10, from.y + ((to.y - from.y) * i) / 10);
|
||||
}
|
||||
if (session === "x11") await xdotool(["mouseup", "1"]);
|
||||
else await ydotool(["click", "0x80"]);
|
||||
return { action_sent: true, from, to };
|
||||
},
|
||||
left_mouse_down: ({ target }) => session === "x11" ? xdotool(["mousedown", "1"]).then(() => ({ action_sent: true })) : ydotool(["click", "0x40"]).then(() => ({ action_sent: true })),
|
||||
left_mouse_up: () => session === "x11" ? xdotool(["mouseup", "1"]).then(() => ({ action_sent: true })) : ydotool(["click", "0x80"]).then(() => ({ action_sent: true })),
|
||||
scroll: async ({ target, direction = "down", amount = 3 }) => {
|
||||
await inputMove(target.x, target.y);
|
||||
if (session === "x11") {
|
||||
const buttons = { down: 5, up: 4, right: 7, left: 6 };
|
||||
await xdotool(["click", "--repeat", String(Math.max(1, Math.min(30, amount))), "--delay", "60", String(buttons[direction] ?? 5)]);
|
||||
return { action_sent: true, direction, amount };
|
||||
}
|
||||
// Wayland: synthesize wheel via ydotool is not wired in this build — honest refusal.
|
||||
throw new ExecError('scroll on Wayland is not available in this build; use swipe-style drags or run an X11/XWayland window. (Roadmap: ydotool wheel events.)');
|
||||
},
|
||||
type: async ({ text }) => {
|
||||
if (!text) return { action_sent: false, note: "empty text" };
|
||||
await probeSession();
|
||||
if (session === "x11") {
|
||||
await xdotool(["type", "--delay", "12", "--", String(text)]);
|
||||
return { action_sent: true, chars: text.length };
|
||||
}
|
||||
need("wtype", "typing on Wayland");
|
||||
const r = await run("wtype", ["--", String(text)], { timeoutMs: 15_000 });
|
||||
if (r.code !== 0) throw new ExecError(`wtype failed: ${r.stderr.slice(0, 200)}`, r);
|
||||
return { action_sent: true, chars: text.length };
|
||||
},
|
||||
key: async ({ text, repeat = 1 }) => {
|
||||
await probeSession();
|
||||
const k = xdotoolKey(text);
|
||||
if (session === "x11") {
|
||||
await xdotool(["key", "--repeat", String(Math.max(1, Math.min(100, repeat))), "--delay", "60", k]);
|
||||
return { action_sent: true, key: k };
|
||||
}
|
||||
need("wtype", "key presses on Wayland");
|
||||
await run("wtype", ["-P", k]);
|
||||
await run("wtype", ["-R", k]);
|
||||
return { action_sent: true, key: k };
|
||||
},
|
||||
hold_key: async ({ text, duration }) => {
|
||||
await probeSession();
|
||||
const k = xdotoolKey(text);
|
||||
const d = Math.max(0.05, Math.min(30, Number(duration) || 1));
|
||||
if (session === "x11") {
|
||||
await xdotool(["keydown", k]);
|
||||
await new Promise((r) => setTimeout(r, d * 1000));
|
||||
await xdotool(["keyup", k]);
|
||||
return { action_sent: true, key: k, heldSec: d };
|
||||
}
|
||||
need("ydotool", "key hold on Wayland");
|
||||
await ydotool(["key", `${k}:1`]);
|
||||
await new Promise((r) => setTimeout(r, d * 1000));
|
||||
await ydotool(["key", `${k}:0`]);
|
||||
return { action_sent: true, key: k, heldSec: d };
|
||||
},
|
||||
set_value: async ({ target, value }) => {
|
||||
const out = await atspiResolve(
|
||||
target.path,
|
||||
` v = found.queryValue()
|
||||
v.currentValue = float(extra)`,
|
||||
value,
|
||||
).catch(async (e) => {
|
||||
// Fall back to the Text interface for text-bearing widgets.
|
||||
const out2 = await atspiResolve(
|
||||
target.path,
|
||||
` t = found.queryText()
|
||||
t.setTextContents(extra)`,
|
||||
String(value),
|
||||
);
|
||||
if (!out2.ok) throw e;
|
||||
return out2;
|
||||
});
|
||||
if (!out.ok) throw new ExecError(`set_value failed: ${out.code}`);
|
||||
return { action_sent: true, strategy: "a11y" };
|
||||
},
|
||||
select_text: async () => { throw new ExecError("select_text is not implemented on the linux backend — fail-closed"); },
|
||||
perform_action: async ({ target, action }) => {
|
||||
const body = ` a = found.queryAction()
|
||||
names = [a.getName(i) for i in range(a.nActions)]
|
||||
want = (extra or "click").lower()
|
||||
match = next((n for n in names if n.lower() == want), None)
|
||||
if match is None and want == "click":
|
||||
match = next((n for n in names if n.lower() in ("click", "press", "activate")), None)
|
||||
if match is None:
|
||||
print(json.dumps({"ok": False, "code": "action_not_found: " + ",".join(names)}))
|
||||
else:
|
||||
a.doAction(names.index(match))
|
||||
print(json.dumps({"ok": True, "sent": True}))`;
|
||||
const out = await atspiResolve(target.path, body, String(action));
|
||||
if (!out.ok) throw new ExecError(`perform_action failed: ${out.code}`);
|
||||
return { action_sent: true, strategy: "a11y", action };
|
||||
},
|
||||
read_clipboard: async () => {
|
||||
await probeSession();
|
||||
const cmd = session === "x11"
|
||||
? (tools.xclip ? ["xclip", "-selection", "clipboard", "-o"] : ["xsel", "--clipboard", "--output"])
|
||||
: ["wl-paste"];
|
||||
need(cmd[0], "clipboard read");
|
||||
const r = await run(cmd[0], cmd.slice(1), { timeoutMs: 10_000 });
|
||||
if (r.code !== 0) throw new ExecError("clipboard read failed", r);
|
||||
return { text: r.stdout, encoding: "utf8" };
|
||||
},
|
||||
write_clipboard: async ({ text }) => {
|
||||
await probeSession();
|
||||
const cmd = session === "x11"
|
||||
? (tools.xclip ? ["xclip", "-selection", "clipboard"] : ["xsel", "--clipboard", "--input"])
|
||||
: ["wl-copy"];
|
||||
need(cmd[0], "clipboard write");
|
||||
spawnDetached(cmd[0], cmd.slice(1), String(text ?? ""), true);
|
||||
return { written: String(text ?? "").length };
|
||||
},
|
||||
cursor_position: async () => {
|
||||
await probeSession();
|
||||
if (session === "x11") {
|
||||
const out = await xdotool(["getmouselocation"]);
|
||||
const m = /(\d+)\s+(\d+)/.exec(out);
|
||||
return { x: Number(m[1]), y: Number(m[2]) };
|
||||
}
|
||||
throw new ExecError("cursor position needs an X11 session in this build");
|
||||
},
|
||||
recordingStart: async ({ fps = 15, region } = {}) => {
|
||||
await probeSession();
|
||||
const dir = recordingsDir();
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
const id = crypto.randomBytes(4).toString("hex");
|
||||
const file = path.join(dir, `rec-${id}.${session === "wayland" ? "mkv" : "mp4"}`);
|
||||
if (session === "x11") {
|
||||
need("ffmpeg", "recording on X11");
|
||||
const dpy = process.env.DISPLAY || ":0";
|
||||
const args = ["-y", "-loglevel", "error", "-f", "x11grab", "-framerate", String(fps)];
|
||||
if (region) args.push("-video_size", `${Math.round(region[2])}x${Math.round(region[3])}`);
|
||||
args.push("-i", `${dpy}${region ? `+${Math.round(region[0])},${Math.round(region[1])}` : ""}`, "-c:v", "libx264", "-pix_fmt", "yuv420p", file);
|
||||
const child = spawnDetached("ffmpeg", args);
|
||||
await new Promise((r) => setTimeout(r, 700));
|
||||
try { process.kill(child.pid, 0); } catch { throw new ExecError("ffmpeg x11grab exited immediately — check DISPLAY, XAUTHORITY and screen permissions"); }
|
||||
recording = { id, pid: child.pid, file, startedAt: new Date().toISOString(), mode: "x11grab" };
|
||||
return { id, pid: child.pid, file, mode: "x11grab", fps };
|
||||
}
|
||||
need("wf-recorder", "recording on Wayland");
|
||||
const args = ["-r", String(fps), "-f", file];
|
||||
if (process.env.CU_WAYLAND_OUTPUT) args.unshift("-o", process.env.CU_WAYLAND_OUTPUT);
|
||||
const child = spawnDetached("wf-recorder", args);
|
||||
await new Promise((r) => setTimeout(r, 700));
|
||||
try { process.kill(child.pid, 0); } catch { throw new ExecError("wf-recorder exited immediately — check compositor support (wlroots)"); }
|
||||
recording = { id, pid: child.pid, file, startedAt: new Date().toISOString(), mode: "wf-recorder" };
|
||||
return { id, pid: child.pid, file, mode: "wf-recorder", fps };
|
||||
},
|
||||
recordingStop: async ({ id }) => {
|
||||
if (!recording || recording.id !== id) throw new ExecError(`unknown recording "${id}"`);
|
||||
process.kill(recording.pid, "SIGINT");
|
||||
await new Promise((r) => setTimeout(r, 1500));
|
||||
const bytes = fs.existsSync(recording.file) ? fs.statSync(recording.file).size : 0;
|
||||
const out = { id, file: recording.file, bytes, mode: recording.mode, startedAt: recording.startedAt, stoppedAt: new Date().toISOString() };
|
||||
recording = null;
|
||||
return out;
|
||||
},
|
||||
recordingStatus: ({ id }) => {
|
||||
if (!recording || recording.id !== id) return { id, running: false };
|
||||
let alive = true;
|
||||
try { process.kill(recording.pid, 0); } catch { alive = false; }
|
||||
return { id, running: alive, file: recording.file, bytes: fs.existsSync(recording.file) ? fs.statSync(recording.file).size : 0, mode: recording.mode };
|
||||
},
|
||||
recordingList: async () => {
|
||||
const dir = recordingsDir();
|
||||
const out = fs.existsSync(dir)
|
||||
? fs.readdirSync(dir).filter((f) => /\.(mp4|mkv|png)$/i.test(f)).map((f) => {
|
||||
const st = fs.statSync(path.join(dir, f));
|
||||
return { file: path.join(dir, f), bytes: st.size, modifiedAt: st.mtime.toISOString() };
|
||||
}).sort((a, b) => b.modifiedAt.localeCompare(a.modifiedAt)).slice(0, 50)
|
||||
: [];
|
||||
return { dir, recordings: out, running: recording ? [recording.id] : [] };
|
||||
},
|
||||
};
|
||||
|
||||
async function inputChain(x, y, act) {
|
||||
await inputMove(x, y);
|
||||
await act();
|
||||
return { action_sent: true, at: { x: Number(x), y: Number(y) } };
|
||||
}
|
||||
|
||||
async function inputMove(x, y) {
|
||||
const nx = Math.round(assertNum(x, "x"));
|
||||
const ny = Math.round(assertNum(y, "y"));
|
||||
if (session === "x11") await xdotool(["mousemove", "--sync", String(nx), String(ny)]);
|
||||
else await ydotool(["moveto", String(nx), String(ny)]);
|
||||
}
|
||||
}
|
||||
|
||||
export default { create };
|
||||
@@ -0,0 +1,474 @@
|
||||
// Windows backend. Every operation is one PowerShell invocation whose script
|
||||
// travels as a base64 -EncodedCommand, so tool arguments never become shell
|
||||
// syntax. Screenshots + UIA accessibility come from .NET; raw pointer and
|
||||
// keyboard events come from user32 P/Invoke (SendInput/mouse_event).
|
||||
// Recording uses ffmpeg gdigrab when ffmpeg is on PATH.
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import crypto from "node:crypto";
|
||||
import { spawn } from "node:child_process";
|
||||
import { run, runOk, ExecError, tryJson } from "../exec.mjs";
|
||||
|
||||
function ps(script, opts = {}) {
|
||||
const encoded = Buffer.from(script, "utf16le").toString("base64");
|
||||
return run("powershell.exe", ["-NoProfile", "-NonInteractive", "-EncodedCommand", encoded], {
|
||||
timeoutMs: opts.timeoutMs ?? 25_000,
|
||||
maxBuffer: 32 * 1024 * 1024,
|
||||
});
|
||||
}
|
||||
|
||||
async function psJson(script, opts = {}) {
|
||||
const r = await ps(script, opts);
|
||||
const out = r.stdout.trim();
|
||||
const j = tryJson(out, null);
|
||||
if (!j) throw new ExecError(`powershell did not return JSON: ${(r.stderr || out).trim().slice(0, 300)}`, r);
|
||||
return j;
|
||||
}
|
||||
|
||||
const USER32 = `
|
||||
using System;
|
||||
using System.Runtime.InteropServices;
|
||||
public static class User32 {
|
||||
[DllImport("user32.dll")] public static extern bool SetCursorPos(int X, int Y);
|
||||
[DllImport("user32.dll")] public static extern void mouse_event(uint dwFlags, int dx, int dy, uint dwData, UIntPtr dwExtraInfo);
|
||||
[DllImport("user32.dll")] public static extern uint SendInput(uint nInputs, INPUT[] pInputs, int cbSize);
|
||||
[DllImport("user32.dll")] public static extern bool GetCursorPos(out POINT lpPoint);
|
||||
[DllImport("user32.dll")] public static extern bool SetForegroundWindow(IntPtr hWnd);
|
||||
[StructLayout(LayoutKind.Sequential)] public struct POINT { public int X; public int Y; }
|
||||
[StructLayout(LayoutKind.Sequential)] public struct RECT { public int Left; public int Top; public int Right; public int Bottom; }
|
||||
[StructLayout(LayoutKind.Sequential)] public struct MOUSEINPUT { public int dx; public int dy; public uint mouseData; public uint dwFlags; public uint time; public IntPtr dwExtraInfo; }
|
||||
[StructLayout(LayoutKind.Sequential)] public struct KEYBDINPUT { public ushort wVk; public ushort wScan; public uint dwFlags; public uint time; public IntPtr dwExtraInfo; }
|
||||
[StructLayout(LayoutKind.Explicit)] public struct INPUTUNION { [FieldOffset(0)] public MOUSEINPUT mi; [FieldOffset(0)] public KEYBDINPUT ki; }
|
||||
[StructLayout(LayoutKind.Sequential)] public struct INPUT { public uint type; public INPUTUNION u; }
|
||||
public const uint MOVED = 0x0001, LEFTDOWN = 0x0002, LEFTUP = 0x0004, RIGHTDOWN = 0x0008, RIGHTUP = 0x0010,
|
||||
MIDDLEDOWN = 0x0020, MIDDLEUP = 0x0040, WHEEL = 0x0800, HWHEEL = 0x1000, ABSOLUTE = 0x8000, VIRTUALKEY = 0x4000, KEYUP = 0x0002, UNICODE = 0x0004;
|
||||
public static INPUT MouseInput(uint flags, int x, int y, uint data) {
|
||||
return new INPUT { type = 0, u = new INPUTUNION { mi = new MOUSEINPUT { dx = x, dy = y, mouseData = data, dwFlags = flags, time = 0, dwExtraInfo = IntPtr.Zero } } };
|
||||
}
|
||||
public static INPUT KeyInput(ushort vk, ushort scan, uint flags) {
|
||||
return new INPUT { type = 1, u = new INPUTUNION { ki = new KEYBDINPUT { wVk = vk, wScan = scan, dwFlags = flags, time = 0, dwExtraInfo = IntPtr.Zero } } };
|
||||
}
|
||||
}`;
|
||||
|
||||
function recordingsDir() {
|
||||
return process.env.CODEWHALE_CU_RECORDINGS_DIR || path.join(os.homedir(), ".codewhale-cu", "recordings");
|
||||
}
|
||||
|
||||
// Windows virtual-key codes for named keys.
|
||||
const VK = {
|
||||
return: 0x0d, enter: 0x0d, tab: 0x09, escape: 0x1b, esc: 0x1b, space: 0x20,
|
||||
backspace: 0x08, delete: 0x2e, home: 0x24, end: 0x23, pageup: 0x21, pagedown: 0x22,
|
||||
left: 0x25, up: 0x26, right: 0x27, down: 0x28, capslock: 0x14, insert: 0x2d,
|
||||
f1: 0x70, f2: 0x71, f3: 0x72, f4: 0x73, f5: 0x74, f6: 0x75, f7: 0x76, f8: 0x77,
|
||||
f9: 0x78, f10: 0x79, f11: 0x7a, f12: 0x7b,
|
||||
};
|
||||
const MODVK = { ctrl: 0x11, control: 0x11, alt: 0x12, shift: 0x10, win: 0x5b, meta: 0x5b, cmd: 0x5b };
|
||||
|
||||
export function create() {
|
||||
const bootstrapped = (async () => {
|
||||
await ps(`Add-Type -TypeDefinition @'\n${USER32}\n'@ -ErrorAction SilentlyContinue; [User32] | Out-Null`, { timeoutMs: 30_000 });
|
||||
})();
|
||||
let lastRaster = null;
|
||||
let recording = null; // {id, pid, file, startedAt, mode}
|
||||
|
||||
async function withUser32(script, opts) {
|
||||
await bootstrapped;
|
||||
return ps(script, opts);
|
||||
}
|
||||
|
||||
return {
|
||||
platform: "win32",
|
||||
probe: async () => {
|
||||
await bootstrapped.catch(() => {});
|
||||
let ffmpeg = true;
|
||||
try { await runOk("ffmpeg", ["-version"], { timeoutMs: 10_000 }); } catch { ffmpeg = false; }
|
||||
const psOk = await ps("Write-Output 'ok'").then((r) => r.code === 0).catch(() => false);
|
||||
return {
|
||||
platform: "win32",
|
||||
powershell: psOk,
|
||||
capabilities: { screenshot: psOk, accessibility_tree: psOk, clipboard: psOk, recording: ffmpeg, raw_input: psOk },
|
||||
note: "Recording needs ffmpeg (gdigrab) on PATH. UIA accessibility works without extra installs.",
|
||||
};
|
||||
},
|
||||
list_displays: async () => {
|
||||
await bootstrapped;
|
||||
const d = await psJson(`Add-Type -AssemblyName System.Windows.Forms;
|
||||
$out = [System.Windows.Forms.Screen]::AllScreens | ForEach-Object { [pscustomobject]@{ name = $_.DeviceName; primary = $_.Primary; x = $_.Bounds.X; y = $_.Bounds.Y; w = $_.Bounds.Width; h = $_.Bounds.Height; } } | ConvertTo-Json -Compress;
|
||||
Write-Output ('{"displays": ' + ($out -replace '^\\[','[' -replace '\\]$/',']') + '}');`, { timeoutMs: 15_000 }).catch(async () => {
|
||||
// Fallback: build the array safely.
|
||||
return psJson(`Add-Type -AssemblyName System.Windows.Forms;
|
||||
$arr = @(); foreach ($s in [System.Windows.Forms.Screen]::AllScreens) { $arr += [pscustomobject]@{ name = $s.DeviceName; primary = $s.Primary; x = $s.Bounds.X; y = $s.Bounds.Y; w = $s.Bounds.Width; h = $s.Bounds.Height } }
|
||||
Write-Output ('{"displays": ' + (ConvertTo-Json $arr -Compress) + '}');`, { timeoutMs: 15_000 });
|
||||
});
|
||||
return d.displays.map((x, i) => ({ index: i + 1, name: x.name, points: { x: x.x, y: x.y, w: x.w, h: x.h }, pixels: { w: x.w, h: x.h }, scale: 1, main: !!x.primary }));
|
||||
},
|
||||
async switch_display({ index }) { return { activeDisplay: index ?? 1, note: "windows screenshots grab the virtual screen; per-display crop applies where supported" }; },
|
||||
list_apps: async () => {
|
||||
await bootstrapped;
|
||||
const j = await psJson(`Add-Type -AssemblyName System.Windows.Forms;
|
||||
$out = Get-Process | Where-Object { $_.MainWindowTitle } | ForEach-Object { [pscustomobject]@{ name = $_.ProcessName; pid2 = $_.Id; title = $_.MainWindowTitle } } | ConvertTo-Json -Compress;
|
||||
if (-not $out) { $out = '[]' }
|
||||
Write-Output ('{"apps": ' + $out + '}');`);
|
||||
return { apps: (Array.isArray(j.apps) ? j.apps : [j.apps]).map((a) => ({ name: a.name, pid: a.pid2, title: a.title })) };
|
||||
},
|
||||
list_windows: async () => {
|
||||
await withUser32(`Add-Type -AssemblyName System.Windows.Forms;
|
||||
Add-Type -TypeDefinition @'
|
||||
using System;
|
||||
using System.Text;
|
||||
using System.Collections.Generic;
|
||||
using System.Runtime.InteropServices;
|
||||
public static class WinEnum {
|
||||
[DllImport("user32.dll")] static extern bool EnumWindows(EnumWindowsProc lpEnumFunc, IntPtr lParam);
|
||||
[DllImport("user32.dll")] static extern bool IsWindowVisible(IntPtr hWnd);
|
||||
[DllImport("user32.dll")] static extern int GetWindowText(IntPtr hWnd, StringBuilder text, int count);
|
||||
[DllImport("user32.dll")] static extern int GetWindowTextLength(IntPtr hWnd);
|
||||
[DllImport("user32.dll")] static extern uint GetWindowThreadProcessId(IntPtr hWnd, out uint pid);
|
||||
[DllImport("user32.dll")] static extern bool GetWindowRect(IntPtr hWnd, out RECT rect);
|
||||
[StructLayout(LayoutKind.Sequential)] public struct RECT { public int Left; public int Top; public int Right; public int Bottom; }
|
||||
delegate bool EnumWindowsProc(IntPtr hWnd, IntPtr lParam);
|
||||
public static List<string> List() {
|
||||
var result = new List<string>();
|
||||
EnumWindows((h, l) => {
|
||||
if (!IsWindowVisible(h)) return true;
|
||||
int len = GetWindowTextLength(h);
|
||||
var sb = new StringBuilder(len + 1);
|
||||
GetWindowText(h, sb, sb.Capacity);
|
||||
uint pid; GetWindowThreadProcessId(h, out pid);
|
||||
RECT r; GetWindowRect(h, out r);
|
||||
if (sb.Length > 0 && r.Right > r.Left)
|
||||
result.Add(pid + "|" + r.Left + "," + r.Top + "," + (r.Right - r.Left) + "," + (r.Bottom - r.Top) + "|" + sb.ToString());
|
||||
return true;
|
||||
}, IntPtr.Zero);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
'@;
|
||||
$json = (WinEnum::List() | ForEach-Object { $p = $_.Split('|', 2); $parts = $p[1].Split('|', 2); [pscustomobject]@{ pid2 = [int]$p[0]; geom = $parts[0]; title = $parts[1] } } | ConvertTo-Json -Compress;
|
||||
if (-not $json) { $json = '[]' }
|
||||
Write-Output ('{"windows": ' + $json + '}');`, { timeoutMs: 25_000 }).then((j) => ({
|
||||
windows: (Array.isArray(j.windows) ? j.windows : [j.windows]).map((w) => {
|
||||
const g = String(w.geom).split(",").map(Number);
|
||||
return { pid: w.pid2, title: w.title, position: { x: g[0], y: g[1] }, size: { w: g[2], h: g[3] } };
|
||||
}),
|
||||
})).catch((e) => { throw e; });
|
||||
},
|
||||
open_application: async ({ name, bundle_id: bid, url: urlArg, activate } = {}) => {
|
||||
const target = name ?? bid;
|
||||
if (!target || !/^[A-Za-z0-9][A-Za-z0-9 .:_-]*$/.test(target)) throw new ExecError("open_application needs a plain app or executable name");
|
||||
const r = await ps(`Start-Process -FilePath "${target.replace(/"/g, "")}"${urlArg ? ` -ArgumentList "${urlArg.replace(/"/g, "")}"` : ""}; Write-Output '{"launched": true}'`, { timeoutMs: 20_000 });
|
||||
if (r.code !== 0) throw new ExecError(`Start-Process failed: ${r.stderr.trim().slice(0, 200)}`, r);
|
||||
return { launched: true, name: target, url: urlArg ?? null, activate };
|
||||
},
|
||||
get_app_state: async ({ app_ref, detail } = {}) => {
|
||||
await bootstrapped;
|
||||
const filter = app_ref?.name ? app_ref.name.replace(/'/g, "''") : "";
|
||||
const maxEls = detail === "full" ? 800 : 400;
|
||||
const j = await psJson(`Add-Type -AssemblyName UIAutomationClient;
|
||||
Add-Type -AssemblyName UIAutomationTypes;
|
||||
$max = ${maxEls};
|
||||
$root = [System.Windows.Automation.AutomationElement]::RootElement;
|
||||
$els = New-Object System.Collections.ArrayList;
|
||||
$found = $false; $truncated = $false; $appName = $null;
|
||||
$targets = $root.FindAll([System.Windows.Automation.TreeScope]::Children, [System.Windows.Automation.Condition]::TrueCondition);
|
||||
foreach ($t in $targets) {
|
||||
$nm = $t.Current.Name;
|
||||
if ('${filter}' -and $nm -notlike '*${filter}*') { continue }
|
||||
$found = $true; $appName = $nm;
|
||||
$stack = New-Object System.Collections.Stack;
|
||||
$stack.Push(@($t, @(0)));
|
||||
while ($stack.Count -gt 0) {
|
||||
$entry = $stack.Pop(); $cur = $entry[0]; $path = $entry[1];
|
||||
if ($els.Count -ge $max) { $truncated = $true; break }
|
||||
$rect = $cur.Current.BoundingRectangle;
|
||||
$acts = @();
|
||||
try { $acts = @($cur.GetSupportedPatterns() | ForEach-Object { $_.ProgrammaticName -replace 'Pattern$','' }) } catch {}
|
||||
[void]$els.Add([pscustomobject]@{ index = $els.Count; path = $path; role = [string]$cur.Current.ControlType.ProgrammaticName; label = [string]$cur.Current.Name; value = $([string]$cur.Current.Value).Substring(0, [Math]::Min(120, [string]$cur.Current.Value).Length); enabled = $cur.Current.IsEnabled;
|
||||
x = [int]$rect.X; y = [int]$rect.Y; w = [int]$rect.Width; h = [int]$rect.Height; actions = $acts });
|
||||
$kids = $cur.FindAll([System.Windows.Automation.TreeScope]::Children, [System.Windows.Automation.Condition]::TrueCondition);
|
||||
for ($i = $kids.Count - 1; $i -ge 0; $i--) { $stack.Push(@($kids[$i], ($path + $i))) }
|
||||
}
|
||||
break;
|
||||
}
|
||||
$result = [pscustomobject]@{ found = $found; name = $appName; truncated = $truncated; elements = @($els | ForEach-Object { [pscustomobject]@{ index = $_.index; path = $_.path; role = ($_.role -replace 'ControlType.',''); label = $_.label; value = $_.value; enabled = $_.enabled; position = [pscustomobject]@{ x = $_.x; y = $_.y }; size = [pscustomobject]@{ w = $_.w; h = $_.h }; actions = $_.actions } }) };
|
||||
Write-Output ($result | ConvertTo-Json -Depth 6 -Compress);`, { timeoutMs: 60_000 });
|
||||
if (!j.found) throw new ExecError("application window not found in UIA tree — pass app_ref.name from list_apps");
|
||||
return j;
|
||||
},
|
||||
screenshot: async ({ display, region, path: outPath } = {}) => {
|
||||
await bootstrapped;
|
||||
const dir = recordingsDir();
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
const file = outPath || path.join(dir, `shot-${new Date().toISOString().replace(/[:.]/g, "-")}-${crypto.randomBytes(3).toString("hex")}.png`);
|
||||
const winPath = file.replace(/\\/g, "\\\\");
|
||||
const script = `Add-Type -AssemblyName System.Windows.Forms; Add-Type -AssemblyName System.Drawing;
|
||||
$bounds = [System.Windows.Forms.SystemInformation]::VirtualScreen;
|
||||
$bmp = New-Object System.Drawing.Bitmap($bounds.Width, $bounds.Height);
|
||||
$g = [System.Drawing.Graphics]::FromImage($bmp);
|
||||
$g.CopyFromScreen($bounds.X, $bounds.Y, 0, 0, $bounds.Size);
|
||||
$g.Dispose();
|
||||
$bmp.Save('${file.replace(/'/g, "''")}', [System.Drawing.Imaging.ImageFormat]::Png);
|
||||
$bmp.Dispose();
|
||||
Write-Output '{"ok": true, "w": ' + $bounds.Width + ', "h": ' + $bounds.Height + '}';`;
|
||||
const r = await ps(script, { timeoutMs: 30_000 });
|
||||
if (r.code !== 0 || !fs.existsSync(file)) throw new ExecError(`screenshot failed: ${(r.stderr || r.stdout).trim().slice(0, 300)}`, r);
|
||||
const meta = tryJson(r.stdout.trim().split("\n").pop(), {});
|
||||
lastRaster = { file, bytes: fs.statSync(file).size, points: { x: 0, y: 0, w: meta.w, h: meta.h }, pixels: { w: meta.w, h: meta.h }, scale: 1, capturedAt: new Date().toISOString() };
|
||||
return { ...lastRaster };
|
||||
},
|
||||
zoom: async ({ source, region, path: outPath }) => {
|
||||
const src = source ?? lastRaster?.file;
|
||||
if (!src) throw new ExecError("no screenshot taken yet on this computer — call screenshot first");
|
||||
const out = outPath || path.join(recordingsDir(), `zoom-${crypto.randomBytes(4).toString("hex")}.png`);
|
||||
const script = `Add-Type -AssemblyName System.Drawing;
|
||||
$img = [System.Drawing.Image]::FromFile('${src.replace(/'/g, "''")}');
|
||||
$rect = New-Object System.Drawing.Rectangle(${Math.round(region[0])}, ${Math.round(region[1])}, ${Math.round(region[2])}, ${Math.round(region[3])});
|
||||
$bmp = New-Object System.Drawing.Bitmap($rect.Width, $rect.Height);
|
||||
$g = [System.Drawing.Graphics]::FromImage($bmp);
|
||||
$g.DrawImage($img, (New-Object System.Drawing.Rectangle(0, 0, $rect.Width, $rect.Height)), $rect, [System.Drawing.GraphicsUnit]::Pixel);
|
||||
$g.Dispose();
|
||||
$bmp.Save('${out.replace(/'/g, "''")}', [System.Drawing.Imaging.ImageFormat]::Png);
|
||||
$bmp.Dispose(); $img.Dispose();
|
||||
Write-Output '{"ok": true}';`;
|
||||
const r = await ps(script, { timeoutMs: 20_000 });
|
||||
if (r.code !== 0 || !fs.existsSync(out)) throw new ExecError(`zoom failed: ${(r.stderr || "").slice(0, 250)}`, r);
|
||||
return { file: out, bytes: fs.statSync(out).size, region, source: src };
|
||||
},
|
||||
left_click: ({ target }) => clickAt(0, target.x, target.y, 1),
|
||||
double_click: ({ target }) => clickAt(0, target.x, target.y, 2),
|
||||
triple_click: ({ target }) => clickAt(0, target.x, target.y, 3),
|
||||
right_click: ({ target }) => clickAt(1, target.x, target.y, 1),
|
||||
middle_click: ({ target }) => clickAt(2, target.x, target.y, 1),
|
||||
mouse_move: async ({ target }) => {
|
||||
await withUser32(`[User32]::SetCursorPos(${Math.round(target.x)}, ${Math.round(target.y)}) | Out-Null; Write-Output '{"ok": true}'`);
|
||||
return { action_sent: true, at: { x: target.x, y: target.y } };
|
||||
},
|
||||
left_click_drag: async ({ from_target: from, to }) => {
|
||||
await withUser32(`[User32]::SetCursorPos(${Math.round(from.x)}, ${Math.round(from.y)}) | Out-Null;
|
||||
Start-Sleep -Milliseconds 80;
|
||||
[User32]::mouse_event([User32]::LEFTDOWN, 0, 0, 0, [UIntPtr]::Zero);
|
||||
Start-Sleep -Milliseconds 80;
|
||||
[User32]::SetCursorPos(${Math.round(to.x)}, ${Math.round(to.y)}) | Out-Null;
|
||||
Start-Sleep -Milliseconds 80;
|
||||
[User32]::mouse_event([User32]::LEFTUP, 0, 0, 0, [UIntPtr]::Zero);
|
||||
Write-Output '{"ok": true}';`, { timeoutMs: 20_000 });
|
||||
return { action_sent: true, from, to };
|
||||
},
|
||||
left_mouse_down: async ({ target }) => {
|
||||
await withUser32(target ? `[User32]::SetCursorPos(${Math.round(target.x)}, ${Math.round(target.y)}) | Out-Null;` : "" + `[User32]::mouse_event([User32]::LEFTDOWN, 0, 0, 0, [UIntPtr]::Zero); Write-Output '{"ok": true}'`);
|
||||
return { action_sent: true };
|
||||
},
|
||||
left_mouse_up: async () => {
|
||||
await withUser32(`[User32]::mouse_event([User32]::LEFTUP, 0, 0, 0, [UIntPtr]::Zero); Write-Output '{"ok": true}'`);
|
||||
return { action_sent: true };
|
||||
},
|
||||
scroll: async ({ target, direction = "down", amount = 3 }) => {
|
||||
const notches = Math.max(1, Math.min(30, amount));
|
||||
const data = (direction === "down" ? -1 : direction === "up" ? 1 : 0) * notches * 120;
|
||||
const hdata = (direction === "right" ? 1 : direction === "left" ? -1 : 0) * notches * 120;
|
||||
await withUser32(`[User32]::SetCursorPos(${Math.round(target.x)}, ${Math.round(target.y)}) | Out-Null;
|
||||
Start-Sleep -Milliseconds 60;
|
||||
[User32]::mouse_event([User32]::WHEEL, 0, 0, [uint32]"$([int64]${data} -band 0xFFFFFFFF)", [UIntPtr]::Zero);
|
||||
[User32]::mouse_event([User32]::HWHEEL, 0, 0, [uint32]"$([int64]${hdata} -band 0xFFFFFFFF)", [UIntPtr]::Zero);
|
||||
Write-Output '{"ok": true}';`);
|
||||
return { action_sent: true, direction, amount };
|
||||
},
|
||||
type: async ({ text }) => {
|
||||
if (!text) return { action_sent: false, note: "empty text" };
|
||||
await bootstrapped;
|
||||
const b64 = Buffer.from(String(text), "utf16le").toString("base64");
|
||||
const script = `Add-Type -TypeDefinition @'
|
||||
using System;
|
||||
using System.Text;
|
||||
using System.Runtime.InteropServices;
|
||||
public static class TypeText {
|
||||
[DllImport("user32.dll")] public static extern uint SendInput(uint n, INPUT[] inputs, int size);
|
||||
[StructLayout(LayoutKind.Sequential)] public struct KEYBDINPUT { public ushort wVk; public ushort wScan; public uint dwFlags; public uint time; public IntPtr extra; }
|
||||
[StructLayout(LayoutKind.Sequential)] public struct INPUT { public uint type; public KEYBDINPUT ki; }
|
||||
public static void SendString(string s) {
|
||||
foreach (char c in s) {
|
||||
var down = new INPUT { type = 1, ki = new KEYBDINPUT { wScan = c, dwFlags = 0x0008 | 0x0004 } };
|
||||
var up = new INPUT { type = 1, ki = new KEYBDINPUT { wScan = c, dwFlags = 0x0008 | 0x0004 | 0x0002 } };
|
||||
INPUT[] arr = new INPUT[2]; arr[0] = down; arr[1] = up;
|
||||
SendInput(2, arr, System.Runtime.InteropServices.Marshal.SizeOf(typeof(INPUT)));
|
||||
}
|
||||
}
|
||||
}
|
||||
'@;
|
||||
$text = [System.Text.Encoding]::Unicode.GetString([Convert]::FromBase64String('${b64}'));
|
||||
[TypeText]::SendString($text);
|
||||
Write-Output ('{"ok": true, "chars": ' + $text.Length + '}');`;
|
||||
const r = await ps(script, { timeoutMs: Math.max(20_000, text.length * 60) });
|
||||
if (r.code !== 0) throw new ExecError(`type failed: ${(r.stderr || "").slice(0, 250)}`, r);
|
||||
return { action_sent: true, chars: text.length, strategy: "unicode-sendinput" };
|
||||
},
|
||||
key: async ({ text, repeat = 1 }) => {
|
||||
const parts = String(text).split("+").map((s) => s.trim().toLowerCase());
|
||||
const mods = parts.filter((p) => MODVK[p] != null).map((p) => MODVK[p]);
|
||||
const key = parts.find((p) => MODVK[p] == null);
|
||||
if (!key) throw new ExecError(`no key in "${text}"`);
|
||||
let vk = VK[key];
|
||||
if (vk == null && key.length === 1) vk = key.toUpperCase().charCodeAt(0);
|
||||
if (vk == null) throw new ExecError(`unknown key "${key}"`);
|
||||
const n = Math.max(1, Math.min(100, repeat));
|
||||
await withUser32(`$ins = New-Object 'User32+INPUT[]' 0;
|
||||
$add = { param($i) };
|
||||
$seq = @();
|
||||
${mods.map((m) => `$seq += [User32]::KeyInput(${m}, 0, 0);`).join("\n")}
|
||||
for ($i = 0; $i -lt ${n}; $i++) { $seq += [User32]::KeyInput(${vk}, 0, 0); $seq += [User32]::KeyInput(${vk}, 0, 2); }
|
||||
${[...mods].reverse().map((m) => `$seq += [User32]::KeyInput(${m}, 0, 2);`).join("\n")}
|
||||
$arr = $seq.ToArray();
|
||||
[void][User32]::SendInput($arr.Length, $arr, [System.Runtime.InteropServices.Marshal]::SizeOf([type][User32+INPUT]));
|
||||
Write-Output '{"ok": true}';`);
|
||||
return { action_sent: true, key, repeat: n };
|
||||
},
|
||||
hold_key: async ({ text, duration }) => {
|
||||
const parts = String(text).split("+").map((s) => s.trim().toLowerCase());
|
||||
const key = parts.find((p) => MODVK[p] == null && VK[p] == null ? p.length === 1 : (VK[p] != null || p.length === 1));
|
||||
const vk = VK[key] ?? (key.length === 1 ? key.toUpperCase().charCodeAt(0) : null);
|
||||
if (vk == null) throw new ExecError(`unknown key "${text}"`);
|
||||
const d = Math.max(0.05, Math.min(30, Number(duration) || 1));
|
||||
await withUser32(`[void][User32]::SendInput(1, @([User32]::KeyInput(${vk}, 0, 0)), [System.Runtime.InteropServices.Marshal]::SizeOf([type][User32+INPUT]));
|
||||
Start-Sleep -Milliseconds ${Math.round(d * 1000)};
|
||||
[void][User32]::SendInput(1, @([User32]::KeyInput(${vk}, 0, 2)), [System.Runtime.InteropServices.Marshal]::SizeOf([type][User32+INPUT]));
|
||||
Write-Output '{"ok": true}';`, { timeoutMs: Math.max(10_000, d * 1000 + 8000) });
|
||||
return { action_sent: true, key, heldSec: d };
|
||||
},
|
||||
set_value: async ({ target, value }) => {
|
||||
// UIA ValuePattern via a re-walk to target.path from the desktop root.
|
||||
await bootstrapped;
|
||||
const b64path = Buffer.from(JSON.stringify(target.path ?? []), "utf8").toString("base64");
|
||||
const b64val = Buffer.from(String(value ?? ""), "utf16le").toString("base64");
|
||||
const j = await psJson(`Add-Type -AssemblyName UIAutomationClient; Add-Type -AssemblyName UIAutomationTypes;
|
||||
$path = [Convert]::FromBase64String('${b64path}'); $textPath = [Text.Encoding]::UTF8.GetString($path);
|
||||
$val = [Text.Encoding]::Unicode.GetString([Convert]::FromBase64String('${b64val}'));
|
||||
$indices = $textPath | ConvertFrom-Json;
|
||||
$root = [System.Windows.Automation.AutomationElement]::RootElement;
|
||||
$targets = $root.FindAll([System.Windows.Automation.TreeScope]::Children, [System.Windows.Automation.Condition]::TrueCondition);
|
||||
$cur = $null;
|
||||
foreach ($t in $targets) { $cur = $t; break }
|
||||
if ($cur -eq $null) { Write-Output '{"ok": false, "code": "app_not_found"}'; exit 0 }
|
||||
foreach ($i in $indices) {
|
||||
if ($i -eq 0) { continue }
|
||||
$kids = $cur.FindAll([System.Windows.Automation.TreeScope]::Children, [System.Windows.Automation.Condition]::TrueCondition);
|
||||
if ($i -ge $kids.Count) { Write-Output '{"ok": false, "code": "element_stale"}'; exit 0 }
|
||||
$cur = $kids[$i];
|
||||
}
|
||||
try {
|
||||
$vp = $cur.GetCurrentPattern([System.Windows.Automation.ValuePattern]::Pattern);
|
||||
$vp.SetValue($val);
|
||||
Write-Output '{"ok": true}';
|
||||
} catch { Write-Output ('{"ok": false, "code": "' + $_.Exception.Message.Replace('"','') + '"}') }`, { timeoutMs: 45_000 });
|
||||
if (!j.ok) throw new ExecError(`set_value failed: ${j.code}`);
|
||||
return { action_sent: true, strategy: "a11y" };
|
||||
},
|
||||
select_text: async () => { throw new ExecError("select_text is not implemented on the win32 backend yet — fail-closed"); },
|
||||
perform_action: async ({ target, action }) => {
|
||||
await bootstrapped;
|
||||
const b64path = Buffer.from(JSON.stringify(target.path ?? []), "utf8").toString("base64");
|
||||
const act = String(action ?? "Invoke").replace(/'/g, "");
|
||||
const j = await psJson(`Add-Type -AssemblyName UIAutomationClient; Add-Type -AssemblyName UIAutomationTypes;
|
||||
$path = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('${b64path}'));
|
||||
$indices = $path | ConvertFrom-Json;
|
||||
$root = [System.Windows.Automation.AutomationElement]::RootElement;
|
||||
$targets = $root.FindAll([System.Windows.Automation.TreeScope]::Children, [System.Windows.Automation.Condition]::TrueCondition);
|
||||
$cur = $null;
|
||||
foreach ($t in $targets) { $cur = $t; break }
|
||||
foreach ($i in $indices) {
|
||||
if ($i -eq 0) { continue }
|
||||
$kids = $cur.FindAll([System.Windows.Automation.TreeScope]::Children, [System.Windows.Automation.Condition]::TrueCondition);
|
||||
if ($i -ge $kids.Count) { Write-Output '{"ok": false, "code": "element_stale"}'; exit 0 }
|
||||
$cur = $kids[$i];
|
||||
}
|
||||
$want = '${act}';
|
||||
try {
|
||||
$pats = $cur.GetSupportedPatterns();
|
||||
$names = @($pats | ForEach-Object { $_.ProgrammaticName -replace 'Pattern\$','' -replace 'Pattern$','' });
|
||||
$chosen = $names | Where-Object { $_ -ieq $want } | Select-Object -First 1;
|
||||
if (-not $chosen -and ($want -ieq 'click' -or $want -ieq 'invoke')) { $chosen = 'Invoke' }
|
||||
if (-not $chosen) { Write-Output ('{"ok": false, "code": "action_not_found: " + ($names -join ",") }'); exit 0 }
|
||||
$pt = $cur.GetCurrentPattern($pats | Where-Object { ($_.ProgrammaticName -replace 'Pattern$','') -ieq $chosen } | Select-Object -First 1);
|
||||
if ($chosen -eq 'Invoke') { $pt.Invoke() } elseif ($chosen -eq 'ExpandCollapse') { $pt.Expand() } elseif ($chosen -eq 'Toggle') { $pt.Toggle() } else { $pt.Invoke() }
|
||||
Write-Output '{"ok": true, "sent": true}';
|
||||
} catch { Write-Output ('{"ok": false, "code": "' + $_.Exception.Message.Replace('"','') + '"}') }`, { timeoutMs: 45_000 });
|
||||
if (!j.ok) throw new ExecError(`perform_action failed: ${j.code}`);
|
||||
return { action_sent: true, strategy: "a11y", action };
|
||||
},
|
||||
read_clipboard: async () => {
|
||||
const j = await psJson(`$t = Get-Clipboard -Raw -ErrorAction SilentlyContinue;
|
||||
Write-Output ('{"text": ' + ($t | ConvertTo-Json -Compress) + '}');`, { timeoutMs: 10_000 });
|
||||
return { text: j.text ?? "", encoding: "utf8" };
|
||||
},
|
||||
write_clipboard: async ({ text }) => {
|
||||
const b64 = Buffer.from(String(text ?? ""), "utf16le").toString("base64");
|
||||
await ps(`Set-Clipboard -Value ([System.Text.Encoding]::Unicode.GetString([Convert]::FromBase64String('${b64}')));
|
||||
Write-Output '{"ok": true}';`, { timeoutMs: 10_000 });
|
||||
return { written: String(text ?? "").length };
|
||||
},
|
||||
cursor_position: async () => {
|
||||
await bootstrapped;
|
||||
const j = await psJson(`$p = New-Object User32+POINT;
|
||||
[void][User32]::GetCursorPos([ref]$p);
|
||||
Write-Output ('{"x": ' + $p.X + ', "y": ' + $p.Y + '}');`);
|
||||
return { x: j.x, y: j.y };
|
||||
},
|
||||
recordingStart: async ({ fps = 15, region } = {}) => {
|
||||
let ffmpegOk = true;
|
||||
try { await runOk("ffmpeg", ["-version"], { timeoutMs: 10_000 }); } catch { ffmpegOk = false; }
|
||||
if (!ffmpegOk) throw new ExecError("recording on Windows needs ffmpeg (gdigrab) on PATH — install ffmpeg and retry");
|
||||
const dir = recordingsDir();
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
const id = crypto.randomBytes(4).toString("hex");
|
||||
const file = path.join(dir, `rec-${id}.mp4`);
|
||||
const args = ["-y", "-loglevel", "error", "-f", "gdigrab", "-framerate", String(fps)];
|
||||
if (region) args.push("-offset_x", String(Math.round(region[0])), "-offset_y", String(Math.round(region[1])), "-video_size", `${Math.round(region[2])}x${Math.round(region[3])}`);
|
||||
args.push("-i", "desktop", "-c:v", "libx264", "-pix_fmt", "yuv420p", file);
|
||||
const child = spawn("ffmpeg", args, { stdio: "ignore", detached: true });
|
||||
child.unref();
|
||||
await new Promise((r) => setTimeout(r, 800));
|
||||
try { process.kill(child.pid, 0); } catch { throw new ExecError("ffmpeg gdigrab exited immediately"); }
|
||||
recording = { id, pid: child.pid, file, startedAt: new Date().toISOString(), mode: "gdigrab" };
|
||||
return { id, pid: child.pid, file, mode: "gdigrab", fps };
|
||||
},
|
||||
recordingStop: async ({ id }) => {
|
||||
if (!recording || recording.id !== id) throw new ExecError(`unknown recording "${id}"`);
|
||||
try { process.kill(recording.pid, "SIGINT"); } catch {}
|
||||
await new Promise((r) => setTimeout(r, 1500));
|
||||
const bytes = fs.existsSync(recording.file) ? fs.statSync(recording.file).size : 0;
|
||||
const out = { id, file: recording.file, bytes, mode: recording.mode, startedAt: recording.startedAt, stoppedAt: new Date().toISOString() };
|
||||
recording = null;
|
||||
return out;
|
||||
},
|
||||
recordingStatus: ({ id }) => {
|
||||
if (!recording || recording.id !== id) return { id, running: false };
|
||||
let alive = true;
|
||||
try { process.kill(recording.pid, 0); } catch { alive = false; }
|
||||
return { id, running: alive, file: recording.file, bytes: fs.existsSync(recording.file) ? fs.statSync(recording.file).size : 0, mode: recording.mode };
|
||||
},
|
||||
recordingList: async () => {
|
||||
const dir = recordingsDir();
|
||||
const out = fs.existsSync(dir)
|
||||
? fs.readdirSync(dir).filter((f) => /\.mp4$/i.test(f)).map((f) => {
|
||||
const st = fs.statSync(path.join(dir, f));
|
||||
return { file: path.join(dir, f), bytes: st.size, modifiedAt: st.mtime.toISOString() };
|
||||
}).sort((a, b) => b.modifiedAt.localeCompare(a.modifiedAt)).slice(0, 50)
|
||||
: [];
|
||||
return { dir, recordings: out, running: recording ? [recording.id] : [] };
|
||||
},
|
||||
};
|
||||
|
||||
async function clickAt(button, x, y, clicks) {
|
||||
if (!Number.isFinite(Number(x)) || !Number.isFinite(Number(y))) throw new ExecError("coordinates must be finite numbers");
|
||||
const flags = button === 1 ? "RIGHTDOWN, RIGHTUP" : button === 2 ? "MIDDLEDOWN, MIDDLEUP" : "LEFTDOWN, LEFTUP";
|
||||
const seq = [];
|
||||
for (let i = 0; i < clicks; i++) seq.push(`[User32]::mouse_event([User32]::${flags.split(",")[0].trim()}, 0, 0, 0, [UIntPtr]::Zero); Start-Sleep -Milliseconds 40; [User32]::mouse_event([User32]::${flags.split(",")[1].trim()}, 0, 0, 0, [UIntPtr]::Zero); Start-Sleep -Milliseconds 60;`);
|
||||
await withUser32(`[User32]::SetCursorPos(${Math.round(x)}, ${Math.round(y)}) | Out-Null;
|
||||
Start-Sleep -Milliseconds 60;
|
||||
${seq.join("\n")}
|
||||
Write-Output '{"ok": true}';`, { timeoutMs: 20_000 });
|
||||
return { action_sent: true, at: { x: Number(x), y: Number(y) }, button, clicks };
|
||||
}
|
||||
}
|
||||
|
||||
export default { create };
|
||||
@@ -0,0 +1,87 @@
|
||||
// Process execution helper: spawn, timeout, text capture. Zero dependencies.
|
||||
import { spawn } from "node:child_process";
|
||||
|
||||
/**
|
||||
* Run a command. Never uses a shell: cmd + args array only, so tool arguments
|
||||
* can never become command injection.
|
||||
* @returns {Promise<{code:number|null, stdout:string, stderr:string, timedOut:boolean, signal:string|null}>}
|
||||
*/
|
||||
export function run(cmd, args = [], opts = {}) {
|
||||
const timeoutMs = opts.timeoutMs ?? 20_000;
|
||||
const maxBuffer = opts.maxBuffer ?? 32 * 1024 * 1024;
|
||||
return new Promise((resolve) => {
|
||||
let child;
|
||||
try {
|
||||
child = spawn(cmd, args, {
|
||||
env: opts.env ? { ...process.env, ...opts.env } : process.env,
|
||||
cwd: opts.cwd,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
// Windows: node handles .cmd/.exe resolution for known tools via shell:false + full name
|
||||
windowsHide: true,
|
||||
});
|
||||
} catch (err) {
|
||||
resolve({ code: -1, stdout: "", stderr: String(err?.message ?? err), timedOut: false, signal: null });
|
||||
return;
|
||||
}
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
let timedOut = false;
|
||||
let settled = false;
|
||||
const timer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
try { child.kill("SIGTERM"); } catch {}
|
||||
// Hard kill after grace period
|
||||
setTimeout(() => { try { child.kill("SIGKILL"); } catch {} }, 1500);
|
||||
}, timeoutMs);
|
||||
child.stdout.on("data", (d) => {
|
||||
if (stdout.length < maxBuffer) stdout += d.toString();
|
||||
});
|
||||
child.stderr.on("data", (d) => {
|
||||
if (stderr.length < maxBuffer) stderr += d.toString();
|
||||
});
|
||||
const finish = (code, signal) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
resolve({ code, stdout, stderr, timedOut, signal });
|
||||
};
|
||||
child.on("error", (err) => {
|
||||
stderr += String(err?.message ?? err);
|
||||
finish(-1, null);
|
||||
});
|
||||
child.on("close", (code, signal) => finish(code, signal));
|
||||
});
|
||||
}
|
||||
|
||||
/** run() and throw a typed error on non-zero exit / timeout. */
|
||||
export async function runOk(cmd, args = [], opts = {}) {
|
||||
const r = await run(cmd, args, opts);
|
||||
if (r.timedOut) throw new ExecError(`timeout after ${opts.timeoutMs ?? 20_000}ms: ${cmd}`, r);
|
||||
if (r.code !== 0) throw new ExecError(`${cmd} exited ${r.code}: ${trim(r.stderr || r.stdout)}`, r);
|
||||
return r;
|
||||
}
|
||||
|
||||
export class ExecError extends Error {
|
||||
constructor(message, result) {
|
||||
super(message);
|
||||
this.name = "ExecError";
|
||||
this.result = result;
|
||||
}
|
||||
}
|
||||
|
||||
/** True when the executable exists on PATH (or opts.fullPath exists). */
|
||||
export async function have(cmd) {
|
||||
const probe = process.platform === "win32" ? "where" : "which";
|
||||
const r = await run(probe, [cmd], { timeoutMs: 5000 });
|
||||
return r.code === 0 && r.stdout.trim().length > 0;
|
||||
}
|
||||
|
||||
export function trim(s, n = 400) {
|
||||
s = String(s ?? "").trim();
|
||||
return s.length > n ? s.slice(0, n) + "…" : s;
|
||||
}
|
||||
|
||||
/** Parse JSON safely, returning fallback on failure. */
|
||||
export function tryJson(s, fallback = null) {
|
||||
try { return JSON.parse(s); } catch { return fallback; }
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
// Computer registry: named computers (local, ssh, hdc) with a sticky active
|
||||
// computer. Every codewhale-cu tool defaults to the active computer; passing
|
||||
// `computer` on any tool switches to it first (switch-by-use).
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import os from "node:os";
|
||||
|
||||
export class RegistryError extends Error {
|
||||
constructor(code, message) { super(message); this.name = "RegistryError"; this.code = code; }
|
||||
}
|
||||
|
||||
const ID_RE = /^[a-zA-Z0-9][a-zA-Z0-9._-]{0,63}$/;
|
||||
|
||||
export function stateDir() {
|
||||
return process.env.CODEWHALE_CU_STATE_DIR || path.join(os.homedir(), ".codewhale-cu");
|
||||
}
|
||||
|
||||
function registryPath() { return path.join(stateDir(), "computers.json"); }
|
||||
|
||||
const PLATFORM = process.platform; // darwin | linux | win32
|
||||
|
||||
function localEntry() {
|
||||
return {
|
||||
id: "local",
|
||||
transport: "local",
|
||||
platform: PLATFORM,
|
||||
label: `This ${PLATFORM === "darwin" ? "Mac" : PLATFORM === "win32" ? "Windows PC" : "Linux machine"}`,
|
||||
registeredAt: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
export function load() {
|
||||
try {
|
||||
const raw = JSON.parse(fs.readFileSync(registryPath(), "utf8"));
|
||||
if (!raw || typeof raw !== "object" || !raw.computers) throw new Error("bad shape");
|
||||
return raw;
|
||||
} catch {
|
||||
const fresh = { version: 1, active: "local", computers: { local: localEntry() } };
|
||||
return fresh;
|
||||
}
|
||||
}
|
||||
|
||||
export function save(reg) {
|
||||
fs.mkdirSync(stateDir(), { recursive: true });
|
||||
const tmp = registryPath() + ".tmp";
|
||||
fs.writeFileSync(tmp, JSON.stringify(reg, null, 2) + "\n");
|
||||
fs.renameSync(tmp, registryPath());
|
||||
return reg;
|
||||
}
|
||||
|
||||
export function list() {
|
||||
const reg = load();
|
||||
// Keep the local entry truthful even if the state file is stale.
|
||||
reg.computers.local = { ...(reg.computers.local ?? {}), ...localEntry() };
|
||||
if (!reg.computers[reg.active]) reg.active = "local";
|
||||
return reg;
|
||||
}
|
||||
|
||||
export function get(id) {
|
||||
const reg = list();
|
||||
const c = reg.computers[id];
|
||||
if (!c) throw new RegistryError("unknown_computer", `no computer registered with id "${id}". Use computer_list.`);
|
||||
return c;
|
||||
}
|
||||
|
||||
export function active() { return get(list().active); }
|
||||
|
||||
/** Switch the active computer. Returns the computer entry. */
|
||||
export function switchTo(id) {
|
||||
const c = get(id); // throws unknown_computer
|
||||
const reg = load();
|
||||
reg.active = c.id;
|
||||
save(reg);
|
||||
return c;
|
||||
}
|
||||
|
||||
/** Register or update a computer. Returns the entry. */
|
||||
export function register({ id, transport, label, ...rest }) {
|
||||
if (!id || !ID_RE.test(id)) throw new RegistryError("invalid_id", "computer id must match " + ID_RE);
|
||||
if (!["local", "ssh", "hdc"].includes(transport)) {
|
||||
throw new RegistryError("invalid_transport", "transport must be one of: local, ssh, hdc");
|
||||
}
|
||||
if (id === "local" && transport !== "local") {
|
||||
throw new RegistryError("reserved_id", '"local" is reserved for this machine');
|
||||
}
|
||||
if (transport === "ssh") {
|
||||
if (!rest.host || !/^[A-Za-z0-9._-]+$/.test(rest.host)) {
|
||||
throw new RegistryError("invalid_host", "ssh computers need a valid host (letters, digits, dot, dash, underscore)");
|
||||
}
|
||||
if (rest.port != null && (!Number.isInteger(rest.port) || rest.port < 1 || rest.port > 65535)) {
|
||||
throw new RegistryError("invalid_port", "port must be an integer in 1..65535");
|
||||
}
|
||||
if (rest.user != null && !/^[a-zA-Z0-9._-]+$/.test(rest.user)) {
|
||||
throw new RegistryError("invalid_user", "user must be a plain name");
|
||||
}
|
||||
}
|
||||
if (transport === "hdc") {
|
||||
if (rest.target != null && !/^[A-Za-z0-9._-]*$/.test(rest.target)) {
|
||||
throw new RegistryError("invalid_target", "hdc target key contains invalid characters");
|
||||
}
|
||||
rest.platform = "harmonyos";
|
||||
}
|
||||
const reg = load();
|
||||
const prev = reg.computers[id];
|
||||
reg.computers[id] = {
|
||||
...prev,
|
||||
id,
|
||||
transport,
|
||||
label: label ?? prev?.label ?? id,
|
||||
registeredAt: prev?.registeredAt ?? new Date().toISOString(),
|
||||
...rest,
|
||||
};
|
||||
save(reg);
|
||||
return reg.computers[id];
|
||||
}
|
||||
|
||||
export function remove(id) {
|
||||
if (id === "local") throw new RegistryError("reserved_id", '"local" cannot be removed');
|
||||
const reg = load();
|
||||
if (!reg.computers[id]) throw new RegistryError("unknown_computer", `no computer registered with id "${id}"`);
|
||||
delete reg.computers[id];
|
||||
if (reg.active === id) reg.active = "local";
|
||||
save(reg);
|
||||
return { removed: id, active: reg.active };
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
// Execution facade passed into backends. Locally it is the plain exec module;
|
||||
// remote agents get the same shape, so every backend works unchanged on either
|
||||
// side of an ssh hop.
|
||||
import * as exec from "./exec.mjs";
|
||||
export { exec };
|
||||
export default exec;
|
||||
@@ -0,0 +1,318 @@
|
||||
// codewhale-cu tool schemas — single source of truth for tools/list.
|
||||
// Every action/observation tool accepts an optional `computer` id; supplying it
|
||||
// switches the active computer first (switch-by-use is the default model).
|
||||
const computerParam = {
|
||||
type: "string",
|
||||
description: "Computer id to act on. Defaults to the active computer. Providing a different registered id switches to it first (sticky).",
|
||||
};
|
||||
|
||||
const targetSchema = {
|
||||
oneOf: [
|
||||
{
|
||||
type: "object",
|
||||
description: "Element target from the latest get_app_state on this computer.",
|
||||
required: ["type", "state_id", "index"],
|
||||
properties: {
|
||||
type: { const: "element" },
|
||||
state_id: { type: "string" },
|
||||
index: { type: "integer", minimum: 0 },
|
||||
},
|
||||
additionalProperties: false,
|
||||
},
|
||||
{
|
||||
type: "object",
|
||||
description: "Pixel coordinates in the latest returned raster (screenshot or zoom) for this computer.",
|
||||
required: ["type", "x", "y"],
|
||||
properties: {
|
||||
type: { const: "coordinate" },
|
||||
x: { type: "integer", minimum: 0 },
|
||||
y: { type: "integer", minimum: 0 },
|
||||
},
|
||||
additionalProperties: false,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
export const TOOLS = [
|
||||
// ---- computers (switching is a default) ----
|
||||
{
|
||||
name: "computer_list",
|
||||
description: "List registered computers (local, ssh, harmony/hdc) and which one is active. Every other tool acts on the active computer unless given `computer`.",
|
||||
inputSchema: { type: "object", properties: {}, additionalProperties: false },
|
||||
},
|
||||
{
|
||||
name: "computer_switch",
|
||||
description: "Switch the active computer. Subsequent tools act on it by default.",
|
||||
inputSchema: { type: "object", required: ["computer"], properties: { computer: { type: "string", description: "Registered computer id (see computer_list)" } }, additionalProperties: false },
|
||||
},
|
||||
{
|
||||
name: "computer_register",
|
||||
description: "Register or update a computer. transport=local (this machine), ssh (runs the bundled remote agent over ssh; agent is pushed automatically), hdc (HarmonyOS device via hdc).",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
required: ["computer", "transport"],
|
||||
properties: {
|
||||
computer: { type: "string", description: "Short id for the computer (letters, digits, dot, dash)" },
|
||||
transport: { enum: ["local", "ssh", "hdc"] },
|
||||
label: { type: "string" },
|
||||
host: { type: "string", description: "ssh: hostname" },
|
||||
port: { type: "integer", description: "ssh: port (default 22)" },
|
||||
user: { type: "string", description: "ssh: user" },
|
||||
target: { type: "string", description: "hdc: target key (omit for the only connected device)" },
|
||||
installAgent: { type: "boolean", description: "ssh: push the remote agent before first use (default true)" },
|
||||
},
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "computer_remove",
|
||||
description: "Remove a registered computer. 'local' cannot be removed.",
|
||||
inputSchema: { type: "object", required: ["computer"], properties: { computer: { type: "string" } }, additionalProperties: false },
|
||||
},
|
||||
// ---- observe & resolve ----
|
||||
{
|
||||
name: "request_access",
|
||||
description: "Probe permissions and capabilities of a computer (accessibility, screen capture, recording, missing tools). Call once when readiness is unknown or a permission failure is explicitly named.",
|
||||
inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
{
|
||||
name: "list_displays",
|
||||
description: "List displays/panels with geometry and pixel scale.",
|
||||
inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
{
|
||||
name: "switch_display",
|
||||
description: "Set which display subsequent screenshots/recordings capture on this computer.",
|
||||
inputSchema: { type: "object", required: ["index"], properties: { index: { type: "integer", minimum: 1 }, computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
{
|
||||
name: "list_apps",
|
||||
description: "List running applications (name, pid, bundle id). If the user names an app that is absent, use open_application once with the exact user-provided name.",
|
||||
inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
{
|
||||
name: "list_windows",
|
||||
description: "List windows of an application (or all windows when app_ref is omitted).",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
app_ref: {
|
||||
type: "object",
|
||||
properties: {
|
||||
pid: { type: "integer" }, name: { type: "string" }, bundle_id: { type: "string" },
|
||||
},
|
||||
additionalProperties: false,
|
||||
},
|
||||
computer: computerParam,
|
||||
},
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "get_app_state",
|
||||
description: "Observe an application once: returns a bounded accessibility/UIA/uitest element tree with stable indices. Elements are the primary action targets; request screenshots only when accessibility cannot express the target.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
app_ref: { type: "object", properties: { pid: { type: "integer" }, name: { type: "string" }, bundle_id: { type: "string" } }, additionalProperties: false },
|
||||
window_id: { type: "integer", description: "Zero-based window index within the app" },
|
||||
detail: { enum: ["compact", "full"] },
|
||||
computer: computerParam,
|
||||
},
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "screenshot",
|
||||
description: "Capture the screen (all or one display, optional region) as PNG/JPEG. The receipt carries raster geometry; later coordinate targets refer to this raster.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
display: { type: ["integer", "string"], description: "Display index or 'all'" },
|
||||
region: { type: "array", items: { type: "number" }, minItems: 4, maxItems: 4, description: "[x, y, w, h] in screen points" },
|
||||
path: { type: "string", description: "Optional output path (absolute). Defaults into the recordings directory." },
|
||||
computer: computerParam,
|
||||
},
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "zoom",
|
||||
description: "Close-up crop of the latest screenshot. Choose points from the returned child raster only.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
required: ["region"],
|
||||
properties: {
|
||||
region: { type: "array", items: { type: "number" }, minItems: 4, maxItems: 4, description: "[x, y, w, h] in last-raster pixels" },
|
||||
path: { type: "string" },
|
||||
computer: computerParam,
|
||||
},
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "cursor_position",
|
||||
description: "Read the current pointer position in screen points.",
|
||||
inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
{
|
||||
name: "open_application",
|
||||
description: "Launch or activate an application. Copy user-provided names character-for-character; never translate, normalize, or strip suffixes. On macOS prefer bundle_id when known.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
name: { type: "string" }, bundle_id: { type: "string" }, url: { type: "string" },
|
||||
activate: { type: "boolean", description: "Bring to foreground" },
|
||||
computer: computerParam,
|
||||
},
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
// ---- pointer ----
|
||||
{
|
||||
name: "left_click", description: "Left-click a coordinate (pixels in the latest raster) or perform the element's press action.",
|
||||
inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
{
|
||||
name: "double_click", description: "Double-click a target.",
|
||||
inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
{
|
||||
name: "triple_click", description: "Triple-click a target (e.g. select a paragraph).",
|
||||
inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
{
|
||||
name: "right_click", description: "Right-click a target (context menu).",
|
||||
inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
{
|
||||
name: "middle_click", description: "Middle-click a target.",
|
||||
inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
{
|
||||
name: "mouse_move", description: "Move the pointer without clicking (hover).",
|
||||
inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
{
|
||||
name: "left_click_drag", description: "Press at from_target, move in steps, release at `to`.",
|
||||
inputSchema: { type: "object", required: ["from_target", "to"], properties: { from_target: targetSchema, to: targetSchema, computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
{
|
||||
name: "left_mouse_down", description: "Press and hold the left button at a target. Release with left_mouse_up.",
|
||||
inputSchema: { type: "object", properties: { target: targetSchema, computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
{
|
||||
name: "left_mouse_up", description: "Release the left button pressed by left_mouse_down.",
|
||||
inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
{
|
||||
name: "scroll", description: "Scroll at a target: direction up/down/left/right, amount in lines/notches.",
|
||||
inputSchema: { type: "object", required: ["target"], properties: { target: targetSchema, direction: { enum: ["up", "down", "left", "right"] }, amount: { type: "integer", minimum: 1, maximum: 100 }, computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
// ---- text & keyboard ----
|
||||
{
|
||||
name: "type", description: "Type text into the focused control (unicode). Focus the field first (click/element action).",
|
||||
inputSchema: { type: "object", required: ["text"], properties: { text: { type: "string" }, computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
{
|
||||
name: "key", description: "Press a key or chord, e.g. 'return', 'cmd+c' (macOS), 'ctrl+c' (Linux/Windows). Repeat with `repeat`.",
|
||||
inputSchema: { type: "object", required: ["text"], properties: { text: { type: "string" }, repeat: { type: "integer", minimum: 1, maximum: 100 }, computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
{
|
||||
name: "hold_key", description: "Hold a key for `duration` seconds (0.05..30).",
|
||||
inputSchema: { type: "object", required: ["text", "duration"], properties: { text: { type: "string" }, duration: { type: "number", minimum: 0.05, maximum: 30 }, computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
{
|
||||
name: "set_value", description: "Set an editable element's value through the accessibility layer (background-safe, no keystrokes). Element targets only.",
|
||||
inputSchema: { type: "object", required: ["target", "value"], properties: { target: targetSchema, value: { type: "string" }, computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
{
|
||||
name: "select_text", description: "Select a text range [start, length] in an element, or place the caret when omitted.",
|
||||
inputSchema: { type: "object", properties: { target: targetSchema, text_range: { type: "array", items: { type: "integer" }, minItems: 2, maxItems: 2 }, computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
{
|
||||
name: "perform_action", description: "Invoke a named accessibility action on an element (e.g. AXPress on macOS, Invoke on Windows/UIA, click on harmony). Only actions the element advertises.",
|
||||
inputSchema: { type: "object", required: ["target", "action"], properties: { target: targetSchema, action: { type: "string" }, computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
// ---- clipboard / runtime ----
|
||||
{
|
||||
name: "read_clipboard", description: "Read the system clipboard as UTF-8 text.",
|
||||
inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
{
|
||||
name: "write_clipboard", description: "Write UTF-8 text to the system clipboard.",
|
||||
inputSchema: { type: "object", required: ["text"], properties: { text: { type: "string" }, computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
// ---- recording ----
|
||||
{
|
||||
name: "recording_start",
|
||||
description: "Start screen recording on a computer (mp4/mov). Darwin: screencapture -v (timed or until recording_stop). Linux: x11grab/wf-recorder. Windows: ffmpeg gdigrab. HarmonyOS: snapshot-series muxed with ffmpeg.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
display: { type: ["integer", "string"] },
|
||||
fps: { type: "integer", minimum: 1, maximum: 60, description: "Linux/Windows/harmony-series only" },
|
||||
region: { type: "array", items: { type: "number" }, minItems: 4, maxItems: 4, description: "Linux/Windows only" },
|
||||
durationSec: { type: "number", minimum: 1, maximum: 7200, description: "macOS only: auto-stop after N seconds" },
|
||||
intervalMs: { type: "integer", minimum: 150, maximum: 5000, description: "harmony snapshot-series frame interval" },
|
||||
computer: computerParam,
|
||||
},
|
||||
additionalProperties: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "recording_stop",
|
||||
description: "Stop a running recording and finalize the file.",
|
||||
inputSchema: { type: "object", required: ["id"], properties: { id: { type: "string" }, computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
{
|
||||
name: "recording_status",
|
||||
description: "Status of one recording (running, bytes so far).",
|
||||
inputSchema: { type: "object", required: ["id"], properties: { id: { type: "string" }, computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
{
|
||||
name: "recording_list",
|
||||
description: "List recordings and screenshots saved on a computer.",
|
||||
inputSchema: { type: "object", properties: { computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
// ---- kill switch ----
|
||||
{
|
||||
name: "stop_computer_control",
|
||||
description: "Kill switch: refuse all further computer-use actions for the rest of the session. Read-only probes stay available.",
|
||||
inputSchema: { type: "object", properties: { reason: { type: "string" } }, additionalProperties: false },
|
||||
},
|
||||
{
|
||||
name: "wait",
|
||||
description: "Pause before the next observation (0..30s). Use after actions that animate or load.",
|
||||
inputSchema: { type: "object", properties: { seconds: { type: "number", minimum: 0, maximum: 30 } }, additionalProperties: false },
|
||||
},
|
||||
];
|
||||
|
||||
export const TOOL_NAMES = new Set(TOOLS.map((t) => t.name));
|
||||
|
||||
/** Tools that never touch a computer (available even after kill switch). */
|
||||
export const READ_ONLY_TOOLS = new Set([
|
||||
"computer_list", "stop_computer_control", "wait", "request_access", "recording_list", "recording_status",
|
||||
]);
|
||||
|
||||
/** Tools dispatchable to a remote agent over ssh (allow-list must match agent.mjs). */
|
||||
export const REMOTE_TOOLS = new Set([
|
||||
"probe", "list_displays", "switch_display", "list_apps", "list_windows",
|
||||
"open_application", "get_app_state", "screenshot", "zoom",
|
||||
"left_click", "double_click", "triple_click", "right_click", "middle_click",
|
||||
"mouse_move", "left_click_drag", "left_mouse_down", "left_mouse_up", "scroll",
|
||||
"type", "key", "hold_key", "set_value", "select_text", "perform_action",
|
||||
"read_clipboard", "write_clipboard", "cursor_position",
|
||||
"recordingStart", "recordingStop", "recordingStatus", "recordingList",
|
||||
]);
|
||||
|
||||
/** Map public tool name -> backend method name. */
|
||||
export const BACKEND_METHOD = Object.fromEntries(
|
||||
TOOLS.filter((t) => !["computer_list", "computer_switch", "computer_register", "computer_remove", "stop_computer_control", "wait"].includes(t.name))
|
||||
.map((t) => [t.name, {
|
||||
recording_start: "recordingStart",
|
||||
recording_stop: "recordingStop",
|
||||
recording_status: "recordingStatus",
|
||||
recording_list: "recordingList",
|
||||
}[t.name] ?? t.name]),
|
||||
);
|
||||
@@ -0,0 +1,151 @@
|
||||
// Transport: turn a registered computer into an executor.
|
||||
// - local: spawn directly
|
||||
// - ssh: run the codewhale-cu remote agent over ssh (args travel as base64 JSON,
|
||||
// so no tool argument can ever become remote shell syntax)
|
||||
// - hdc: HarmonyOS device over `hdc` shell / file push-pull
|
||||
import { run, runOk, ExecError } from "./exec.mjs";
|
||||
import { spawn } from "node:child_process";
|
||||
import crypto from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import os from "node:os";
|
||||
import url from "node:url";
|
||||
|
||||
const __dirname = path.dirname(url.fileURLToPath(import.meta.url));
|
||||
export const PLUGIN_ROOT = path.resolve(__dirname, "..");
|
||||
|
||||
export function b64(obj) {
|
||||
return Buffer.from(JSON.stringify(obj), "utf8").toString("base64");
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a remote-side filesystem path we construct ourselves.
|
||||
* Blocks shell metacharacters and traversal outside the agent dir.
|
||||
*/
|
||||
export function safeRemotePath(p) {
|
||||
if (typeof p !== "string" || !/^[A-Za-z0-9.][A-Za-z0-9/._-]{0,511}$/.test(p) || p.includes("..")) {
|
||||
throw new ExecError(`refusing unsafe remote path: ${JSON.stringify(p)}`);
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
/**
|
||||
* Local executor bound to a platform backend name.
|
||||
* All backends receive this shape.
|
||||
*/
|
||||
export function localExec() {
|
||||
return {
|
||||
kind: "local",
|
||||
run,
|
||||
runOk,
|
||||
async readFile(p) { return fs.promises.readFile(p); },
|
||||
async writeFile(p, data) { return fs.promises.writeFile(p, data); },
|
||||
tmpFile(prefix) {
|
||||
return path.join(fs.mkdtempSync(path.join(os.tmpdir(), prefix)), "out");
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** ssh executor: speaks to the remote agent installed by installRemoteAgent(). */
|
||||
export function sshExec(computer) {
|
||||
const userHost = computer.user ? `${computer.user}@${computer.host}` : computer.host;
|
||||
const portArgs = computer.port ? ["-p", String(computer.port)] : [];
|
||||
const remoteAgent = safeRemotePath(computer.agentPath ?? ".codewhale-cu/agent/agent.mjs");
|
||||
const base = ["-o", "BatchMode=yes", "-o", "ConnectTimeout=8", "-o", "StrictHostKeyChecking=accept-new", ...portArgs, userHost];
|
||||
return {
|
||||
kind: "ssh",
|
||||
base,
|
||||
userHost,
|
||||
remoteAgent,
|
||||
run(cmd, args = [], opts = {}) {
|
||||
// Local side commands (e.g. ssh itself) run directly.
|
||||
return run(cmd, args, opts);
|
||||
},
|
||||
async remote(request, opts = {}) {
|
||||
const r = await run("ssh", [...base, "node", remoteAgent, b64({ args: request.args ?? {}, tool: request.tool, nonce: crypto.randomBytes(6).toString("hex") })], {
|
||||
timeoutMs: opts.timeoutMs ?? 25_000,
|
||||
});
|
||||
if (r.timedOut) throw new ExecError(`ssh ${userHost}: timed out`, r);
|
||||
if (r.code !== 0) throw new ExecError(`ssh ${userHost} exited ${r.code}: ${r.stderr.trim().slice(0, 400)}`, r);
|
||||
// The agent prints exactly one JSON line; anything before it is MOTD noise.
|
||||
const line = r.stdout.trim().split("\n").filter((l) => l.startsWith("{")).pop();
|
||||
const reply = line ? JSON.parse(line) : null;
|
||||
if (!reply) throw new ExecError(`ssh ${userHost}: agent returned no JSON receipt`, r);
|
||||
return reply;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Push the self-contained remote agent + src tree to an ssh computer. */
|
||||
export async function installRemoteAgent(computer) {
|
||||
const ex = sshExec(computer);
|
||||
const srcDir = path.join(PLUGIN_ROOT, "src");
|
||||
const rels = ["agent.mjs"];
|
||||
for (const dir of ["", "backends"]) {
|
||||
const full = path.join(srcDir, dir);
|
||||
for (const f of fs.readdirSync(full)) {
|
||||
if (f.endsWith(".mjs")) rels.push(`src/${dir ? dir + "/" : ""}${f}`);
|
||||
}
|
||||
}
|
||||
const marker = ".codewhale-cu/agent";
|
||||
let r = await run("ssh", [...ex.base, "mkdir", "-p", `${marker}/src/backends`], { timeoutMs: 15_000 });
|
||||
if (r.code !== 0) throw new ExecError(`ssh ${ex.userHost}: mkdir failed: ${r.stderr.trim().slice(0, 300)}`, r);
|
||||
for (const rel of rels) {
|
||||
const localPath = rel === "agent.mjs" ? path.join(PLUGIN_ROOT, "agent.mjs") : path.join(srcDir, rel.slice(4));
|
||||
const dest = safeRemotePath(`${marker}/${rel}`);
|
||||
r = await run("scp", [...(computer.port ? ["-P", String(computer.port)] : []), localPath, `${ex.userHost}:${dest}`], { timeoutMs: 30_000 });
|
||||
if (r.code !== 0) throw new ExecError(`scp ${rel} failed: ${r.stderr.trim().slice(0, 300)}`, r);
|
||||
}
|
||||
// Probe remote platform via the agent itself.
|
||||
const reply = await ex.remote({ tool: "platform" });
|
||||
return { installed: rels.length, remotePlatform: reply.platform, agentPath: `${marker}/agent.mjs` };
|
||||
}
|
||||
|
||||
/** hdc (HarmonyOS) executor. Commands run on-device; files pull to local tmp. */
|
||||
export function hdcExec(computer) {
|
||||
const targetArgs = computer.target ? ["-t", computer.target] : [];
|
||||
const shell = (args, opts = {}) => run("hdc", [...targetArgs, "shell", ...args], opts);
|
||||
return {
|
||||
kind: "hdc",
|
||||
targetArgs,
|
||||
run,
|
||||
runOk,
|
||||
shell,
|
||||
async pullFile(remotePath, localPath, opts = {}) {
|
||||
safeRemotePath(remotePath);
|
||||
const r = await run("hdc", [...targetArgs, "file", "recv", remotePath, localPath], opts);
|
||||
if (r.code !== 0) throw new ExecError(`hdc file recv failed: ${r.stderr.trim().slice(0, 300)}`, r);
|
||||
return localPath;
|
||||
},
|
||||
async readFile(remotePath, opts = {}) {
|
||||
const tmp = path.join(os.tmpdir(), `cu-hdc-${crypto.randomBytes(4).toString("hex")}`);
|
||||
await this.pullFile(remotePath, tmp, opts);
|
||||
const data = await fs.promises.readFile(tmp);
|
||||
await fs.promises.rm(path.dirname(tmp), { recursive: true, force: true }).catch(() => {});
|
||||
return data;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function executorFor(computer) {
|
||||
if (computer.transport === "local") return localExec();
|
||||
if (computer.transport === "ssh") return sshExec(computer);
|
||||
if (computer.transport === "hdc") return hdcExec(computer);
|
||||
throw new ExecError(`unknown transport ${computer.transport}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a computer to its backend module. Local platform is fixed; ssh
|
||||
* computers may carry platformHint (probed at registration).
|
||||
*/
|
||||
export async function backendFor(computer) {
|
||||
let platform = computer.platform ?? computer.platformHint;
|
||||
if (!platform) {
|
||||
if (computer.transport === "local") platform = process.platform;
|
||||
else if (computer.transport === "hdc") platform = "harmonyos";
|
||||
else platform = "linux"; // conservative default for ssh; registration probes it
|
||||
}
|
||||
const mod = await import(`./backends/${platform}.mjs`);
|
||||
const exec = await executorFor(computer);
|
||||
return { backend: mod.create({ exec, computer, platform }), platform };
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
// Backend tests that can run on any host: harmony logic via a mocked hdc
|
||||
// exec, linux fail-closed probing, and module-shape checks for win32.
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { parseBounds, flatten } from "../src/backends/harmonyos.mjs";
|
||||
|
||||
function fakeJpeg(w, h) {
|
||||
// Minimal JPEG with an SOF0 marker carrying the dimensions.
|
||||
return Buffer.from([
|
||||
0xff, 0xd8, 0xff, 0xc0, 0x00, 0x0b, 0x08, (h >> 8) & 0xff, h & 0xff,
|
||||
(w >> 8) & 0xff, w & 0xff, 0x01, 0x01, 0x11, 0x00, 0xff, 0xd9,
|
||||
]);
|
||||
}
|
||||
|
||||
function harmonyFixtureExec(t) {
|
||||
const layout = {
|
||||
attributes: { bundleName: "com.example.app", type: "FrameNode" },
|
||||
children: [
|
||||
{
|
||||
attributes: { type: "Button", text: "OK", id: "ok_btn", bounds: "[100,200][300,260]" },
|
||||
children: [],
|
||||
},
|
||||
{
|
||||
attributes: { type: "Text", text: "Hello", bounds: "[0,0][100,50]" },
|
||||
children: [{ attributes: { type: "Text", text: "nested", bounds: "[10,10][90,40]" }, children: [] }],
|
||||
},
|
||||
],
|
||||
};
|
||||
const calls = [];
|
||||
const exec = {
|
||||
targetArgs: [],
|
||||
run: async () => ({ code: 0, stdout: "", stderr: "" }),
|
||||
runOk: async () => ({ code: 0, stdout: "", stderr: "" }),
|
||||
shell: async (args) => { calls.push({ kind: "shell", args }); return { code: 0, stdout: "", stderr: "" }; },
|
||||
async pullFile(remote, local) {
|
||||
calls.push({ kind: "pull", remote, local });
|
||||
if (remote.includes("layout")) fs.writeFileSync(local, Buffer.from(JSON.stringify(layout)));
|
||||
else fs.writeFileSync(local, fakeJpeg(168, 120));
|
||||
return local;
|
||||
},
|
||||
async readFile(remote) {
|
||||
if (remote.includes("layout")) return Buffer.from(JSON.stringify(layout));
|
||||
return fakeJpeg(168, 120);
|
||||
},
|
||||
};
|
||||
return { exec, calls };
|
||||
}
|
||||
|
||||
test("harmony: parseBounds handles uitest bounds strings", () => {
|
||||
assert.deepEqual(parseBounds("[100,200][300,260]"), { x: 100, y: 200, w: 200, h: 60, cx: 200, cy: 230 });
|
||||
assert.equal(parseBounds("garbage"), null);
|
||||
});
|
||||
|
||||
test("harmony: flatten produces indexed elements with paths and geometry", () => {
|
||||
const tree = { attributes: { type: "root" }, children: [{ attributes: { type: "Button", text: "OK", bounds: "[0,0][10,10]" } }] };
|
||||
const els = flatten(tree);
|
||||
assert.equal(els[0].role, "root");
|
||||
assert.equal(els[0].path.length, 0);
|
||||
assert.equal(els[1].label, "OK");
|
||||
assert.deepEqual(els[1].path, [0]);
|
||||
assert.equal(els[1].bounds.cx, 5);
|
||||
});
|
||||
|
||||
test("harmony: get_app_state flattens dumpLayout with indices and actions", async () => {
|
||||
const { exec } = harmonyFixtureExec();
|
||||
const mod = await import("../src/backends/harmonyos.mjs");
|
||||
const b = mod.create({ exec });
|
||||
const st = await b.get_app_state({});
|
||||
assert.equal(st.bundle_id, "com.example.app");
|
||||
assert.ok(st.elements.length >= 4);
|
||||
const ok = st.elements.find((e) => e.label === "OK");
|
||||
assert.ok(ok, "OK button flattened");
|
||||
assert.deepEqual(ok.bounds, { x: 100, y: 200, w: 200, h: 60, cx: 200, cy: 230 });
|
||||
assert.ok(ok.actions.includes("click"));
|
||||
});
|
||||
|
||||
test("harmony: screenshot pulls the file and reports panel dimensions", async () => {
|
||||
const { exec } = harmonyFixtureExec();
|
||||
const mod = await import("../src/backends/harmonyos.mjs");
|
||||
const b = mod.create({ exec });
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "cu-hm-test-"));
|
||||
const shot = await b.screenshot({ path: path.join(dir, "shot.jpeg") });
|
||||
assert.equal(shot.pixels.w, 168);
|
||||
assert.equal(shot.pixels.h, 120);
|
||||
assert.ok(fs.existsSync(shot.file));
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("harmony: click routes through uitest uiInput with validated args", async () => {
|
||||
const { exec, calls } = harmonyFixtureExec();
|
||||
const mod = await import("../src/backends/harmonyos.mjs");
|
||||
const b = mod.create({ exec });
|
||||
const r = await b.left_click({ target: { x: 123.6, y: 45.2 } });
|
||||
assert.equal(r.action_sent, true);
|
||||
const ui = calls.find((c) => c.args[0] === "uitest");
|
||||
assert.deepEqual(ui.args, ["uitest", "uiInput", "click", "124", "45"]);
|
||||
});
|
||||
|
||||
test("harmony: clipboard and select_text fail closed with named reasons", async () => {
|
||||
const { exec } = harmonyFixtureExec();
|
||||
const mod = await import("../src/backends/harmonyos.mjs");
|
||||
const b = mod.create({ exec });
|
||||
await assert.rejects(() => b.read_clipboard(), /not exposed by hdc/);
|
||||
await assert.rejects(() => b.select_text({}), /not exposed by uitest/);
|
||||
assert.throws(() => b.key({ text: "cmd+c" }), /unsupported key/);
|
||||
});
|
||||
|
||||
test("linux: probe reports the session and names missing tools (fail-closed)", async () => {
|
||||
const mod = await import("../src/backends/linux.mjs");
|
||||
const b = mod.create({ exec: (await import("../src/remote-runtime.mjs")).exec });
|
||||
const p = await b.probe();
|
||||
assert.equal(p.platform, "linux");
|
||||
assert.ok(["x11", "wayland", null].includes(p.session));
|
||||
assert.equal(typeof p.missing.length, "number");
|
||||
// On a host with no display session, screenshot must fail closed.
|
||||
if (p.session === null) {
|
||||
await assert.rejects(() => b.screenshot({}), /X11|Wayland/);
|
||||
}
|
||||
});
|
||||
|
||||
test("win32: module loads with the full backend surface", async () => {
|
||||
const mod = await import("../src/backends/win32.mjs");
|
||||
assert.equal(typeof mod.create, "function");
|
||||
if (process.platform !== "win32") {
|
||||
const b = mod.create();
|
||||
for (const m of ["probe", "screenshot", "left_click", "type", "key", "recordingStart", "get_app_state", "set_value"]) {
|
||||
assert.equal(typeof b[m], "function", `win32 backend missing ${m}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("remote agent refuses tools outside the allow-list", async () => {
|
||||
const { run } = await import("../src/exec.mjs");
|
||||
const sentinel = path.join(fs.mkdtempSync(path.join(os.tmpdir(), "cu-agent-deny-")), "x");
|
||||
const payload = Buffer.from(JSON.stringify({ tool: "write_file", args: { path: sentinel } })).toString("base64");
|
||||
const r = await run("node", [new URL("../agent.mjs", import.meta.url).pathname, payload]);
|
||||
const reply = JSON.parse(r.stdout.trim());
|
||||
assert.equal(reply.ok, false);
|
||||
assert.equal(reply.error.code, "tool_not_allowed");
|
||||
assert.ok(!fs.existsSync(sentinel));
|
||||
});
|
||||
|
||||
test("remote agent answers the platform probe", async () => {
|
||||
const { run } = await import("../src/exec.mjs");
|
||||
const payload = Buffer.from(JSON.stringify({ tool: "platform" })).toString("base64");
|
||||
const r = await run("node", [new URL("../agent.mjs", import.meta.url).pathname, payload]);
|
||||
const reply = JSON.parse(r.stdout.trim());
|
||||
assert.equal(reply.ok, true);
|
||||
assert.equal(reply.platform, process.platform);
|
||||
});
|
||||
@@ -0,0 +1,57 @@
|
||||
// exec + transport safety tests.
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { run, runOk, ExecError, have, trim } from "../src/exec.mjs";
|
||||
import { safeRemotePath, b64, localExec } from "../src/transport.mjs";
|
||||
|
||||
test("run captures stdout/stderr and exit codes without a shell", async () => {
|
||||
const r = await run("node", ["-e", "console.log('hello'); console.error('boo')"]);
|
||||
assert.equal(r.code, 0);
|
||||
assert.equal(r.stdout.trim(), "hello");
|
||||
assert.match(r.stderr, /boo/);
|
||||
});
|
||||
|
||||
test("run reports missing executables as code -1 with ENOENT, never throws", async () => {
|
||||
const r = await run("definitely-not-a-real-tool-xyz", ["--version"]);
|
||||
assert.equal(r.code, -1);
|
||||
assert.match(r.stderr, /ENOENT/);
|
||||
});
|
||||
|
||||
test("runOk throws ExecError on non-zero exit and includes stderr", async () => {
|
||||
await assert.rejects(() => runOk("node", ["-e", "console.error('reason-here'); process.exit(3)"]), (e) => {
|
||||
assert.ok(e instanceof ExecError);
|
||||
assert.match(e.message, /exited 3/);
|
||||
assert.match(e.message, /reason-here/);
|
||||
return true;
|
||||
});
|
||||
});
|
||||
|
||||
test("run enforces timeouts", async () => {
|
||||
const r = await run("node", ["-e", "setInterval(()=>{},1000)"], { timeoutMs: 300 });
|
||||
assert.equal(r.timedOut, true);
|
||||
});
|
||||
|
||||
test("have() detects real and missing tools", async () => {
|
||||
assert.equal(await have("node"), true);
|
||||
assert.equal(await have("definitely-not-a-real-tool-xyz"), false);
|
||||
});
|
||||
|
||||
test("safeRemotePath blocks traversal, metacharacters, and absolute escapes", () => {
|
||||
assert.equal(safeRemotePath(".codewhale-cu/agent/agent.mjs"), ".codewhale-cu/agent/agent.mjs");
|
||||
for (const bad of ["../../etc/passwd", "/etc/passwd", "a;rm -rf /", "a b", "$(id)", "a\nb", "a'b", ".codewhale-cu/../escape"]) {
|
||||
assert.throws(() => safeRemotePath(bad), ExecError, `should reject: ${bad}`);
|
||||
}
|
||||
});
|
||||
|
||||
test("b64 round-trips JSON payloads", () => {
|
||||
const obj = { tool: "screenshot", args: { region: [0, 0, 10, 10] } };
|
||||
assert.deepEqual(JSON.parse(Buffer.from(b64(obj), "base64").toString("utf8")), obj);
|
||||
});
|
||||
|
||||
test("localExec provides run/runOk/tmpFile", async () => {
|
||||
const ex = localExec();
|
||||
const r = await ex.run("echo", ["hi"]);
|
||||
assert.equal(r.code, 0);
|
||||
const f = ex.tmpFile("cu-test-");
|
||||
assert.ok(typeof f === "string");
|
||||
});
|
||||
@@ -0,0 +1,52 @@
|
||||
// Packaging tests: the plugin bundle must satisfy the Codewhale Engine's
|
||||
// Agent Plugins v1 contract (plugin.json + sibling mcp.json), because the
|
||||
// Engine — not this repo — installs it.
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import url from "node:url";
|
||||
|
||||
const ROOT = path.dirname(path.dirname(url.fileURLToPath(import.meta.url)));
|
||||
const read = (name) => JSON.parse(fs.readFileSync(path.join(ROOT, name), "utf8"));
|
||||
|
||||
test("plugin.json is an Agent Plugins v1 manifest with the Codewhale extension", () => {
|
||||
const manifest = read("plugin.json");
|
||||
assert.equal(manifest.$schema, "https://agent-plugins.org/schemas/plugin.json");
|
||||
assert.equal(manifest.name, "computer-use");
|
||||
assert.match(manifest.version, /^\d+\.\d+\.\d+$/);
|
||||
|
||||
const ext = manifest.extensions["net.codewhale"];
|
||||
assert.deepEqual(ext.commands, { path: "commands" });
|
||||
assert.deepEqual(ext.skills, { path: "skills" });
|
||||
assert.deepEqual(ext.when.binaries, ["node"]);
|
||||
// HarmonyOS is a target device, never a host that runs the server.
|
||||
assert.deepEqual([...ext.when.os].sort(), ["linux", "macos", "windows"]);
|
||||
|
||||
const rootKeys = ["$schema", "name", "version", "description", "author", "homepage", "repository", "license", "keywords", "extensions"];
|
||||
for (const key of Object.keys(manifest)) assert.ok(rootKeys.includes(key), `unknown root key ${key}`);
|
||||
});
|
||||
|
||||
test("mcp.json declares one stdio server on a contained relative path", () => {
|
||||
const servers = read("mcp.json").mcpServers;
|
||||
assert.deepEqual(Object.keys(servers), ["computer"]);
|
||||
const server = servers.computer;
|
||||
assert.equal(server.type, "stdio");
|
||||
assert.equal(server.command, "node");
|
||||
assert.equal(server.cwd, ".");
|
||||
for (const arg of server.args) {
|
||||
assert.ok(!path.isAbsolute(arg), `${arg} must be relative`);
|
||||
assert.ok(!arg.split("/").includes(".."), `${arg} must not escape the plugin root`);
|
||||
}
|
||||
assert.ok(fs.existsSync(path.join(ROOT, server.args[0])), "server entrypoint exists");
|
||||
});
|
||||
|
||||
test("declared components exist and every skill is named for its directory", () => {
|
||||
assert.ok(fs.existsSync(path.join(ROOT, "commands/computer.md")));
|
||||
const skills = fs.readdirSync(path.join(ROOT, "skills"));
|
||||
assert.deepEqual(skills.sort(), ["computer-use", "recording"]);
|
||||
for (const skill of skills) {
|
||||
const text = fs.readFileSync(path.join(ROOT, "skills", skill, "SKILL.md"), "utf8");
|
||||
assert.match(text, new RegExp(`^---\\nname: ${skill}\\ndescription: .+`), `${skill} frontmatter`);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,58 @@
|
||||
// Registry tests: registration validation, switching, removal, persistence.
|
||||
import { test, beforeEach } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "cu-reg-test-"));
|
||||
process.env.CODEWHALE_CU_STATE_DIR = tmp;
|
||||
|
||||
const registry = await import("../src/registry.mjs");
|
||||
|
||||
beforeEach(() => {
|
||||
fs.rmSync(tmp, { recursive: true, force: true });
|
||||
fs.mkdirSync(tmp, { recursive: true });
|
||||
});
|
||||
|
||||
test("local computer is always registered and active by default", () => {
|
||||
const reg = registry.list();
|
||||
assert.equal(reg.active, "local");
|
||||
assert.equal(reg.computers.local.transport, "local");
|
||||
assert.equal(reg.computers.local.platform, process.platform);
|
||||
});
|
||||
|
||||
test("register accepts valid ssh and hdc computers and persists them", () => {
|
||||
registry.register({ id: "winbox", transport: "ssh", host: "winbox.lan", user: "me", port: 2222 });
|
||||
registry.register({ id: "pad", transport: "hdc", target: "ABC123" });
|
||||
const reg = registry.list();
|
||||
assert.equal(reg.computers.winbox.host, "winbox.lan");
|
||||
assert.equal(reg.computers.pad.platform, "harmonyos");
|
||||
// persisted across a fresh load
|
||||
const again = registry.load();
|
||||
assert.ok(again.computers.winbox && again.computers.pad);
|
||||
});
|
||||
|
||||
test("register rejects invalid ids, hosts, ports, transports", () => {
|
||||
assert.throws(() => registry.register({ id: "bad id!", transport: "ssh", host: "h" }), (e) => e.code === "invalid_id");
|
||||
assert.throws(() => registry.register({ id: "x", transport: "carrier-pigeon" }), (e) => e.code === "invalid_transport");
|
||||
assert.throws(() => registry.register({ id: "x", transport: "ssh", host: "bad host;rm -rf" }), (e) => e.code === "invalid_host");
|
||||
assert.throws(() => registry.register({ id: "x", transport: "ssh", host: "h", port: 99_999 }), (e) => e.code === "invalid_port");
|
||||
assert.throws(() => registry.register({ id: "local", transport: "ssh", host: "h" }), (e) => e.code === "reserved_id");
|
||||
});
|
||||
|
||||
test("switchTo validates and persists the active computer", () => {
|
||||
registry.register({ id: "pad", transport: "hdc" });
|
||||
registry.switchTo("pad");
|
||||
assert.equal(registry.active().id, "pad");
|
||||
assert.throws(() => registry.switchTo("nope"), (e) => e.code === "unknown_computer");
|
||||
});
|
||||
|
||||
test("remove falls back to local when removing the active computer", () => {
|
||||
registry.register({ id: "pad", transport: "hdc" });
|
||||
registry.switchTo("pad");
|
||||
const res = registry.remove("pad");
|
||||
assert.equal(res.active, "local");
|
||||
assert.throws(() => registry.remove("local"), (e) => e.code === "reserved_id");
|
||||
assert.throws(() => registry.remove("pad"), (e) => e.code === "unknown_computer");
|
||||
});
|
||||
@@ -0,0 +1,162 @@
|
||||
// Server protocol tests: real MCP server process over stdio, isolated state.
|
||||
// The ssh/scp shims stand in for a remote machine, proving the full remote
|
||||
// agent loop (install -> platform probe -> tool dispatch) without real ssh.
|
||||
import { test, before, after } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { spawn } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import url from "node:url";
|
||||
|
||||
const __dirname = path.dirname(url.fileURLToPath(import.meta.url));
|
||||
const ROOT = path.resolve(__dirname, "..");
|
||||
|
||||
const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "cu-proto-state-"));
|
||||
const recDir = fs.mkdtempSync(path.join(os.tmpdir(), "cu-proto-rec-"));
|
||||
const fakeHome = fs.mkdtempSync(path.join(os.tmpdir(), "cu-proto-home-"));
|
||||
const binDir = fs.mkdtempSync(path.join(os.tmpdir(), "cu-proto-bin-"));
|
||||
|
||||
// Fake ssh: rebuild the remote command after user@host, then either run the
|
||||
// agent or emulate the one remote command the installer needs (mkdir -p).
|
||||
fs.writeFileSync(path.join(binDir, "ssh"), `#!/bin/bash
|
||||
CMD=()
|
||||
FOUND=0
|
||||
for a in "$@"; do
|
||||
if [ "$FOUND" -eq 1 ]; then CMD+=("$a"); fi
|
||||
case "$a" in *@*) [ "$FOUND" -eq 0 ] && FOUND=1 ;; esac
|
||||
done
|
||||
SUB="\${CMD[0]}"
|
||||
if [ "$SUB" = "node" ]; then
|
||||
exec node "$FAKE_HOME/\${CMD[1]}" "\${CMD[2]}"
|
||||
fi
|
||||
if [ "$SUB" = "mkdir" ]; then
|
||||
LAST="\${CMD[\${#CMD[@]}-1]}"
|
||||
mkdir -p "$FAKE_HOME/$LAST"
|
||||
exit 0
|
||||
fi
|
||||
exit 0
|
||||
`);
|
||||
// Fake scp: copies <src> to <user@host:dest> under FAKE_HOME.
|
||||
fs.writeFileSync(path.join(binDir, "scp"), `#!/bin/bash
|
||||
SRC="$(printf '%s\\n' "$@" | tail -n 2 | head -n 1)"
|
||||
DEST="$(printf '%s\\n' "$@" | tail -n 1)"
|
||||
DEST="$FAKE_HOME/\${DEST#*:}"
|
||||
mkdir -p "$(dirname "$DEST")"
|
||||
cp "$SRC" "$DEST"
|
||||
`);
|
||||
fs.chmodSync(path.join(binDir, "ssh"), 0o755);
|
||||
fs.chmodSync(path.join(binDir, "scp"), 0o755);
|
||||
|
||||
let server;
|
||||
let buf = "";
|
||||
const pending = new Map();
|
||||
let nextId = 1;
|
||||
|
||||
function rpc(method, params, timeoutMs = 90_000) {
|
||||
const id = nextId++;
|
||||
return new Promise((resolve, reject) => {
|
||||
const t = setTimeout(() => { pending.delete(id); reject(new Error(`timeout: ${method}`)); }, timeoutMs);
|
||||
pending.set(id, (msg) => { clearTimeout(t); resolve(msg); });
|
||||
server.stdin.write(JSON.stringify({ jsonrpc: "2.0", id, method, params }) + "\n");
|
||||
});
|
||||
}
|
||||
|
||||
async function tool(name, args = {}) {
|
||||
const res = await rpc("tools/call", { name, arguments: args });
|
||||
assert.ok(res.result, `${name}: protocol error ${JSON.stringify(res.error ?? {})}`);
|
||||
return JSON.parse(res.result.content[0].text);
|
||||
}
|
||||
|
||||
before(async () => {
|
||||
server = spawn("node", [path.join(ROOT, "mcp", "server.mjs")], {
|
||||
env: {
|
||||
...process.env,
|
||||
PATH: `${binDir}:${process.env.PATH}`,
|
||||
FAKE_HOME: fakeHome,
|
||||
CODEWHALE_CU_STATE_DIR: stateDir,
|
||||
CODEWHALE_CU_RECORDINGS_DIR: recDir,
|
||||
},
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
});
|
||||
server.stderr.on("data", (d) => process.stderr.write(`[server] ${d}`));
|
||||
server.stdout.setEncoding("utf8");
|
||||
server.stdout.on("data", (d) => {
|
||||
buf += d;
|
||||
let i;
|
||||
while ((i = buf.indexOf("\n")) !== -1) {
|
||||
const line = buf.slice(0, i).trim();
|
||||
buf = buf.slice(i + 1);
|
||||
if (!line) continue;
|
||||
try {
|
||||
const msg = JSON.parse(line);
|
||||
if (msg.id && pending.has(msg.id)) { pending.get(msg.id)(msg); pending.delete(msg.id); }
|
||||
} catch {}
|
||||
}
|
||||
});
|
||||
const init = await rpc("initialize", { protocolVersion: "2025-06-18" });
|
||||
assert.equal(init.result.serverInfo.name, "codewhale-cu");
|
||||
});
|
||||
|
||||
after(() => {
|
||||
server?.kill("SIGTERM");
|
||||
for (const d of [stateDir, recDir, fakeHome]) { try { fs.rmSync(d, { recursive: true, force: true }); } catch {} }
|
||||
});
|
||||
|
||||
test("tools/list exposes the full frontier surface with valid schemas", async () => {
|
||||
const res = await rpc("tools/list", {});
|
||||
const tools = res.result.tools;
|
||||
assert.ok(tools.length >= 38, `${tools.length} tools`);
|
||||
for (const t of tools) {
|
||||
assert.ok(t.name && t.description && t.inputSchema, `schema incomplete for ${t.name}`);
|
||||
}
|
||||
const names = new Set(tools.map((t) => t.name));
|
||||
for (const required of ["screenshot", "zoom", "left_click", "double_click", "triple_click", "right_click", "middle_click",
|
||||
"mouse_move", "left_click_drag", "left_mouse_down", "left_mouse_up", "scroll", "type", "key", "hold_key",
|
||||
"set_value", "select_text", "perform_action", "get_app_state", "list_apps", "list_windows", "list_displays",
|
||||
"switch_display", "open_application", "read_clipboard", "write_clipboard", "cursor_position", "wait",
|
||||
"recording_start", "recording_stop", "recording_status", "recording_list",
|
||||
"computer_list", "computer_switch", "computer_register", "computer_remove", "request_access", "stop_computer_control"]) {
|
||||
assert.ok(names.has(required), `missing tool ${required}`);
|
||||
}
|
||||
});
|
||||
|
||||
test("computer registry round-trip over the protocol", async () => {
|
||||
let r = await tool("computer_list");
|
||||
assert.equal(r.ok, true);
|
||||
assert.equal(r.active, "local");
|
||||
r = await tool("computer_register", { computer: "pad", transport: "hdc" });
|
||||
assert.equal(r.registered.platform, "harmonyos");
|
||||
r = await tool("computer_switch", { computer: "pad" });
|
||||
assert.equal(r.active, "pad");
|
||||
r = await tool("computer_remove", { computer: "pad" });
|
||||
assert.equal(r.active, "local");
|
||||
});
|
||||
|
||||
test("registering an ssh computer installs the agent and probes the platform", async () => {
|
||||
const r = await tool("computer_register", { computer: "box", transport: "ssh", host: "box.test", user: "me" });
|
||||
assert.equal(r.ok, true, JSON.stringify(r.error ?? {}));
|
||||
assert.equal(r.agentInstall.remotePlatform, process.platform, "platform probed via agent");
|
||||
assert.ok(fs.existsSync(path.join(fakeHome, ".codewhale-cu", "agent", "agent.mjs")), "agent pushed");
|
||||
assert.ok(fs.existsSync(path.join(fakeHome, ".codewhale-cu", "agent", "src", "backends", "darwin.mjs")), "src tree pushed");
|
||||
// dispatch a real tool to the "remote" computer
|
||||
const apps = await tool("list_apps", { computer: "box" });
|
||||
assert.equal(apps.ok, true, JSON.stringify(apps.error ?? {}));
|
||||
assert.equal(apps.computer.id, "box");
|
||||
assert.ok(Array.isArray(apps.apps) && apps.apps.length > 0, "apps returned over the wire");
|
||||
});
|
||||
|
||||
test("unknown computer fails closed with a named error", async () => {
|
||||
const r = await tool("screenshot", { computer: "ghost" });
|
||||
assert.equal(r.ok, false);
|
||||
assert.equal(r.error.code, "unknown_computer");
|
||||
});
|
||||
|
||||
test("kill switch refuses mutating tools but keeps read-only probes", async () => {
|
||||
let r = await tool("stop_computer_control", { reason: "protocol-test" });
|
||||
assert.equal(r.stopped, true);
|
||||
r = await tool("screenshot");
|
||||
assert.equal(r.error.code, "control_stopped");
|
||||
r = await tool("computer_list");
|
||||
assert.equal(r.ok, true);
|
||||
});
|
||||
Reference in new issue
Block a user