mirror of
https://github.com/codewhale-hq/Codewhale.git
synced 2026-11-07 17:09:13 -10:00
fix(cu): mirror the Linux input-owner consent fix from upstream
Refresh the vendored Computer Use bundle to codewhale-cu-plugin@ef0cd8a1d69e (PR #15): keystrokes and pointer events follow the window that receives them and fail closed without an owner; Linux gains AT-SPI focus/get_value/ select_text, popup-open key refusal, settle before Return, app_ref.name windows and captures, server-owned ffmpeg x11grab recording, kill_app and list_sessions, and the refusal/doc/skill updates. Adds the input-owner and linux-desktop suites to the vendored tree; the three deliberate Core variants (package.json, README.md, tests/manifest.test.mjs) are unchanged. Evidence: node --test tests/*.test.mjs 470 tests (450 pass, 0 fail, 20 platform-gated skips); check-cu-sync reports marketplace == upstream @ ef0cd8a1d69e and the Core tree matching the marketplace runtime set. Only tests/ files are new and no runtime file name moved, so the builtin embed-list test and Rust sources are unaffected. No exhaustive Rust suite, installed app or release qualification claim.
This commit is contained in:
CodeWhale Bot committed
1 parent
6f30b07359
commit
2b0e251d9d
14 files changed
+1167
-73
No files matched your search
@@ -1 +1 @@
|
||||
724f9c422db1a51880dc81154dae70816c475ed8
|
||||
ef0cd8a1d69ec2456c63715d5a5bf48e7f470d66
|
||||
@@ -859,6 +859,51 @@ async function consentCheck(computer, name, args) {
|
||||
return grant ? { grant } : null;
|
||||
}
|
||||
|
||||
// ---------- input ownership ----------
|
||||
// Keystrokes and pointer events go to whichever window the OS routes them to,
|
||||
// which is not necessarily the app the agent last observed. A backend that
|
||||
// implements input_owner reports the app behind that window (keyboard: the
|
||||
// active window; pointer: the topmost window under the point), and the
|
||||
// consent ledger decides on that identity. Backends without input_owner keep
|
||||
// the bound-app and element checks above.
|
||||
const KEYBOARD_INPUT_TOOLS = new Set(["type", "key", "hold_key"]);
|
||||
const POINTER_INPUT_TOOLS = new Set(["left_click", "double_click", "triple_click", "right_click", "middle_click", "mouse_move", "scroll", "left_mouse_down"]);
|
||||
|
||||
/** Every ledger key that identifies this owner: its window class, its process name and its pid. */
|
||||
function ownerKeys(owner) {
|
||||
const keys = new Set();
|
||||
for (const name of [owner.name, owner.comm]) for (const key of consent.appKeys({ name })) keys.add(key);
|
||||
for (const key of consent.appKeys({ pid: owner.pid })) keys.add(key);
|
||||
return [...keys];
|
||||
}
|
||||
|
||||
async function inputOwnerCheck(computer, name, prepared, backend) {
|
||||
if (computer.transport !== "local" || computer.owned === true || typeof backend.input_owner !== "function") return;
|
||||
const checks = [];
|
||||
if (KEYBOARD_INPUT_TOOLS.has(name)) checks.push({ kind: "keyboard" });
|
||||
else if (name === "left_click_drag") checks.push({ kind: "pointer", at: prepared.from_target }, { kind: "pointer", at: prepared.to });
|
||||
else if (POINTER_INPUT_TOOLS.has(name)) checks.push({ kind: "pointer", at: prepared.target });
|
||||
else return;
|
||||
for (const check of checks) {
|
||||
const at = check.at && Number.isFinite(check.at.x) && Number.isFinite(check.at.y) ? { x: check.at.x, y: check.at.y } : undefined;
|
||||
const owner = await backend.input_owner({ kind: check.kind, ...(at ? { point: at } : {}) });
|
||||
if (!owner) continue; // the desktop itself: no application receives this input
|
||||
const verdict = consent.decisionFor(computer.id, ownerKeys(owner));
|
||||
const desc = owner.name ?? (owner.pid ? `pid ${owner.pid}` : "the application");
|
||||
const arg = owner.name ?? `pid:${owner.pid}`;
|
||||
if (verdict.state === "denied") {
|
||||
throw new ServerError("app_denied",
|
||||
`the user denied access to ${desc} on this computer — do not work around it; only they can change it (consent {action:"revoke"}).`,
|
||||
{ app: owner });
|
||||
}
|
||||
if (verdict.state === "undecided") {
|
||||
throw new ServerError("consent_required",
|
||||
`Codewhale needs the user's permission to use ${desc} on this computer — ask them, then record their answer with consent {action:"allow"|"deny", app:"${arg}"}. This ${check.kind} input would reach ${desc}, the app that owns the window receiving it.`,
|
||||
{ app: owner });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------- irreversible-action confirmation ----------
|
||||
// A click or press on a control labelled pay, buy, send, transfer, delete (and
|
||||
// their close relatives) moves money or destroys something, and the text that
|
||||
@@ -1166,9 +1211,15 @@ async function callTool(params, bindWhen = null) {
|
||||
}
|
||||
|
||||
if (name === "computer_switch") {
|
||||
const c = registry.get(args.computer);
|
||||
activeComputerId = c.id;
|
||||
return { content: [{ type: "text", text: JSON.stringify(receipt(c, { ok: true, active: c.id })) }] };
|
||||
// An unknown id is the caller's mistake: report it as a tool failure with
|
||||
// the registry's code, not as a protocol error.
|
||||
try {
|
||||
const c = registry.get(args.computer);
|
||||
activeComputerId = c.id;
|
||||
return { content: [{ type: "text", text: JSON.stringify(receipt(c, { ok: true, active: c.id })) }] };
|
||||
} catch (err) {
|
||||
return { content: [{ type: "text", text: JSON.stringify(fail(null, err.code ?? "computer_error", err.message ?? String(err), { tool: name })) }], isError: true };
|
||||
}
|
||||
}
|
||||
|
||||
// Everything below acts on a computer.
|
||||
@@ -1393,6 +1444,7 @@ async function callTool(params, bindWhen = null) {
|
||||
}
|
||||
const resolve = typeof backend.resolve_element === "function" ? (req) => backend.resolve_element(req) : null;
|
||||
const prepared = await prepareArgs(computer, name, args, resolve, sink);
|
||||
await inputOwnerCheck(computer, name, prepared, backend);
|
||||
throwIfAborted();
|
||||
await assertCurrentRoute(computer, binding);
|
||||
if (controlStopped && !READ_ONLY_TOOLS.has(name)) throw new ServerError("control_stopped", "stop_computer_control is active; no further actions are permitted this session");
|
||||
|
||||
@@ -10,12 +10,17 @@ import url from "node:url";
|
||||
const __dirname = path.dirname(url.fileURLToPath(import.meta.url));
|
||||
const ROOT = path.resolve(__dirname, "..");
|
||||
const results = [];
|
||||
const skipped = [];
|
||||
|
||||
function log(name, pass, detail = "") {
|
||||
results.push({ name, pass, detail: String(detail).slice(0, 500) });
|
||||
console.log(`${pass ? "PASS" : "FAIL"} ${name}${detail ? ` — ${detail}` : ""}`);
|
||||
}
|
||||
|
||||
// Optional: the one app whose consent this run may exercise (for example
|
||||
// CU_SMOKE_APP=mousepad). Unset, the consent-gated checks are reported as
|
||||
// skipped rather than faked.
|
||||
const smokeApp = process.env.CU_SMOKE_APP?.trim() || null;
|
||||
const stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "cu-smoke-state-"));
|
||||
const recDir = fs.mkdtempSync(path.join(os.tmpdir(), "cu-smoke-rec-"));
|
||||
|
||||
@@ -36,6 +41,14 @@ server.stdout.on("data", (d) => {
|
||||
if (!line) continue;
|
||||
try {
|
||||
const msg = JSON.parse(line);
|
||||
if (msg.method === "elicitation/create" && msg.id) {
|
||||
// The consent decision the operator asked for: only the app named in
|
||||
// CU_SMOKE_APP, recorded for this session. Anything else is declined.
|
||||
const approved = smokeApp && msg.params?.message === `Computer Use asks for your decision: consent_allow {"app":"${smokeApp}"}`;
|
||||
console.log(`[smoke] consent prompt ${approved ? "approved" : "declined"}: ${msg.params?.message}`);
|
||||
server.stdin.write(JSON.stringify({ jsonrpc: "2.0", id: msg.id, result: { action: approved ? "accept" : "decline" } }) + "\n");
|
||||
continue;
|
||||
}
|
||||
if (msg.id && pending.has(msg.id)) {
|
||||
pending.get(msg.id)(msg);
|
||||
pending.delete(msg.id);
|
||||
@@ -64,7 +77,7 @@ async function tool(name, args = {}) {
|
||||
|
||||
try {
|
||||
// --- protocol ---
|
||||
const init = await rpc("initialize", { protocolVersion: "2025-06-18", capabilities: {} });
|
||||
const init = await rpc("initialize", { protocolVersion: "2025-06-18", capabilities: { elicitation: {} } });
|
||||
log("initialize", !!init.result?.serverInfo?.name, `server=${init.result?.serverInfo?.name} v${init.result?.serverInfo?.version}`);
|
||||
await rpc("notifications/initialized", undefined, 5_000).catch(() => {});
|
||||
const tl = await rpc("tools/list", {});
|
||||
@@ -82,17 +95,25 @@ try {
|
||||
let r = await tool("computer_list");
|
||||
log("computer_list", r.parsed?.ok === true && r.parsed?.computers?.length >= 1, `active=${r.parsed?.active}`);
|
||||
|
||||
// Registering a computer is the user's own decision: a model call is refused.
|
||||
r = await tool("computer_register", { computer: "pad", transport: "hdc", label: "Harmony device" });
|
||||
log("computer_register(hdc)", r.parsed?.ok === true, JSON.stringify(r.parsed?.registered ?? r.parsed?.error));
|
||||
// consent_declined: this client answered the registration prompt with a
|
||||
// decline. consent_needs_user: the host offered no prompt at all. Either way
|
||||
// the registration did not happen.
|
||||
log("computer_register refused without the user's decision", ["consent_declined", "consent_needs_user"].includes(r.parsed?.error?.code), `code=${r.parsed?.error?.code}`);
|
||||
|
||||
// An unregistered computer must fail closed, both to switch and to act.
|
||||
r = await tool("computer_switch", { computer: "pad" });
|
||||
log("computer_switch", r.parsed?.ok === true && r.parsed?.active === "pad", `active=${r.parsed?.active}`);
|
||||
const listed = await tool("computer_list");
|
||||
log("computer_switch to an unregistered computer fails closed", r.parsed?.ok === false && r.parsed?.error?.code === "unknown_computer" && listed.parsed?.active === "local", `code=${r.parsed?.error?.code} active=${listed.parsed?.active}`);
|
||||
|
||||
r = await tool("list_apps");
|
||||
log("harmony fail-closed (no hdc device)", r.parsed?.ok === false, `code=${r.parsed?.error?.code}`);
|
||||
r = await tool("list_apps", { computer: "pad" });
|
||||
log("harmony fail-closed (no registered device)", r.parsed?.ok === false && r.parsed?.computer?.id !== "pad", `code=${r.parsed?.error?.code}`);
|
||||
|
||||
// Switching by use is sticky, but a screenshot on the already-active computer
|
||||
// reports no switch. The receipt must name the computer it acted on.
|
||||
r = await tool("screenshot", { computer: "local" });
|
||||
log("switch-by-use (computer:local on screenshot)", r.parsed?.ok === true && r.parsed?.switched === true && r.parsed?.computer?.id === "local", `file=${path.basename(r.parsed?.file ?? "")}`);
|
||||
log("screenshot names its computer (computer:local)", r.parsed?.ok === true && r.parsed?.computer?.id === "local", `file=${path.basename(r.parsed?.file ?? "")} switched=${r.parsed?.switched}`);
|
||||
|
||||
// --- local darwin live tools ---
|
||||
r = await tool("request_access");
|
||||
@@ -104,22 +125,31 @@ try {
|
||||
r = await tool("list_apps");
|
||||
log("list_apps", r.parsed?.ok === true && r.parsed?.apps?.length > 0, `${r.parsed?.apps?.length} apps`);
|
||||
|
||||
// No backend reports windowCount today, so fall back to the app in front —
|
||||
// it is the one guaranteed to have a window worth observing.
|
||||
// Consent is exercised only on the app the operator named (CU_SMOKE_APP).
|
||||
// Without it, these checks are listed as skipped, never reported as passed.
|
||||
const apps = r.parsed?.apps ?? [];
|
||||
const someApp = apps.find((a) => a.windowCount > 0) ?? apps.find((a) => a.frontmost) ?? apps[0];
|
||||
if (someApp) {
|
||||
// The consent ledger gates first app contact on the local computer —
|
||||
// smoke exercises the real flow: refuse, record the user's allow, retry.
|
||||
const gated = await tool("get_app_state", { app_ref: { pid: someApp.pid } });
|
||||
log("consent_required on first app contact", gated.parsed?.error?.code === "consent_required", `code=${gated.parsed?.error?.code}`);
|
||||
const c = await tool("consent", { action: "allow", app: `pid:${someApp.pid}` });
|
||||
log("consent allow", c.parsed?.ok === true, `${someApp.name}: keys=${JSON.stringify(c.parsed?.keys ?? c.parsed?.error)}`);
|
||||
const st = await tool("get_app_state", { app_ref: { pid: someApp.pid } });
|
||||
log("get_app_state", st.parsed?.ok === true && st.parsed?.elements?.length > 0, `${someApp.name}: ${st.parsed?.elements?.length} elements, state_id=${st.parsed?.state_id}`);
|
||||
globalThis.__state = st.parsed;
|
||||
const target = smokeApp ? apps.find((a) => (a.name ?? "").toLowerCase() === smokeApp.toLowerCase()) : null;
|
||||
if (!smokeApp) {
|
||||
skipped.push("consent-gated app checks (set CU_SMOKE_APP to the one app this run may drive)");
|
||||
} else if (!target) {
|
||||
log("consent target running", false, `no running app named "${smokeApp}" — start it and rerun`);
|
||||
} else {
|
||||
log("get_app_state", false, "list_apps returned no applications to observe");
|
||||
const ref = { name: target.name };
|
||||
// First contact refuses until the user decides; the decision is recorded
|
||||
// through elicitation, so the operator approves the exact call.
|
||||
const gated = await tool("get_app_state", { app_ref: ref });
|
||||
log("consent_required on first app contact", gated.parsed?.error?.code === "consent_required", `code=${gated.parsed?.error?.code}`);
|
||||
const c = await tool("consent", { action: "allow", app: target.name });
|
||||
log("consent allow (approved by the operator via elicitation)", c.parsed?.ok === true, `${target.name}: ${JSON.stringify(c.parsed?.keys ?? c.parsed?.error?.code)}`);
|
||||
const st = await tool("get_app_state", { app_ref: ref });
|
||||
log("get_app_state after consent", st.parsed?.ok === true && st.parsed?.elements?.length > 0, `${target.name}: ${st.parsed?.elements?.length} elements`);
|
||||
const other = apps.find((a) => a.name && a.name.toLowerCase() !== smokeApp.toLowerCase());
|
||||
if (other) {
|
||||
const blocked = await tool("get_app_state", { app_ref: { name: other.name } });
|
||||
log("unapproved app stays blocked", blocked.parsed?.error?.code === "consent_required", `${other.name}: code=${blocked.parsed?.error?.code}`);
|
||||
} else {
|
||||
skipped.push("unapproved-app block (no second running app to test against)");
|
||||
}
|
||||
}
|
||||
|
||||
r = await tool("cursor_position");
|
||||
@@ -168,12 +198,13 @@ try {
|
||||
log("smoke-run", false, err.stack ?? err.message);
|
||||
} finally {
|
||||
server.kill("SIGTERM");
|
||||
const receipt = { at: new Date().toISOString(), host: `${process.platform} ${os.release()}`, results };
|
||||
const receipt = { at: new Date().toISOString(), host: `${process.platform} ${os.release()}`, results, skipped };
|
||||
const outDir = path.join(ROOT, "receipts");
|
||||
fs.mkdirSync(outDir, { recursive: true });
|
||||
const out = path.join(outDir, `smoke-${new Date().toISOString().replace(/[:.]/g, "-")}.json`);
|
||||
fs.writeFileSync(out, JSON.stringify(receipt, null, 2));
|
||||
const failed = results.filter((r) => !r.pass);
|
||||
for (const item of skipped) console.log(`SKIP ${item}`);
|
||||
console.log(`\n${results.length - failed.length}/${results.length} passed. Receipt: ${out}`);
|
||||
try { fs.rmSync(stateDir, { recursive: true, force: true }); } catch {}
|
||||
process.exit(failed.length ? 1 : 0);
|
||||
|
||||
@@ -17,7 +17,7 @@ Prefer the host's files, shell and app connectors for work that does not need a
|
||||
|
||||
1. `list_apps` finds targets. If a named app is absent, try `open_application` once with the user's exact spelling, then report the result; do not guess names. Pass `computer` explicitly when switching machines. Read each receipt's computer identity.
|
||||
2. Start with `get_app_state` for text, controls, element indices and `state_id`. Use `query`, `role`, `limit` or `find_elements` to narrow. Repeat the same view with `since:state_id` (or `"latest"`) for changed rows; `resync_required:true` supplies a new full baseline. `removed_indices` left that view. Missing labels are unknown; do not guess them. Re-observe shallow/loading trees.
|
||||
3. Prefer element targets, `set_value`, or `focus` then `type`/`key`. Newlines and `press_enter` send Return. `wait_for` handles UI transitions. Re-observe after stale targets or a timeout; never replay an action whose receipt says `action_sent:true`.
|
||||
3. Prefer element targets, `set_value`, or `focus` then `type`/`key`. On Linux, keys go to the active window and are refused while a menu is open; observe before typing, and send `key escape` to close a menu. Newlines and `press_enter` send Return. `wait_for` handles UI transitions. Re-observe after stale targets or a timeout; never replay an action whose receipt says `action_sent:true`.
|
||||
4. Use screenshots/zoom only when the tree cannot answer the task. Coordinates are pixels in the returned raster unless explicitly `space:"screen"`. Pass its `raster_id` in every raster coordinate target and as the parent of `zoom`; use the zoom's new ID for child-image points. OCR targets already carry it. `raster_stale` means observe again; never drop the ID to retry. A pin detects capture replacement, not a changed UI. Never calculate from a file path, omitted image, stale raster or invented state. Text-only models may use macOS OCR, not infer graphical meaning.
|
||||
5. Verify with fresh state, field readback or an observed task result. A sent action is not proof of success. When typing says `verified:false`, inspect the requested screenshot before relying on it.
|
||||
|
||||
|
||||
@@ -53,7 +53,7 @@ a raster ID. Browser viewport points use the browser's separate contract.
|
||||
## Apps & computers
|
||||
- `open_application {name|bundle_id|pid, activate?}` — bind the input target; `app_not_found` when the selector resolves nowhere.
|
||||
- `list_apps {installed:true}` — the installed catalog (openable apps, running or not, one subdirectory deep) instead of the running list.
|
||||
- `kill_app {name|bundle_id|pid, force?}` — quit an app; refuses an ambiguous name match (pass pid); never the helper itself.
|
||||
- `kill_app {name|bundle_id|pid, force?}` — quit an app; refuses an ambiguous name match (pass pid); never the helper itself. Linux: closes the window first; `still_running` means the app refused to quit; `force:true` sends SIGKILL and discards unsaved work.
|
||||
- `set_window_frame {app_ref?, window_id, frame:{x,y,w,h}}` — move/resize one window; readbacks report what the app actually did (`verified`, `ax_errors`).
|
||||
- `preview {enabled}` — floating panel: captured window + agent/user cursors; live while bound.
|
||||
- `computer {action, id?}` — list / switch / register / **spawn** / remove.
|
||||
@@ -61,7 +61,7 @@ a raster ID. Browser viewport points use the browser's separate contract.
|
||||
Linux desktop (registered `owned:true`, becomes active) — the default
|
||||
workspace for anything that does not need the user's own session. `remove`
|
||||
or session end destroys it. `local` stays for work in the user's session.
|
||||
- `recording {action, …}` — start / stop / status / list (opt-in screen recordings).
|
||||
- `recording {action, …}` — start / stop / status / list (opt-in screen recordings: macOS ScreenCaptureKit, Linux X11 ffmpeg; Windows unavailable).
|
||||
|
||||
## Browser (CDP)
|
||||
- `browser {action:"start", url?}` — self-owned Chromium profile + this session's tab; the user's own browser is never touched.
|
||||
|
||||
@@ -58,6 +58,15 @@ Never retry a refusal unchanged — re-observe, re-target, or change route.
|
||||
| `frame_refused` | the app refused both the position and the size write | the window is fullscreen, tiled or otherwise not movable by the app |
|
||||
| `trajectory_not_found` | no trajectory file matches the id (or none exist) | `trajectory {action:"status"}` lists recent files |
|
||||
| `replay_too_large` | the trajectory exceeds the 200-turn replay cap | split it, or replay a pruned copy |
|
||||
| `input_owner_unknown` | Linux cannot tell which app would receive this input: no window manager publishes its stacking order, no managed window has keyboard focus, or the session is Wayland | bring the target app's window to the front (observe it or click it), then retry; on Wayland use an X11 session |
|
||||
| `popup_open` | a menu or popup item is open in the app that would receive the keys | send `key escape` (allowed while a menu is open), observe the app, then retry |
|
||||
| `window_not_active` | the element is focused, but its window is not the active one, so keys would go to another window | bring the app forward (`open_application activate:true`, which needs foreground consent) or ask the user to focus it |
|
||||
| `no_window` | the app has no visible window to capture (minimized or unmapped) | restore it, or ask the user; capture the display instead |
|
||||
| `no_window_manager` | window geometry is not published, so app-window screenshots and recordings cannot be cropped | start a window manager, or capture the display |
|
||||
| `focus_failed` | AT-SPI refused focus on the element (not focusable, or focus refused) | observe again and choose a focusable control |
|
||||
| `select_text_failed` | AT-SPI refused the text range or the caret position | observe the element; the range must lie inside its text |
|
||||
| `no_value` | the element exposes neither a text nor a numeric value | read a labelled control or a different element |
|
||||
| `recording_failed` | ffmpeg could not start or finish the recording | check the display and the ffmpeg install; the receipt names the partial file |
|
||||
| `app_upgrade_required` | the helper predates the feature or is not running | restart/update the Codewhale Computer Use app |
|
||||
| `unsupported_on_backend` | tool not implemented on that platform backend | check the platform note in the main skill |
|
||||
| `unsupported_on_transport` | `app_script` sent to an ssh/docker/hdc computer — scripting is local-only so a remote channel never becomes a shell | run it on `local`, or use the host's own remote access |
|
||||
|
||||
@@ -44,8 +44,14 @@ function appName(ref) {
|
||||
return ref.name;
|
||||
}
|
||||
|
||||
/** Linux selects windows by exact app name (app_ref.name); numeric window ids are not exposed. */
|
||||
function rejectWindowSelectors(args) {
|
||||
if (Object.hasOwn(args, "app_ref") || Object.hasOwn(args, "window_id")) throw Object.assign(new ExecError("Linux window listing and screenshots do not support app_ref or window_id selectors"), { code: "unsupported_selector" });
|
||||
if (Object.hasOwn(args, "window_id")) throw Object.assign(new ExecError("Linux window selection is by app_ref.name only; window_id is unavailable"), { code: "unsupported_selector" });
|
||||
}
|
||||
|
||||
/** Exact, case-insensitive app-name match used everywhere a name is compared. */
|
||||
function sameName(a, b) {
|
||||
return typeof a === "string" && typeof b === "string" && a.trim().length > 0 && a.trim().replace(/\.app$/i, "").toLowerCase() === b.trim().replace(/\.app$/i, "").toLowerCase();
|
||||
}
|
||||
|
||||
function assertAppRootWindow(index, windowId) {
|
||||
@@ -57,6 +63,25 @@ function outputPath(file) {
|
||||
return file;
|
||||
}
|
||||
|
||||
/**
|
||||
* ffmpeg arguments for an x11grab H.264 recording of one rectangle. H.264 with
|
||||
* yuv420p needs even dimensions, so the rectangle is trimmed to even sizes.
|
||||
* stdin stays open: 'q' on it is how the recorder finishes the file.
|
||||
*/
|
||||
export function x11GrabArgs({ display, rect, fps, file, durationSec = null }) {
|
||||
const w = rect[2] - (rect[2] % 2);
|
||||
const h = rect[3] - (rect[3] % 2);
|
||||
if (w < 2 || h < 2) throw Object.assign(new ExecError("the recording area must be at least 2x2 pixels"), { code: "bad_args" });
|
||||
return [
|
||||
"-y", "-loglevel", "error",
|
||||
"-f", "x11grab", "-framerate", String(fps), "-video_size", `${w}x${h}`,
|
||||
"-i", `${display}+${rect[0]},${rect[1]}`,
|
||||
...(durationSec ? ["-t", String(durationSec)] : []),
|
||||
"-c:v", "libx264", "-preset", "veryfast", "-crf", "23", "-pix_fmt", "yuv420p", "-movflags", "+faststart",
|
||||
file,
|
||||
];
|
||||
}
|
||||
|
||||
export function create({ exec } = {}) {
|
||||
const run = exec?.run ?? nativeRun;
|
||||
const have = exec?.have ?? nativeHave;
|
||||
@@ -73,6 +98,13 @@ export function create({ exec } = {}) {
|
||||
let lastRaster = null;
|
||||
let mouseHeld = false;
|
||||
const heldKeys = new Set();
|
||||
let lastTypedAt = 0;
|
||||
const recordings = new Map(); // id -> { child, file, rect, fps, startedAt, stderr }
|
||||
|
||||
async function settleAfterTyping() {
|
||||
const since = Date.now() - lastTypedAt;
|
||||
if (lastTypedAt && since < 300) await wait(300 - since);
|
||||
}
|
||||
|
||||
async function releaseMouse() {
|
||||
if (!mouseHeld) return;
|
||||
@@ -115,6 +147,161 @@ export function create({ exec } = {}) {
|
||||
if (!tools[tool]) throw new ExecError(`linux backend needs "${tool}" for ${purpose} — install it and retry`);
|
||||
}
|
||||
|
||||
// ---------- X11 windows: stacking, frames and owner identity ----------
|
||||
// Input lands on the active window (keys) or the topmost window under the
|
||||
// point (pointer). Consent must be checked against that window's app, not
|
||||
// against whichever app the agent last observed. The window manager
|
||||
// publishes both facts through EWMH; without it we cannot attribute input,
|
||||
// so callers fail closed.
|
||||
async function softRun(cmd, args) {
|
||||
const r = await run(cmd, args, { timeoutMs: 5_000 });
|
||||
if (r.aborted) throwIfAborted();
|
||||
return r.code === 0 ? r.stdout : "";
|
||||
}
|
||||
|
||||
function normWindowId(value) {
|
||||
const n = Number(String(value ?? "").trim());
|
||||
return Number.isFinite(n) && n > 0 ? `0x${n.toString(16).padStart(8, "0")}` : null;
|
||||
}
|
||||
|
||||
function procComm(pid) {
|
||||
if (!Number.isInteger(pid) || pid <= 0) return null;
|
||||
try { return fs.readFileSync(`/proc/${pid}/comm`, "utf8").trim() || null; } catch { return null; }
|
||||
}
|
||||
|
||||
function parseFrameExtents(text) {
|
||||
const m = /_NET_FRAME_EXTENTS\(CARDINAL\) = ([\d,\s-]+)/.exec(text);
|
||||
const v = m ? m[1].split(",").map((s) => Number(s.trim()) || 0) : [0, 0, 0, 0];
|
||||
return { left: v[0], right: v[1], top: v[2], bottom: v[3] };
|
||||
}
|
||||
|
||||
/** Client rectangle and mapping state from `xwininfo -id`; null when the window vanished. */
|
||||
function parseXwininfo(text) {
|
||||
const x = /Absolute upper-left X:\s+(-?\d+)/.exec(text);
|
||||
const y = /Absolute upper-left Y:\s+(-?\d+)/.exec(text);
|
||||
const w = /Width:\s+(\d+)/.exec(text);
|
||||
const h = /Height:\s+(\d+)/.exec(text);
|
||||
if (!x || !y || !w || !h) return null;
|
||||
const map = /Map State:\s+(\S+)/.exec(text);
|
||||
return { x: Number(x[1]), y: Number(y[1]), w: Number(w[1]), h: Number(h[1]), viewable: map ? map[1] === "IsViewable" : true };
|
||||
}
|
||||
|
||||
/** Every managed window with its owner (wmctrl -lpx), in no particular order. */
|
||||
async function wmctrlWindows() {
|
||||
const out = [];
|
||||
for (const line of (await softRun("wmctrl", ["-lpx"])).split("\n")) {
|
||||
const m = /^(0x[0-9a-f]+)\s+(-?\d+)\s+(\d+)\s+(\S+)\s+\S+\s+(.*)$/i.exec(line.trim());
|
||||
if (!m) continue;
|
||||
const wm_class = m[4];
|
||||
out.push({ id: normWindowId(m[1]), pid: Number(m[3]), wm_class, name: wm_class.split(".")[0] || wm_class, title: m[5] });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Managed windows bottom to top: id, owner (wm class, process, pid), client and frame rectangles. Null without a window manager. */
|
||||
async function x11Windows() {
|
||||
const stack = await softRun("xprop", ["-root", "_NET_CLIENT_LIST_STACKING"]);
|
||||
if (!/_NET_CLIENT_LIST_STACKING\(WINDOW\)/.test(stack)) return null;
|
||||
const order = [...stack.matchAll(/0x[0-9a-f]+/gi)].map((m) => normWindowId(m[0])).filter(Boolean);
|
||||
const listed = new Map((await wmctrlWindows()).map((w) => [w.id, w]));
|
||||
const windows = [];
|
||||
for (const id of order) {
|
||||
const meta = listed.get(id);
|
||||
if (!meta) continue;
|
||||
const client = parseXwininfo(await softRun("xwininfo", ["-id", id]));
|
||||
const ext = parseFrameExtents(await softRun("xprop", ["-id", id, "_NET_FRAME_EXTENTS"]));
|
||||
const frame = client ? {
|
||||
x: client.x - ext.left, y: client.y - ext.top,
|
||||
w: client.w + ext.left + ext.right, h: client.h + ext.top + ext.bottom,
|
||||
} : null;
|
||||
windows.push({
|
||||
id, ...meta, name: meta.wm_class.split(".")[0] || meta.wm_class,
|
||||
comm: procComm(meta.pid), viewable: client?.viewable ?? false, frame,
|
||||
});
|
||||
}
|
||||
return windows;
|
||||
}
|
||||
|
||||
function identityOf(win) {
|
||||
return { name: win.name, comm: win.comm, pid: win.pid, wm_class: win.wm_class, title: win.title, window: win.id };
|
||||
}
|
||||
|
||||
const ownerUnknown = (message) => Object.assign(new ExecError(message), { code: "input_owner_unknown" });
|
||||
|
||||
/** The app whose window has keyboard focus. Fails closed: never guesses. */
|
||||
async function keyboardOwner() {
|
||||
await probeSession();
|
||||
if (session !== "x11") throw ownerUnknown("Linux Wayland does not expose the focused window to Codewhale, so keyboard input is refused rather than sent to an unknown window; use an X11 session");
|
||||
const windows = await x11Windows();
|
||||
if (!windows) throw ownerUnknown("the window manager does not publish its stacking order, so Codewhale cannot tell which app would receive keys");
|
||||
const active = normWindowId((await softRun("xdotool", ["getactivewindow"])).trim());
|
||||
const win = active ? windows.find((w) => w.id === active) : null;
|
||||
if (!win) throw ownerUnknown("no managed application window has keyboard focus; click the target app's window first, then retry");
|
||||
return identityOf(win);
|
||||
}
|
||||
|
||||
/** The app whose window is topmost under a screen point, or null when the point is on the desktop. */
|
||||
async function pointOwner(x, y) {
|
||||
await probeSession();
|
||||
if (session !== "x11") throw ownerUnknown("Linux Wayland does not expose pointer ownership to Codewhale, so pointer input is refused; use an X11 session");
|
||||
const windows = await x11Windows();
|
||||
if (!windows) throw ownerUnknown("the window manager does not publish its stacking order, so Codewhale cannot tell which app a pointer target belongs to");
|
||||
for (let i = windows.length - 1; i >= 0; i--) {
|
||||
const win = windows[i];
|
||||
if (!win.viewable) continue;
|
||||
if (!win.frame) throw ownerUnknown(`the geometry of window "${win.title}" could not be read, so this pointer target cannot be attributed`);
|
||||
const f = win.frame;
|
||||
if (x >= f.x && y >= f.y && x < f.x + f.w && y < f.y + f.h) return identityOf(win);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function cursorPoint() {
|
||||
await probeSession();
|
||||
const out = await xdotool(["getmouselocation"]);
|
||||
const m = /x:(-?\d+)\s+y:(-?\d+)/.exec(out);
|
||||
if (!m) throw ownerUnknown(`could not read the pointer location (${out.slice(0, 80)})`);
|
||||
return { x: Number(m[1]), y: Number(m[2]) };
|
||||
}
|
||||
|
||||
/** Geometry of an app's topmost viewable window, plus the windows stacked above it that overlap it. */
|
||||
async function appFrame(name) {
|
||||
const windows = await x11Windows();
|
||||
if (!windows) throw Object.assign(new ExecError("the window manager does not publish window geometry; start one, or capture the whole display"), { code: "no_window_manager" });
|
||||
const owned = windows.filter((w) => sameName(w.name, name) || sameName(w.comm, name));
|
||||
if (!owned.length) throw Object.assign(new ExecError(`no window belongs to "${name}" — call list_apps and check the name`), { code: "app_not_found" });
|
||||
const win = [...owned].reverse().find((w) => w.viewable && w.frame);
|
||||
if (!win) throw Object.assign(new ExecError(`"${name}" has no visible window to capture (it may be minimized)`), { code: "no_window" });
|
||||
const top = windows.indexOf(win);
|
||||
const f = win.frame;
|
||||
const occludedBy = windows.slice(top + 1)
|
||||
.filter((w) => w.viewable && w.frame && w.frame.x < f.x + f.w && f.x < w.frame.x + w.frame.w && w.frame.y < f.y + f.h && f.y < w.frame.y + w.frame.h)
|
||||
.map((w) => ({ name: w.name, title: w.title }));
|
||||
return { win, rect: [f.x, f.y, f.w, f.h], occludedBy };
|
||||
}
|
||||
|
||||
/** Refuse keystrokes when an open menu would take them, or when the element's window is not the active one. */
|
||||
async function assertKeyboardReady({ target, allowEscape = false } = {}) {
|
||||
if (session !== "x11") return;
|
||||
if (target?.app_ref?.name) {
|
||||
const owner = await keyboardOwner();
|
||||
if (!sameName(owner.name, target.app_ref.name) && !sameName(owner.comm, target.app_ref.name)) {
|
||||
throw Object.assign(new ExecError(`the keystrokes would go to "${owner.name}", not "${target.app_ref.name}" — the element is focused but its window is not the active one. Bring the app forward (open_application activate:true needs foreground consent) or ask the user to focus it, then retry.`), { code: "window_not_active" });
|
||||
}
|
||||
}
|
||||
if (allowEscape || !tools.pyatspi) return;
|
||||
// Without an owner the server already refused this keystroke, so there
|
||||
// is no menu to probe for.
|
||||
const owner = await keyboardOwner().catch(() => null);
|
||||
if (!owner) return;
|
||||
const r = await run("python3", ["-c", PYATSPI_POPUP, owner.name], { timeoutMs: 10_000 });
|
||||
throwIfAborted();
|
||||
const out = tryJson(r.stdout.trim().split("\n").pop() ?? "", null);
|
||||
if (out?.popup) {
|
||||
throw Object.assign(new ExecError(`a ${out.popup.role}${out.popup.label ? ` "${out.popup.label}"` : ""} is open in ${owner.name}, and keystrokes would go into it. Send key Escape (allowed while a menu is open), observe the app, then retry.`), { code: "popup_open" });
|
||||
}
|
||||
}
|
||||
|
||||
async function shotTool() {
|
||||
if (session === "wayland") { need("grim", "screenshots on Wayland"); return { cmd: "grim", base: [] }; }
|
||||
if (session === "x11") {
|
||||
@@ -212,6 +399,46 @@ export function create({ exec } = {}) {
|
||||
return matches[0] if len(matches) == 1 else None
|
||||
return next((app for app in apps if app and app.childCount), None)
|
||||
`;
|
||||
// The popup probe looks for menu items and popup menus that are showing.
|
||||
// Menu-bar entries stay "showing" while closed, so only items count.
|
||||
const PYATSPI_POPUP = `import json, sys, pyatspi
|
||||
${PYATSPI_APP}
|
||||
desktop = pyatspi.Registry.getDesktop(0)
|
||||
root = resolve_app(desktop, sys.argv[1] if len(sys.argv) > 1 else None)
|
||||
if root is None:
|
||||
print(json.dumps({"found": False, "popup": None}))
|
||||
sys.exit(0)
|
||||
popup = None
|
||||
seen = 0
|
||||
queue = [(root, 0)]
|
||||
while queue and popup is None and seen < 1000:
|
||||
node, depth = queue.pop(0)
|
||||
if depth > 8:
|
||||
continue
|
||||
try:
|
||||
count = node.childCount
|
||||
except Exception:
|
||||
continue
|
||||
for i in range(count):
|
||||
try:
|
||||
child = node.getChildAtIndex(i)
|
||||
except Exception:
|
||||
continue
|
||||
if child is None:
|
||||
continue
|
||||
seen += 1
|
||||
role = child.getRoleName() or ""
|
||||
if role in ("menu item", "popup menu"):
|
||||
try:
|
||||
showing = child.getState().contains(pyatspi.STATE_SHOWING)
|
||||
except Exception:
|
||||
showing = False
|
||||
if showing:
|
||||
popup = {"role": role, "label": (child.name or "").strip() or None}
|
||||
break
|
||||
queue.append((child, depth + 1))
|
||||
print(json.dumps({"found": True, "popup": popup, "visited": seen}))`;
|
||||
|
||||
const PYATSPI_WALK = `import json, sys, pyatspi
|
||||
${PYATSPI_APP}
|
||||
app_name = sys.argv[1] if len(sys.argv) > 1 else None
|
||||
@@ -326,13 +553,17 @@ except Exception as e:
|
||||
browser_type: browser.type,
|
||||
browser_screenshot: browser.screenshot,
|
||||
browser_stop: browser.stop,
|
||||
closeSession: async () => { await browser.close().catch(() => {}); },
|
||||
closeSession: async () => {
|
||||
await Promise.all([...recordings.values()].map((entry) => stopRecording(entry, 3_000)));
|
||||
recordings.clear();
|
||||
await browser.close().catch(() => {});
|
||||
},
|
||||
probe: async () => {
|
||||
const s = await probeSession();
|
||||
const caps = {
|
||||
screenshot: !!((session === "wayland" && tools.grim) || (session === "x11" && (tools.scrot || tools.import))),
|
||||
clipboard: !!(tools.xclip || tools.xsel || (tools["wl-copy"] && tools["wl-paste"])),
|
||||
recording: false,
|
||||
recording: session === "x11" && !!tools.ffmpeg,
|
||||
accessibility_tree: tools.pyatspi,
|
||||
held_input: exec?.persistentInputOwner === true && !!(session === "x11" ? tools.xdotool : tools.wtype && tools.ydotool),
|
||||
};
|
||||
@@ -342,7 +573,7 @@ except Exception as e:
|
||||
if (session === "wayland" && !tools.ydotool) missing.push("ydotool+ydotoold (mouse input)");
|
||||
if (session === "wayland" && !tools.grim) missing.push("grim (screenshots)");
|
||||
if (session === "x11" && !tools.scrot && !tools.import) missing.push("scrot or imagemagick (screenshots)");
|
||||
missing.push("session-owned recording (unavailable in this version; use screenshots)");
|
||||
if (!(session === "x11" && tools.ffmpeg)) missing.push("ffmpeg with x11grab (screen recording; X11 only)");
|
||||
if (!tools.pyatspi) missing.push("python3-pyatspi (accessibility tree)");
|
||||
// Real permission probes, not just `have()`: each check is bounded to 10s.
|
||||
const permissions = { input: "failed", screen_capture: "failed", accessibility: "unavailable" };
|
||||
@@ -406,12 +637,14 @@ except Exception as e:
|
||||
list_apps: async () => {
|
||||
await probeSession();
|
||||
if (session === "x11" && tools.wmctrl) {
|
||||
const r = await runOk("wmctrl", ["-lx"], { timeoutMs: 15_000 });
|
||||
// One entry per process: the pid is what lets a consent decision or
|
||||
// kill_app name exactly one of several instances of the same program.
|
||||
const seen = new Map();
|
||||
for (const line of r.stdout.split("\n")) {
|
||||
const parts = line.split(/\s+/);
|
||||
const wmClass = parts[2];
|
||||
if (wmClass) seen.set(wmClass, { name: wmClass.split(".")[0], wm_class: wmClass });
|
||||
for (const w of await wmctrlWindows()) {
|
||||
const key = `${w.pid}|${w.wm_class}`;
|
||||
const entry = seen.get(key) ?? { name: w.name, wm_class: w.wm_class, pid: w.pid, process: procComm(w.pid), windows: 0 };
|
||||
entry.windows++;
|
||||
seen.set(key, entry);
|
||||
}
|
||||
return { apps: [...seen.values()] };
|
||||
}
|
||||
@@ -428,14 +661,18 @@ except Exception as e:
|
||||
},
|
||||
list_windows: async (args = {}) => {
|
||||
rejectWindowSelectors(args);
|
||||
const app = Object.hasOwn(args, "app_ref") ? appName(args.app_ref) : null;
|
||||
await probeSession();
|
||||
if (session === "x11" && tools.wmctrl) {
|
||||
const r = await runOk("wmctrl", ["-lGx"], { timeoutMs: 15_000 });
|
||||
const windows = [];
|
||||
for (const line of r.stdout.split("\n")) {
|
||||
const m = /^(\S+)\s+(-?\d+)\s+(-?\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(.*)$/.exec(line);
|
||||
if (m) windows.push({ id: m[1], desktop: m[2], position: { x: Number(m[3]), y: Number(m[4]) }, size: { w: Number(m[5]), h: Number(m[6]) }, wm_class: m[7], title: m[8] });
|
||||
}
|
||||
// Top-most first. Frames include decorations, the same rectangles the
|
||||
// screenshot and recording paths crop to.
|
||||
const all = (await x11Windows()) ?? (await wmctrlWindows()).map((w) => ({ ...w, frame: null, viewable: null, comm: procComm(w.pid) }));
|
||||
const windows = all.filter((w) => !app || sameName(w.name, app) || sameName(w.comm, app)).reverse().map((w) => ({
|
||||
id: w.id, title: w.title, wm_class: w.wm_class, pid: w.pid, process: w.comm ?? null,
|
||||
position: w.frame ? { x: w.frame.x, y: w.frame.y } : null,
|
||||
size: w.frame ? { w: w.frame.w, h: w.frame.h } : null,
|
||||
viewable: w.viewable,
|
||||
}));
|
||||
return { windows };
|
||||
}
|
||||
if (session === "wayland" && tools.swaymsg) {
|
||||
@@ -492,25 +729,39 @@ except Exception as e:
|
||||
},
|
||||
screenshot: async (args = {}) => {
|
||||
rejectWindowSelectors(args);
|
||||
const appRef = Object.hasOwn(args, "app_ref") ? appName(args.app_ref) : null;
|
||||
const { display, region } = args;
|
||||
const outPath = recordingsOutputPath(args.path);
|
||||
await probeSession();
|
||||
const dir = recordingsDir();
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
const file = outPath ?? path.join(dir, `shot-${new Date().toISOString().replace(/[:.]/g, "-")}-${crypto.randomBytes(3).toString("hex")}.png`);
|
||||
await takeShot(file, region);
|
||||
// app_ref crops to the app's window frame. The crop is the screen
|
||||
// there, so anything stacked above the window appears in it; the
|
||||
// receipt names those windows rather than pretending they are absent.
|
||||
let shotRegion = region;
|
||||
let windowInfo = null;
|
||||
let occludedBy = null;
|
||||
if (appRef !== null) {
|
||||
if (region) throw Object.assign(new ExecError("choose app_ref or region, not both"), { code: "bad_args" });
|
||||
const frame = await appFrame(appRef);
|
||||
shotRegion = frame.rect;
|
||||
windowInfo = { id: frame.win.id, title: frame.win.title, wm_class: frame.win.wm_class, pid: frame.win.pid };
|
||||
occludedBy = frame.occludedBy;
|
||||
}
|
||||
await takeShot(file, shotRegion);
|
||||
const dims = pngSize(file);
|
||||
lastRaster = {
|
||||
file,
|
||||
bytes: fs.statSync(file).size,
|
||||
// Region rasters describe the region; full shots get geometry from the
|
||||
// PNG itself (Linux shots are always scale 1: points == pixels).
|
||||
points: region ? { x: region[0], y: region[1], w: region[2], h: region[3] } : dims ? { x: 0, y: 0, w: dims.w, h: dims.h } : null,
|
||||
pixels: dims ?? (region ? { w: Math.round(region[2]), h: Math.round(region[3]) } : null),
|
||||
points: shotRegion ? { x: shotRegion[0], y: shotRegion[1], w: shotRegion[2], h: shotRegion[3] } : dims ? { x: 0, y: 0, w: dims.w, h: dims.h } : null,
|
||||
pixels: dims ?? (shotRegion ? { w: Math.round(shotRegion[2]), h: Math.round(shotRegion[3]) } : null),
|
||||
scale: 1,
|
||||
capturedAt: new Date().toISOString(),
|
||||
};
|
||||
return { ...lastRaster };
|
||||
return { ...lastRaster, ...(windowInfo ? { window: windowInfo, occluded_by: occludedBy, occlusion_note: occludedBy.length ? "windows listed in occluded_by are stacked above this window and appear in the crop" : null } : {}) };
|
||||
},
|
||||
resolve_element: async ({ app_ref, windowIndex, window_id, path: pathArr } = {}) => {
|
||||
const name = appName(app_ref);
|
||||
@@ -622,9 +873,10 @@ print(json.dumps({"found": True, "reason": None, "element": {
|
||||
// Wayland: synthesize wheel via ydotool is not wired in this build — honest refusal.
|
||||
throw new ExecError('scroll on Wayland is not available in this build; use swipe-style drags or run an X11/XWayland window. (Roadmap: ydotool wheel events.)');
|
||||
},
|
||||
type: async ({ text }) => {
|
||||
type: async ({ text, target }) => {
|
||||
if (!text) return { action_sent: false, note: "empty text" };
|
||||
await probeSession();
|
||||
await assertKeyboardReady({ target });
|
||||
if (session === "x11") {
|
||||
// xdotool `type` remaps a spare keycode for characters absent from the
|
||||
// current keymap. Two failure modes follow: a cased letter produces a
|
||||
@@ -661,6 +913,7 @@ print(json.dumps({"found": True, "reason": None, "element": {
|
||||
await xdotool(["type", "--delay", "12", "--", chunk]);
|
||||
}
|
||||
}
|
||||
lastTypedAt = Date.now();
|
||||
return { action_sent: true, chars: text.length };
|
||||
}
|
||||
need("wtype", "typing on Wayland");
|
||||
@@ -668,11 +921,16 @@ print(json.dumps({"found": True, "reason": None, "element": {
|
||||
if (r.code !== 0) throw new ExecError(`wtype failed: ${r.stderr.slice(0, 200)}`, r);
|
||||
return { action_sent: true, chars: text.length };
|
||||
},
|
||||
key: async ({ text, repeat = 1 }) => {
|
||||
key: async ({ text, repeat = 1, target }) => {
|
||||
await probeSession();
|
||||
const k = xdotoolKey(text);
|
||||
const n = Math.max(1, Math.min(100, Number(repeat) || 1));
|
||||
await assertKeyboardReady({ target, allowEscape: k === "Escape" });
|
||||
if (session === "x11") {
|
||||
// Return right after typing races GTK's async completion popups (the
|
||||
// file-chooser location bar swallows the first Return to accept a
|
||||
// completion), so give the app a moment to settle first.
|
||||
if (/^(Return|KP_Enter)$/.test(k)) await settleAfterTyping();
|
||||
throwIfAborted();
|
||||
heldKeys.add(k);
|
||||
try {
|
||||
@@ -685,6 +943,7 @@ print(json.dumps({"found": True, "reason": None, "element": {
|
||||
hold_key: async ({ text, duration }) => {
|
||||
requireInputOwner();
|
||||
await probeSession();
|
||||
await assertKeyboardReady({});
|
||||
const k = xdotoolKey(text);
|
||||
const d = Math.max(0.05, Math.min(30, Number(duration) || 1));
|
||||
if (session === "x11") {
|
||||
@@ -730,7 +989,127 @@ print(json.dumps({"found": True, "reason": None, "element": {
|
||||
if (!out.ok) throw new ExecError(`set_value failed: ${out.code}`);
|
||||
return { action_sent: true, strategy: "a11y", verified: true, after: out.after };
|
||||
},
|
||||
select_text: async () => { throw new ExecError("select_text is not implemented on the linux backend — fail-closed"); },
|
||||
focus: async ({ target }) => {
|
||||
const out = await atspiResolve(target, ` if not found.getState().contains(pyatspi.STATE_FOCUSABLE):
|
||||
raise RuntimeError("element_not_focusable")
|
||||
if not found.queryComponent().grabFocus():
|
||||
raise RuntimeError("focus_refused")
|
||||
print(json.dumps({"ok": True}))`);
|
||||
if (!out.ok) throw Object.assign(new ExecError(`focus failed: ${out.code}`), { code: "focus_failed", reason: out.code });
|
||||
// Focusing an element inside a background window does not make that
|
||||
// window active. Report it honestly: keystrokes would still go to the
|
||||
// active window, so type and key refuse with this target until it is.
|
||||
const owner = await keyboardOwner().catch(() => null);
|
||||
const appName = target.app_ref?.name ?? "";
|
||||
const windowActive = !!owner && (sameName(owner.name, appName) || sameName(owner.comm, appName));
|
||||
return { action_sent: true, strategy: "a11y", focused: true, window_active: windowActive,
|
||||
...(windowActive ? {} : { note: `the element is focused inside ${appName || "its app"}, but that window is not the active one, so keystrokes would go elsewhere` }) };
|
||||
},
|
||||
get_value: async ({ target }) => {
|
||||
const out = await atspiResolve(target, ` result = {"ok": False, "code": "no_value_interface"}
|
||||
try:
|
||||
text = found.queryText()
|
||||
total = text.characterCount
|
||||
result = {"ok": True, "kind": "text", "value": text.getText(0, min(total, 4000)), "characters": total, "truncated": total > 4000}
|
||||
except Exception:
|
||||
try:
|
||||
numeric = found.queryValue()
|
||||
result = {"ok": True, "kind": "numeric", "value": numeric.currentValue}
|
||||
except Exception:
|
||||
pass
|
||||
print(json.dumps(result))`);
|
||||
if (!out.ok) throw Object.assign(new ExecError(`get_value failed: ${out.code}`), { code: out.code === "no_value_interface" ? "no_value" : "get_value_failed" });
|
||||
return { strategy: "a11y", kind: out.kind, value: out.value, characters: out.characters ?? null, truncated: out.truncated ?? false };
|
||||
},
|
||||
select_text: async ({ target, text_range }) => {
|
||||
const range = text_range == null ? null : text_range.map((n) => Math.trunc(Number(n)));
|
||||
if (range && (range.length !== 2 || range.some((n) => !Number.isFinite(n) || n < 0))) {
|
||||
throw Object.assign(new ExecError("text_range must be [start, length] with non-negative integers"), { code: "bad_args" });
|
||||
}
|
||||
const out = await atspiResolve(target, ` text = found.queryText()
|
||||
total = text.characterCount
|
||||
rng = None if extra in (None, "null") else json.loads(extra)
|
||||
if rng is None:
|
||||
if not text.setCaretOffset(total):
|
||||
raise RuntimeError("caret_rejected")
|
||||
print(json.dumps({"ok": True, "caret": total, "characters": total}))
|
||||
else:
|
||||
start, length = rng
|
||||
if start + length > total:
|
||||
raise RuntimeError("range_out_of_bounds")
|
||||
# AT-SPI replaces an existing selection with setSelection, but a widget
|
||||
# with none (GTK's usual state) only accepts addSelection.
|
||||
if text.getNSelections() == 0:
|
||||
selected = text.addSelection(start, start + length)
|
||||
else:
|
||||
selected = text.setSelection(0, start, start + length)
|
||||
if not selected:
|
||||
raise RuntimeError("selection_rejected")
|
||||
begin, end = text.getSelection(0)
|
||||
print(json.dumps({"ok": True, "selected": {"start": begin, "end": end}, "text": text.getText(begin, end)[:500]}))`, range ? JSON.stringify(range) : "null");
|
||||
if (!out.ok) throw Object.assign(new ExecError(`select_text failed: ${out.code}`), { code: "select_text_failed", reason: out.code });
|
||||
return { action_sent: true, strategy: "a11y", ...(range ? { selected: out.selected, text: out.text } : { caret: out.caret, characters: out.characters }) };
|
||||
},
|
||||
input_owner: async ({ kind, point } = {}) => {
|
||||
if (kind === "keyboard") return keyboardOwner();
|
||||
if (kind !== "pointer") throw new ExecError('input_owner kind must be "keyboard" or "pointer"');
|
||||
await probeSession();
|
||||
const at = point ?? (await cursorPoint());
|
||||
return pointOwner(assertNum(at.x, "x"), assertNum(at.y, "y"));
|
||||
},
|
||||
list_sessions: async () => {
|
||||
const owner = await keyboardOwner().catch(() => null);
|
||||
return {
|
||||
via: "direct", count: 1,
|
||||
sessions: [{
|
||||
target: owner ? { name: owner.name, pid: owner.pid } : null,
|
||||
mode: "background", action: null, ageSec: 0,
|
||||
inputHeld: mouseHeld || heldKeys.size > 0,
|
||||
}],
|
||||
};
|
||||
},
|
||||
kill_app: async ({ name, bundle_id, pid, force = false } = {}) => {
|
||||
if (bundle_id) throw Object.assign(new ExecError("bundle_id is unavailable on Linux; kill by name or pid"), { code: "unsupported_selector" });
|
||||
if (!name && pid == null) throw Object.assign(new ExecError("kill_app needs name or pid"), { code: "bad_args" });
|
||||
await probeSession();
|
||||
const guarded = protectedPids();
|
||||
let target;
|
||||
if (pid != null) {
|
||||
target = Math.trunc(Number(pid));
|
||||
if (guarded.has(target)) throw Object.assign(new ExecError("refusing to terminate the Computer Use server or one of its parents"), { code: "bad_args" });
|
||||
if (!alive(target)) throw Object.assign(new ExecError(`no running process has pid ${target}`), { code: "app_not_found" });
|
||||
} else {
|
||||
const want = String(name).replace(/\.app$/i, "");
|
||||
const pids = new Set(processesNamed(want));
|
||||
for (const w of await wmctrlWindows()) if (sameName(w.name, want) && w.pid) pids.add(w.pid);
|
||||
const found = [...pids].filter((p) => !guarded.has(p) && alive(p));
|
||||
if (!found.length) throw Object.assign(new ExecError(`no running application matches "${want}"`), { code: "app_not_found" });
|
||||
if (found.length > 1) throw Object.assign(new ExecError(`${found.length} running processes match "${want}" (pids ${found.join(", ")}); pass pid to choose one`), { code: "ambiguous_application", pids: found });
|
||||
target = found[0];
|
||||
}
|
||||
if (force === true) {
|
||||
try { process.kill(target, "SIGKILL"); } catch { /* already gone */ }
|
||||
return { killed: await waitGone(target, 3_000), pid: target, method: "sigkill", forced: true };
|
||||
}
|
||||
const windows = (await wmctrlWindows()).filter((w) => w.pid === target);
|
||||
let method;
|
||||
if (windows.length) {
|
||||
// A window close is the app's own quit: it can ask to save, which is
|
||||
// what keeps unsaved work safe.
|
||||
for (const w of windows) await softRun("wmctrl", ["-i", "-c", w.id]);
|
||||
method = "window_close";
|
||||
} else {
|
||||
try { process.kill(target, "SIGTERM"); } catch { /* already gone */ }
|
||||
method = "sigterm";
|
||||
}
|
||||
if (await waitGone(target, 5_000)) return { killed: true, pid: target, method, forced: false };
|
||||
return {
|
||||
killed: false, pid: target, method, forced: false, still_running: true,
|
||||
note: method === "window_close"
|
||||
? "the app received a close request and is still running; it may be asking whether to save. Ask the user; retry with force:true only if unsaved work can be discarded."
|
||||
: "the process ignored SIGTERM; retry with force:true to terminate it",
|
||||
};
|
||||
},
|
||||
perform_action: async ({ target, action }) => {
|
||||
const body = ` a = found.queryAction()
|
||||
names = [a.getName(i) for i in range(a.nActions)]
|
||||
@@ -776,23 +1155,153 @@ print(json.dumps({"found": True, "reason": None, "element": {
|
||||
}
|
||||
throw new ExecError("cursor position needs an X11 session in this build");
|
||||
},
|
||||
recordingStart: async () => {
|
||||
throw Object.assign(new ExecError("Recording is unavailable on this platform until the recorder has session-owned cleanup. Use screenshots instead."), { code: "owned_recording_unavailable" });
|
||||
// Session-owned recording: the ffmpeg child belongs to this server. It is
|
||||
// stopped by recording_stop, by session close, and by process exit.
|
||||
recordingStart: async ({ fps = 15, durationSec, region, app_ref } = {}) => {
|
||||
if (app_ref !== undefined) appName(app_ref);
|
||||
await probeSession();
|
||||
if (session !== "x11") throw Object.assign(new ExecError("Linux screen recording needs an X11 session in this build"), { code: "owned_recording_unavailable" });
|
||||
need("ffmpeg", "screen recording");
|
||||
if (region && app_ref) throw Object.assign(new ExecError("choose app_ref or region, not both"), { code: "bad_args" });
|
||||
if (durationSec != null && !(Number(durationSec) > 0)) throw Object.assign(new ExecError("durationSec must be positive"), { code: "bad_args" });
|
||||
const rate = Math.max(1, Math.min(60, Math.round(Number(fps) || 15)));
|
||||
let rect;
|
||||
let windowInfo = null;
|
||||
let occludedBy = null;
|
||||
if (app_ref !== undefined) {
|
||||
const frame = await appFrame(appName(app_ref));
|
||||
rect = frame.rect;
|
||||
windowInfo = { id: frame.win.id, title: frame.win.title, wm_class: frame.win.wm_class, pid: frame.win.pid };
|
||||
occludedBy = frame.occludedBy;
|
||||
} else if (region) {
|
||||
if (!Array.isArray(region) || region.length !== 4 || region.some((n) => !Number.isFinite(Number(n)))) throw Object.assign(new ExecError("region must be [x, y, w, h]"), { code: "bad_args" });
|
||||
rect = region.map((n) => Math.round(Number(n)));
|
||||
} else {
|
||||
rect = await displayRect();
|
||||
}
|
||||
const id = crypto.randomBytes(4).toString("hex");
|
||||
const dir = recordingsDir();
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
const file = path.join(dir, `rec-${id}.mp4`);
|
||||
const display = process.env.DISPLAY;
|
||||
const args = x11GrabArgs({ display, rect, fps: rate, file, durationSec: durationSec == null ? null : Math.ceil(Number(durationSec)) });
|
||||
const child = spawn("ffmpeg", args, { stdio: ["pipe", "ignore", "pipe"] });
|
||||
const entry = { id, child, file, rect, fps: rate, startedAt: new Date().toISOString(), stderr: "" };
|
||||
child.stdin.on("error", () => {});
|
||||
child.stderr.on("data", (chunk) => { entry.stderr = (entry.stderr + chunk).slice(-4000); });
|
||||
const killOnExit = () => { try { if (child.exitCode == null && child.signalCode == null) child.kill("SIGKILL"); } catch { /* gone */ } };
|
||||
process.once("exit", killOnExit);
|
||||
child.once("close", () => process.off("exit", killOnExit));
|
||||
recordings.set(id, entry);
|
||||
try {
|
||||
await wait(500);
|
||||
} catch (err) {
|
||||
await stopRecording(entry, 2_000);
|
||||
recordings.delete(id);
|
||||
throw err;
|
||||
}
|
||||
if (child.exitCode != null || child.signalCode != null) {
|
||||
recordings.delete(id);
|
||||
throw Object.assign(new ExecError(`ffmpeg could not start the recording: ${entry.stderr.trim().slice(-300) || `exit ${child.exitCode}`}`), { code: "recording_failed" });
|
||||
}
|
||||
return {
|
||||
id, pid: child.pid, file, display, region: rect, durationSec: durationSec ?? null, fps: rate, mode: "x11grab", startedAt: entry.startedAt,
|
||||
...(windowInfo ? { window: windowInfo, occluded_by: occludedBy, note: "Recording the app's window frame as it was at start; it does not follow moves or resizes." } : {}),
|
||||
};
|
||||
},
|
||||
recordingStop: async ({ id }) => {
|
||||
const entry = recordings.get(id);
|
||||
if (!entry) throw new ExecError(`unknown or already-finished recording "${id}"`);
|
||||
const result = await stopRecording(entry, 20_000);
|
||||
recordings.delete(id);
|
||||
if (result.code !== 0) throw Object.assign(new ExecError(`ffmpeg exited ${result.code}: ${result.error.slice(-300) || "partial file retained"}`), { code: "recording_failed", file: entry.file });
|
||||
const bytes = fs.existsSync(entry.file) ? fs.statSync(entry.file).size : 0;
|
||||
if (!bytes) throw new ExecError("the recorder produced no video");
|
||||
return { id, file: entry.file, bytes, mode: "x11grab", startedAt: entry.startedAt, stoppedAt: new Date().toISOString() };
|
||||
},
|
||||
recordingStatus: ({ id }) => {
|
||||
const entry = recordings.get(id);
|
||||
if (!entry) return { id, running: false };
|
||||
const running = entry.child.exitCode == null && entry.child.signalCode == null;
|
||||
return { id, running, pid: entry.child.pid, file: entry.file, bytes: fs.existsSync(entry.file) ? fs.statSync(entry.file).size : 0, startedAt: entry.startedAt };
|
||||
},
|
||||
recordingStop: async ({ id }) => { throw new ExecError(`unknown recording "${id}"`); },
|
||||
recordingStatus: ({ id }) => ({ id, running: false }),
|
||||
recordingList: async () => {
|
||||
const dir = recordingsDir();
|
||||
const out = fs.existsSync(dir)
|
||||
? fs.readdirSync(dir).filter((f) => /\.(mp4|mkv|png)$/i.test(f)).map((f) => {
|
||||
? fs.readdirSync(dir).filter((f) => /\.(mp4|mkv|png|jpe?g)$/i.test(f)).map((f) => {
|
||||
const st = fs.statSync(path.join(dir, f));
|
||||
return { file: path.join(dir, f), bytes: st.size, modifiedAt: st.mtime.toISOString() };
|
||||
}).sort((a, b) => b.modifiedAt.localeCompare(a.modifiedAt)).slice(0, 50)
|
||||
: [];
|
||||
return { dir, recordings: out, running: [] };
|
||||
return { dir, recordings: out, running: [...recordings.keys()] };
|
||||
},
|
||||
};
|
||||
|
||||
/** Process state from /proc: state letter and parent pid, or null when the process is gone. */
|
||||
function procStat(pid) {
|
||||
try {
|
||||
const raw = fs.readFileSync(`/proc/${pid}/stat`, "utf8");
|
||||
const fields = raw.slice(raw.lastIndexOf(")") + 2).split(" ");
|
||||
return { state: fields[0], ppid: Number(fields[1]) };
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
/** Running, not a zombie: a process that exited but was not yet reaped is gone. */
|
||||
function alive(pid) {
|
||||
const st = procStat(pid);
|
||||
return !!st && st.state !== "Z" && st.state !== "X";
|
||||
}
|
||||
|
||||
/** The server and its ancestors: kill_app must never end the process that is serving the request. */
|
||||
function protectedPids() {
|
||||
const set = new Set([0, 1, process.pid]);
|
||||
let pid = process.pid;
|
||||
for (let i = 0; i < 64; i++) {
|
||||
const st = procStat(pid);
|
||||
if (!st || st.ppid <= 0) break;
|
||||
set.add(st.ppid);
|
||||
pid = st.ppid;
|
||||
}
|
||||
return set;
|
||||
}
|
||||
|
||||
/** Processes whose kernel name (comm, 15 bytes at most) equals the requested app name. */
|
||||
function processesNamed(name) {
|
||||
const want = String(name).toLowerCase().slice(0, 15);
|
||||
let entries = [];
|
||||
try { entries = fs.readdirSync("/proc"); } catch { return []; }
|
||||
return entries.filter((e) => /^\d+$/.test(e)).map(Number).filter((pid) => (procComm(pid) ?? "").toLowerCase() === want);
|
||||
}
|
||||
|
||||
async function waitGone(pid, ms) {
|
||||
const until = Date.now() + ms;
|
||||
while (alive(pid)) {
|
||||
if (Date.now() >= until) return false;
|
||||
await wait(100);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
async function displayRect() {
|
||||
const out = (await xdotool(["getdisplaygeometry"])).trim().split(/\s+/).map(Number);
|
||||
if (out.length !== 2 || out.some((n) => !Number.isFinite(n) || n <= 0)) throw new ExecError("could not read the display size for recording");
|
||||
return [0, 0, out[0], out[1]];
|
||||
}
|
||||
|
||||
/** Stop a recorder: ask ffmpeg to finish the file ('q'), kill it if it does not. */
|
||||
function stopRecording(entry, timeoutMs) {
|
||||
return new Promise((resolve) => {
|
||||
const { child } = entry;
|
||||
if (child.exitCode != null || child.signalCode != null) {
|
||||
resolve({ code: child.exitCode, error: entry.stderr.trim() });
|
||||
return;
|
||||
}
|
||||
const killer = setTimeout(() => { try { child.kill("SIGKILL"); } catch { /* gone */ } }, timeoutMs);
|
||||
child.once("close", (code) => { clearTimeout(killer); resolve({ code, error: entry.stderr.trim() }); });
|
||||
try { child.stdin.write("q"); child.stdin.end(); } catch { /* already exiting */ }
|
||||
});
|
||||
}
|
||||
|
||||
async function inputChain(x, y, act) {
|
||||
await inputMove(x, y);
|
||||
await act();
|
||||
|
||||
@@ -91,7 +91,11 @@ export function findBrowserApp(platform = process.platform, env = process.env, e
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Launch detached so the browser is its own process, never a child we must reap. */
|
||||
/**
|
||||
* Launch detached so the browser is its own process, never a child we must
|
||||
* reap. Returns a handle that keeps the tail of its stderr and its exit, so a
|
||||
* browser that dies at start can say why instead of timing out silently.
|
||||
*/
|
||||
function defaultLaunch({ app, profileDir, url, platform = process.platform }) {
|
||||
const flags = ["--remote-debugging-port=0", `--user-data-dir=${profileDir}`, "--no-first-run", "--no-default-browser-check"];
|
||||
const target = url || "about:blank";
|
||||
@@ -100,9 +104,30 @@ function defaultLaunch({ app, profileDir, url, platform = process.platform }) {
|
||||
// the person already has Chrome running — the args never reach a new process.
|
||||
if (platform === "darwin") { cmd = "open"; args = ["-g", "-n", "-a", app, "--args", ...flags, target]; }
|
||||
else { cmd = app; args = [...flags, target]; }
|
||||
const child = spawn(cmd, args, { detached: true, stdio: "ignore" });
|
||||
child.on("error", () => {});
|
||||
const child = spawn(cmd, args, { detached: true, stdio: ["ignore", "ignore", "pipe"] });
|
||||
const handle = { stderr: "", exit: null };
|
||||
child.stderr.on("data", (chunk) => { handle.stderr = (handle.stderr + chunk).slice(-4000); });
|
||||
child.on("error", (error) => { handle.exit = { code: null, signal: null, error: error.message }; });
|
||||
child.on("exit", (code, signal) => { handle.exit = { code, signal }; });
|
||||
child.unref();
|
||||
return handle;
|
||||
}
|
||||
|
||||
/**
|
||||
* The error for a browser that did not produce a debugging endpoint. Chromium
|
||||
* refuses to run as root without --no-sandbox; Codewhale does not add that
|
||||
* flag, so that case names the two supported ways forward instead.
|
||||
*/
|
||||
export function browserStartFailure(handle, profileDir) {
|
||||
const tail = handle?.stderr ?? "";
|
||||
if (/Running as root without --no-sandbox/.test(tail)) {
|
||||
return Object.assign(new ExecError("Chromium refused to start because it runs as root without its sandbox. Codewhale does not disable the sandbox. Run the Computer Use server as a non-root user, or set CODEWHALE_CU_BROWSER_APP to a launcher you control that supplies the flags you accept."), { code: "browser_unavailable" });
|
||||
}
|
||||
const exit = handle?.exit
|
||||
? `the browser exited before it opened its debugging endpoint (${handle.exit.error ?? `code ${handle.exit.code ?? "none"}${handle.exit.signal ? `, signal ${handle.exit.signal}` : ""}`})`
|
||||
: "the browser started but its debugging endpoint never came up";
|
||||
const detail = tail.trim().split("\n").slice(-2).join(" | ").slice(-300);
|
||||
return Object.assign(new ExecError(`${exit} (profile ${profileDir})${detail ? `: ${detail}` : ""}`), { code: "browser_unavailable" });
|
||||
}
|
||||
|
||||
/** The CDP bridge socket to attach to, or null for launch mode. */
|
||||
@@ -459,7 +484,7 @@ export function createBrowser({
|
||||
else { try { fs.rmSync(portFile, { force: true }); } catch {} }
|
||||
}
|
||||
if (!ws) {
|
||||
await launch({ app, profileDir: dir, url: "about:blank", platform });
|
||||
const launched = await launch({ app, profileDir: dir, url: "about:blank", platform });
|
||||
const deadline = Date.now() + 20_000;
|
||||
for (;;) {
|
||||
if (currentSignal()?.aborted) throw Object.assign(new ExecError("computer request cancelled"), { code: "cancelled" });
|
||||
@@ -468,7 +493,9 @@ export function createBrowser({
|
||||
ws = await attempt(file);
|
||||
if (ws) { state.port = file.port; break; }
|
||||
}
|
||||
if (Date.now() > deadline) throw Object.assign(new ExecError(`the browser started but its debugging endpoint never came up (profile ${dir}); is it running with a usable profile?`), { code: "browser_unavailable" });
|
||||
// A browser that already exited will not open the endpoint: report
|
||||
// now, with its stderr, rather than after the full timeout.
|
||||
if (launched?.exit || Date.now() > deadline) throw browserStartFailure(launched, dir);
|
||||
await sleep(300);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -160,7 +160,7 @@ export const TOOLS = [
|
||||
},
|
||||
{
|
||||
name: "list_windows",
|
||||
description: "List application windows. On macOS, app_ref selects the app; omission follows the app selected by open_application, or the frontmost app before a selection. Other platforms list all windows and reject app_ref selectors.",
|
||||
description: "List application windows. On macOS, app_ref selects the app; omission follows the app selected by open_application, or the frontmost app before a selection. On Linux (X11), app_ref.name selects the app by exact name. Other platforms list all windows and reject app_ref selectors.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
@@ -222,7 +222,7 @@ export const TOOLS = [
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
app_ref: { type: "object", properties: { name: { type: "string" }, bundle_id: { type: "string" }, pid: { type: "integer" } }, description: "macOS: capture this app window even when it is in the background." },
|
||||
app_ref: { type: "object", properties: { name: { type: "string" }, bundle_id: { type: "string" }, pid: { type: "integer" } }, description: "macOS: capture this app window even when it is in the background. Linux (X11): capture the app's window frame by exact name; the receipt names windows stacked above it." },
|
||||
display: { type: ["integer", "string"], description: "Display index or 'all'" },
|
||||
region: { type: "array", items: { type: "number" }, minItems: 4, maxItems: 4, description: "[x, y, w, h] in screen points" },
|
||||
path: { type: "string", description: "Optional absolute .png/.jpg/.jpeg path inside the recordings directory. Omit to use a generated name there." },
|
||||
@@ -410,7 +410,7 @@ export const TOOLS = [
|
||||
},
|
||||
// ---- text & keyboard ----
|
||||
{
|
||||
name: "type", description: "Type unicode text into the focused control. Newlines in `text` are Return/Enter key presses, not literal characters — never put \\n in a composer by hoping it will send. Focus the field first (click, focus, or set_value), or pass an element `target` to focus it in the same call. On macOS the receipt carries `verified:true` only when the focused control's value actually reflects the typed text; on `verified:false` the text may have gone nowhere — observe again before relying on it.",
|
||||
name: "type", description: "Type unicode text into the focused control. On Linux, keystrokes are refused while a menu item is open (send key escape first) and when an element target's window is not the active one. Newlines in `text` are Return/Enter key presses, not literal characters — never put \\n in a composer by hoping it will send. Focus the field first (click, focus, or set_value), or pass an element `target` to focus it in the same call. On macOS the receipt carries `verified:true` only when the focused control's value actually reflects the typed text; on `verified:false` the text may have gone nowhere — observe again before relying on it.",
|
||||
inputSchema: { type: "object", required: ["text"], properties: { text: { type: "string" }, press_enter: { type: "boolean", description: "After typing, press Return/Enter once. Prefer this to putting a newline in `text` when you want to send." }, target: { ...elementTargetSchema, description: "Element target from get_app_state; it is accessibility-focused first, then the text is typed. Element targets only." }, computer: computerParam }, additionalProperties: false },
|
||||
},
|
||||
{
|
||||
@@ -519,7 +519,7 @@ export const TOOLS = [
|
||||
},
|
||||
{
|
||||
name: "recording_start",
|
||||
description: "Start screen recording on a computer (mp4/mov). Darwin: ScreenCaptureKit via the native helper (timed or until recording_stop; honors region, no recorder overlay, stops on session exit). Pass app_ref to record only the selected app's window rect — captured at start and not tracked across moves. Linux and Windows: unavailable pending session-owned recorder cleanup; use screenshots. HarmonyOS: snapshot-series muxed with ffmpeg.",
|
||||
description: "Start screen recording on a computer (mp4/mov). Darwin: ScreenCaptureKit via the native helper (timed or until recording_stop; honors region, no recorder overlay, stops on session exit). Pass app_ref to record only the selected app's window rect — captured at start and not tracked across moves. Linux (X11): ffmpeg x11grab, owned by this server and finished by recording_stop or session close; honors region and app_ref (window frame at start). Windows: unavailable; use screenshots. HarmonyOS: snapshot-series muxed with ffmpeg.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
|
||||
@@ -114,6 +114,13 @@ export function create() {
|
||||
async focus(args) { record("focus", args); return { action_sent: true, focused: true, strategy: "a11y" }; },
|
||||
async get_value(args) { record("get_value", args); return { value: "Fixture text", strategy: "a11y" }; },
|
||||
async invoke_menu(args) { record("invoke_menu", args); return { action_sent: true, strategy: "a11y" }; },
|
||||
// Who would receive input: FAKE_BACKEND_OWNER is the owning app as JSON, or
|
||||
// "null" for the desktop. The server must consult this before any input.
|
||||
async input_owner({ kind, point } = {}) {
|
||||
record("input_owner", { kind, point });
|
||||
const raw = process.env.FAKE_BACKEND_OWNER;
|
||||
return raw && raw !== "null" ? JSON.parse(raw) : null;
|
||||
},
|
||||
async app_script(args) { record("app_script", args); return { result: "fake", language: args.language ?? "applescript" }; },
|
||||
};
|
||||
}
|
||||
|
||||
@@ -110,14 +110,28 @@ test("Wayland child cancellation is reported as cancelled instead of success", a
|
||||
await assert.rejects(withSignal(controller.signal, () => backend.hold_key({ text: "shift", duration: 30 })), (err) => err.code === "cancelled");
|
||||
});
|
||||
|
||||
test("Windows refuses recording before spawning an unowned recorder", async () => {
|
||||
const calls = [];
|
||||
const backend = windows.create({ exec: { persistentInputOwner: true, run: async (...args) => { calls.push(args); return success; } } });
|
||||
await assert.rejects(backend.recordingStart({ fps: 15 }), (err) => err.code === "owned_recording_unavailable");
|
||||
assert.deepEqual(calls, []);
|
||||
assert.deepEqual(await backend.recordingStatus({ id: "not-started" }), { id: "not-started", running: false });
|
||||
});
|
||||
|
||||
test("Linux recording without an X11 session refuses before spawning ffmpeg", async (t) => {
|
||||
// Clear every display variable so this test never starts a real recorder,
|
||||
// even on a developer machine that runs inside X.
|
||||
const saved = Object.fromEntries(["DISPLAY", "WAYLAND_DISPLAY", "XDG_SESSION_TYPE"].map((key) => [key, process.env[key]]));
|
||||
for (const key of Object.keys(saved)) delete process.env[key];
|
||||
t.after(() => { for (const [key, value] of Object.entries(saved)) { if (value !== undefined) process.env[key] = value; } });
|
||||
const calls = [];
|
||||
const backend = linux.create({ exec: { persistentInputOwner: true, have: async () => true, run: async (...args) => { calls.push(args); return success; } } });
|
||||
await assert.rejects(backend.recordingStart({ fps: 15 }), (err) => err.code === "no_session");
|
||||
assert.deepEqual(calls, []);
|
||||
assert.deepEqual(await backend.recordingStatus({ id: "not-started" }), { id: "not-started", running: false });
|
||||
});
|
||||
|
||||
for (const [name, platform] of [["Windows", windows], ["Linux", linux]]) {
|
||||
test(`${name} refuses recording before spawning an unowned recorder`, async () => {
|
||||
const calls = [];
|
||||
const backend = platform.create({ exec: { persistentInputOwner: true, run: async (...args) => { calls.push(args); return success; } } });
|
||||
await assert.rejects(backend.recordingStart({ fps: 15 }), (err) => err.code === "owned_recording_unavailable");
|
||||
assert.deepEqual(calls, []);
|
||||
assert.deepEqual(await backend.recordingStatus({ id: "not-started" }), { id: "not-started", running: false });
|
||||
});
|
||||
|
||||
test(`${name} direct mode refuses held gestures before moving or pressing`, async () => {
|
||||
const calls = [];
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
// Input ownership at the MCP boundary: keystrokes and pointer events are judged
|
||||
// against the app that owns the window receiving them, not the app the agent
|
||||
// last observed. Runs the real server with the fake backend, which reports the
|
||||
// owner it is told about (FAKE_BACKEND_OWNER) and records every backend call.
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { spawn } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import url from "node:url";
|
||||
|
||||
const ROOT = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), "..");
|
||||
const XTERM = { name: "xterm", comm: "xterm", pid: 4242, wm_class: "xterm.XTerm", title: "root", window: "0x00e0000c" };
|
||||
|
||||
function startServer(t, owner) {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "cu-owner-"));
|
||||
t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
|
||||
const calls = path.join(dir, "calls.jsonl");
|
||||
const server = spawn("node", [path.join(ROOT, "mcp/server.mjs")], {
|
||||
env: {
|
||||
...process.env,
|
||||
CODEWHALE_CU_STATE_DIR: path.join(dir, "state"),
|
||||
CODEWHALE_CU_RECORDINGS_DIR: path.join(dir, "rec"),
|
||||
CODEWHALE_CU_TEST_BACKEND: path.join(ROOT, "tests/fixtures/fake-backend.mjs"),
|
||||
FAKE_BACKEND_CALLS: calls,
|
||||
FAKE_BACKEND_OWNER: owner === null ? "null" : JSON.stringify(owner),
|
||||
},
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
});
|
||||
t.after(() => { try { server.kill("SIGTERM"); } catch {} });
|
||||
const pending = new Map();
|
||||
const prompts = [];
|
||||
let buf = "";
|
||||
let nextId = 1;
|
||||
server.stdout.setEncoding("utf8");
|
||||
server.stdout.on("data", (chunk) => {
|
||||
buf += chunk;
|
||||
let i;
|
||||
while ((i = buf.indexOf("\n")) !== -1) {
|
||||
const line = buf.slice(0, i).trim();
|
||||
buf = buf.slice(i + 1);
|
||||
if (!line) continue;
|
||||
const msg = JSON.parse(line);
|
||||
if (msg.method === "elicitation/create" && msg.id !== undefined) {
|
||||
// The user approves exactly one call here: consent for xterm, nothing else.
|
||||
prompts.push(msg.params.message);
|
||||
const approve = msg.params.message === 'Computer Use asks for your decision: consent_allow {"app":"xterm"}';
|
||||
server.stdin.write(JSON.stringify({ jsonrpc: "2.0", id: msg.id, result: { action: approve ? "accept" : "decline" } }) + "\n");
|
||||
continue;
|
||||
}
|
||||
if (msg.id !== undefined && pending.has(msg.id)) { pending.get(msg.id)(msg); pending.delete(msg.id); }
|
||||
}
|
||||
});
|
||||
const rpc = (method, params) => new Promise((resolve) => {
|
||||
const id = nextId++;
|
||||
pending.set(id, resolve);
|
||||
server.stdin.write(JSON.stringify({ jsonrpc: "2.0", id, method, params }) + "\n");
|
||||
});
|
||||
const tool = async (name, args = {}) => {
|
||||
const res = await rpc("tools/call", { name, arguments: args });
|
||||
const parsed = JSON.parse(res.result.content[0].text);
|
||||
return { ...parsed, isError: res.result.isError === true };
|
||||
};
|
||||
const backendCalls = () => (fs.existsSync(calls) ? fs.readFileSync(calls, "utf8").trim().split("\n").filter(Boolean).map((l) => JSON.parse(l)) : []);
|
||||
return {
|
||||
init: () => rpc("initialize", { protocolVersion: "2025-06-18", capabilities: { elicitation: {} }, clientInfo: { name: "owner-test", version: "0" } }),
|
||||
tool,
|
||||
prompts,
|
||||
backendCalls,
|
||||
methods: () => backendCalls().map((c) => c.method),
|
||||
};
|
||||
}
|
||||
|
||||
test("keystrokes are refused when the app that owns the active window has no decision, before the backend sees them", async (t) => {
|
||||
const s = startServer(t, XTERM);
|
||||
await s.init();
|
||||
const r = await s.tool("type", { text: "hello" });
|
||||
assert.equal(r.ok, false);
|
||||
assert.equal(r.error.code, "consent_required");
|
||||
assert.match(r.error.message, /xterm/);
|
||||
assert.ok(!s.methods().includes("type"), "the backend must not receive the keystrokes");
|
||||
assert.ok(s.methods().includes("input_owner"), "the owner is resolved before input");
|
||||
});
|
||||
|
||||
test("pointer input is judged against the window under the point, after a screenshot pins the frame", async (t) => {
|
||||
const s = startServer(t, XTERM);
|
||||
await s.init();
|
||||
assert.equal((await s.tool("screenshot", {})).ok, true);
|
||||
const r = await s.tool("left_click", { target: { type: "coordinate", x: 10, y: 20 } });
|
||||
assert.equal(r.error.code, "consent_required");
|
||||
assert.match(r.error.message, /xterm/);
|
||||
assert.ok(!s.methods().includes("left_click"));
|
||||
const owner = s.backendCalls().find((c) => c.method === "input_owner");
|
||||
assert.equal(owner.args.kind, "pointer");
|
||||
// The raster pixel (10,20) is converted to screen points before ownership
|
||||
// is judged; the fake screenshot is at scale 2, so the point is (5,10).
|
||||
assert.deepEqual(owner.args.point, { x: 5, y: 10 });
|
||||
});
|
||||
|
||||
test("an app the user approved receives input; the approval is recorded through the host, not by the model", async (t) => {
|
||||
const s = startServer(t, XTERM);
|
||||
await s.init();
|
||||
const refused = await s.tool("consent", { action: "allow", app: "xterm" });
|
||||
assert.equal(refused.ok, true, "the user approves through elicitation, which this test answers");
|
||||
assert.deepEqual(s.prompts, ['Computer Use asks for your decision: consent_allow {"app":"xterm"}']);
|
||||
const typed = await s.tool("type", { text: "hi" });
|
||||
assert.equal(typed.ok, true);
|
||||
assert.ok(s.methods().includes("type"));
|
||||
});
|
||||
|
||||
test("a denied app is refused with app_denied even when it is the only window", async (t) => {
|
||||
const s = startServer(t, XTERM);
|
||||
await s.init();
|
||||
assert.equal((await s.tool("consent", { action: "deny", app: "xterm" })).ok, true);
|
||||
const r = await s.tool("key", { text: "Return" });
|
||||
assert.equal(r.error.code, "app_denied");
|
||||
assert.ok(!s.methods().includes("key"));
|
||||
});
|
||||
|
||||
test("with the desktop as owner there is no application to consent to", async (t) => {
|
||||
const s = startServer(t, null);
|
||||
await s.init();
|
||||
const r = await s.tool("type", { text: "desktop" });
|
||||
assert.equal(r.ok, true);
|
||||
assert.ok(s.methods().includes("type"));
|
||||
});
|
||||
@@ -0,0 +1,315 @@
|
||||
// Linux desktop behavior that does not need a display: window ownership and
|
||||
// geometry, keyboard guards, AT-SPI element actions, screenshots of app
|
||||
// windows, recording arguments, kill_app and session listing. Commands are
|
||||
// injected through `exec`, so each test states exactly what the window manager
|
||||
// reports and asserts what the backend sends.
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { spawn } from "node:child_process";
|
||||
import { create, x11GrabArgs } from "../src/backends/linux.mjs";
|
||||
import { browserStartFailure } from "../src/browser-cdp.mjs";
|
||||
|
||||
const PNG_1X1 = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==", "base64");
|
||||
|
||||
// Two managed windows, bottom to top: Mousepad, then xterm stacked above it.
|
||||
// xwininfo reports client rectangles; _NET_FRAME_EXTENTS grows them to frames.
|
||||
const WINDOWS = {
|
||||
"0x800003": { pid: 10165, wm_class: "mousepad.Mousepad", title: "Untitled 1 - Mousepad", client: [1, 315, 640, 480], ext: [1, 1, 20, 5] },
|
||||
"0xe0000c": { pid: 10400, wm_class: "xterm.XTerm", title: "root@vm: /home/user", client: [401, 290, 362, 135], ext: [1, 1, 20, 5] },
|
||||
};
|
||||
|
||||
/** X window ids compare by number: the backend pads them to eight hex digits, the fixture does not. */
|
||||
function windowFor(id) {
|
||||
const key = Object.keys(WINDOWS).find((k) => parseInt(k, 16) === parseInt(id, 16));
|
||||
return key ? WINDOWS[key] : undefined;
|
||||
}
|
||||
|
||||
/** A fake desktop: answers the commands the Linux backend asks the window manager. */
|
||||
function fakeDesktop({ active = "0x800003", stack = ["0x800003", "0xe0000c"], onCommand = () => null } = {}) {
|
||||
const calls = [];
|
||||
const run = async (cmd, args = []) => {
|
||||
calls.push({ cmd, args });
|
||||
const custom = onCommand(cmd, args);
|
||||
if (custom) return custom;
|
||||
const ok = (stdout) => ({ code: 0, stdout, stderr: "", timedOut: false, signal: null });
|
||||
const fail = { code: 1, stdout: "", stderr: "", timedOut: false, signal: null };
|
||||
if (cmd === "xprop" && args[0] === "-root" && args[1] === "_NET_CLIENT_LIST_STACKING") {
|
||||
return stack.length ? ok(`_NET_CLIENT_LIST_STACKING(WINDOW): window id # ${stack.join(", ")}\n`) : ok("_NET_CLIENT_LIST_STACKING: not found.\n");
|
||||
}
|
||||
if (cmd === "wmctrl" && args[0] === "-lpx") {
|
||||
return ok(Object.entries(WINDOWS).map(([id, w]) => `${id} 0 ${w.pid} ${w.wm_class.padEnd(20)} vm ${w.title}`).join("\n") + "\n");
|
||||
}
|
||||
if (cmd === "xwininfo" && args[0] === "-id") {
|
||||
const w = windowFor(args[1]);
|
||||
if (!w) return fail;
|
||||
return ok(` Absolute upper-left X: ${w.client[0]}\n Absolute upper-left Y: ${w.client[1]}\n Width: ${w.client[2]}\n Height: ${w.client[3]}\n Map State: IsViewable\n`);
|
||||
}
|
||||
if (cmd === "xprop" && args[0] === "-id") {
|
||||
const w = windowFor(args[1]);
|
||||
if (!w) return fail;
|
||||
return ok(`_NET_FRAME_EXTENTS(CARDINAL) = ${w.ext.join(", ")}\n`);
|
||||
}
|
||||
if (cmd === "xdotool" && args[0] === "getactivewindow") return ok(`${parseInt(active, 16)}\n`);
|
||||
if (cmd === "xdotool" && args[0] === "getdisplaygeometry") return ok("1280 800\n");
|
||||
if (cmd === "xdotool" && args[0] === "getmouselocation") return ok("x:700 y:350 screen:0 window:12345\n");
|
||||
return ok("");
|
||||
};
|
||||
return { run, calls };
|
||||
}
|
||||
|
||||
/** Environment that makes the backend believe it is on an X11 session, restored afterwards. */
|
||||
function x11Env(t) {
|
||||
const saved = Object.fromEntries(["DISPLAY", "WAYLAND_DISPLAY", "XDG_SESSION_TYPE"].map((k) => [k, process.env[k]]));
|
||||
delete process.env.WAYLAND_DISPLAY;
|
||||
process.env.XDG_SESSION_TYPE = "x11";
|
||||
process.env.DISPLAY = ":test";
|
||||
t.after(() => {
|
||||
for (const [k, v] of Object.entries(saved)) { if (v === undefined) delete process.env[k]; else process.env[k] = v; }
|
||||
});
|
||||
}
|
||||
|
||||
const have = async () => true;
|
||||
|
||||
test("keyboard owner is the app with keyboard focus, not the last observed app", async (t) => {
|
||||
x11Env(t);
|
||||
const desk = fakeDesktop({ active: "0xe0000c" });
|
||||
const backend = create({ exec: { run: desk.run, have } });
|
||||
const owner = await backend.input_owner({ kind: "keyboard" });
|
||||
assert.equal(owner.name, "xterm");
|
||||
assert.equal(owner.pid, 10400);
|
||||
assert.equal(owner.window, "0x00e0000c");
|
||||
});
|
||||
|
||||
test("pointer owner is the topmost window under the point; the desktop has no owner", async (t) => {
|
||||
x11Env(t);
|
||||
const desk = fakeDesktop();
|
||||
const backend = create({ exec: { run: desk.run, have } });
|
||||
// (700, 350) lies inside xterm's frame (x 400..764, y 270..430) and outside Mousepad's frame (x 0..642).
|
||||
assert.equal((await backend.input_owner({ kind: "pointer", point: { x: 700, y: 350 } })).name, "xterm");
|
||||
// (100, 600) lies only inside Mousepad's frame.
|
||||
assert.equal((await backend.input_owner({ kind: "pointer", point: { x: 100, y: 600 } })).name, "mousepad");
|
||||
// (10, 10) is on the desktop.
|
||||
assert.equal(await backend.input_owner({ kind: "pointer", point: { x: 10, y: 10 } }), null);
|
||||
});
|
||||
|
||||
test("pointer owner without a point reads the cursor position", async (t) => {
|
||||
x11Env(t);
|
||||
const backend = create({ exec: { run: fakeDesktop().run, have } });
|
||||
assert.equal((await backend.input_owner({ kind: "pointer" })).name, "xterm"); // cursor at (700, 350)
|
||||
});
|
||||
|
||||
test("without a window manager the owner is unknown and input fails closed", async (t) => {
|
||||
x11Env(t);
|
||||
const backend = create({ exec: { run: fakeDesktop({ stack: [] }).run, have } });
|
||||
await assert.rejects(backend.input_owner({ kind: "keyboard" }), (err) => err.code === "input_owner_unknown");
|
||||
await assert.rejects(backend.input_owner({ kind: "pointer", point: { x: 1, y: 1 } }), (err) => err.code === "input_owner_unknown");
|
||||
});
|
||||
|
||||
test("keyboard input is refused while an open menu item is showing", async (t) => {
|
||||
x11Env(t);
|
||||
const desk = fakeDesktop({
|
||||
onCommand: (cmd, args) => (cmd === "python3" && args[1].includes("STATE_SHOWING") && args[1].includes("popup"))
|
||||
? { code: 0, stdout: '{"found": true, "popup": {"role": "menu item", "label": "Save As..."}, "visited": 9}\n', stderr: "", timedOut: false, signal: null }
|
||||
: null,
|
||||
});
|
||||
const backend = create({ exec: { run: desk.run, have } });
|
||||
await assert.rejects(backend.type({ text: "x" }), (err) => err.code === "popup_open");
|
||||
await assert.rejects(backend.key({ text: "Return" }), (err) => err.code === "popup_open");
|
||||
// Escape is how the agent closes the menu, so it is never refused for that reason.
|
||||
await backend.key({ text: "Escape" });
|
||||
assert.ok(desk.calls.some((c) => c.cmd === "xdotool" && c.args.includes("Escape")));
|
||||
});
|
||||
|
||||
test("targeted keystrokes refuse when the element's window is not the active one", async (t) => {
|
||||
x11Env(t);
|
||||
const backend = create({ exec: { run: fakeDesktop({ active: "0x800003" }).run, have } });
|
||||
const target = { app_ref: { name: "xterm" }, windowIndex: 0, path: [0], strategy: "a11y" };
|
||||
await assert.rejects(backend.type({ text: "x", target }), (err) => err.code === "window_not_active");
|
||||
await assert.rejects(backend.key({ text: "Return", target }), (err) => err.code === "window_not_active");
|
||||
});
|
||||
|
||||
test("focus reports whether the focused element's window is active; it never activates a window", async (t) => {
|
||||
x11Env(t);
|
||||
const desk = fakeDesktop({
|
||||
active: "0x800003",
|
||||
onCommand: (cmd, args) => (cmd === "python3" && args[1].includes("target_path") && args[1].includes("grabFocus"))
|
||||
? { code: 0, stdout: '{"ok": true}\n', stderr: "", timedOut: false, signal: null }
|
||||
: null,
|
||||
});
|
||||
const backend = create({ exec: { run: desk.run, have } });
|
||||
const out = await backend.focus({ target: { app_ref: { name: "xterm" }, windowIndex: 0, path: [0] } });
|
||||
assert.equal(out.focused, true);
|
||||
assert.equal(out.window_active, false);
|
||||
assert.ok(!desk.calls.some((c) => c.cmd === "wmctrl" && c.args.includes("-a")), "focus must not raise windows");
|
||||
assert.ok(!desk.calls.some((c) => c.cmd === "xdotool" && c.args[0] === "windowactivate"), "focus must not raise windows");
|
||||
});
|
||||
|
||||
test("get_value and select_text pass their arguments to AT-SPI and return what it reports", async (t) => {
|
||||
x11Env(t);
|
||||
const seen = [];
|
||||
const desk = fakeDesktop({
|
||||
onCommand: (cmd, args) => {
|
||||
if (cmd !== "python3" || !args[1].includes("target_path")) return null;
|
||||
seen.push(args);
|
||||
// The select script also calls getText, so match on the call only it makes.
|
||||
if (args[1].includes("setSelection") || args[1].includes("setCaretOffset")) return { code: 0, stdout: '{"ok": true, "selected": {"start": 1, "end": 3}, "text": "el"}\n', stderr: "", timedOut: false, signal: null };
|
||||
return { code: 0, stdout: '{"ok": true, "kind": "text", "value": "hello", "characters": 5, "truncated": false}\n', stderr: "", timedOut: false, signal: null };
|
||||
},
|
||||
});
|
||||
const backend = create({ exec: { run: desk.run, have } });
|
||||
const target = { app_ref: { name: "mousepad" }, windowIndex: 0, path: [318] };
|
||||
const value = await backend.get_value({ target });
|
||||
assert.deepEqual([value.kind, value.value, value.characters], ["text", "hello", 5]);
|
||||
const selected = await backend.select_text({ target, text_range: [1, 2] });
|
||||
assert.deepEqual(selected.selected, { start: 1, end: 3 });
|
||||
// The range reaches the generated Python as JSON, never spliced into the script.
|
||||
assert.equal(seen.at(-1)[4], "[1,2]");
|
||||
await assert.rejects(backend.select_text({ target, text_range: [-1, 2] }), (err) => err.code === "bad_args");
|
||||
});
|
||||
|
||||
test("get_value reports a missing value interface as no_value", async (t) => {
|
||||
x11Env(t);
|
||||
const desk = fakeDesktop({
|
||||
onCommand: (cmd, args) => (cmd === "python3" && args[1].includes("target_path"))
|
||||
? { code: 0, stdout: '{"ok": false, "code": "no_value_interface"}\n', stderr: "", timedOut: false, signal: null }
|
||||
: null,
|
||||
});
|
||||
const backend = create({ exec: { run: desk.run, have } });
|
||||
await assert.rejects(backend.get_value({ target: { app_ref: { name: "mousepad" }, windowIndex: 0, path: [1] } }), (err) => err.code === "no_value");
|
||||
});
|
||||
|
||||
test("list_apps returns one entry per process, with the pid", async (t) => {
|
||||
x11Env(t);
|
||||
const backend = create({ exec: { run: async (cmd, args) => (cmd === "wmctrl" && args[0] === "-lpx")
|
||||
? { code: 0, stdout: "0x00800001 0 500 mousepad.Mousepad vm A\n0x00800002 0 500 mousepad.Mousepad vm B\n0x00800003 0 600 xterm.XTerm vm C\n", stderr: "", timedOut: false, signal: null }
|
||||
: { code: 0, stdout: "", stderr: "", timedOut: false, signal: null }, have } });
|
||||
const { apps } = await backend.list_apps();
|
||||
assert.deepEqual(apps.map((a) => [a.name, a.pid, a.windows]), [["mousepad", 500, 2], ["xterm", 600, 1]]);
|
||||
});
|
||||
|
||||
test("screenshot with app_ref crops to the app's frame and names the window stacked above it", async (t) => {
|
||||
x11Env(t);
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "cu-linux-app-shot-"));
|
||||
t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
|
||||
process.env.CODEWHALE_CU_RECORDINGS_DIR = dir;
|
||||
t.after(() => { delete process.env.CODEWHALE_CU_RECORDINGS_DIR; });
|
||||
const desk = fakeDesktop({
|
||||
onCommand: (cmd, args) => {
|
||||
if (cmd === "scrot") { fs.writeFileSync(args.at(-1), PNG_1X1); return { code: 0, stdout: "", stderr: "", timedOut: false, signal: null }; }
|
||||
return null;
|
||||
},
|
||||
});
|
||||
const backend = create({ exec: { run: desk.run, have } });
|
||||
const shot = await backend.screenshot({ app_ref: { name: "mousepad" } });
|
||||
// Client 640x480 at (1,315); extents left 1, right 1, top 20, bottom 5 give the frame.
|
||||
assert.deepEqual(shot.points, { x: 0, y: 295, w: 642, h: 505 });
|
||||
assert.equal(shot.window.wm_class, "mousepad.Mousepad");
|
||||
assert.equal(shot.occluded_by[0].name, "xterm");
|
||||
const scrot = desk.calls.find((c) => c.cmd === "scrot");
|
||||
assert.ok(scrot.args.includes("-a") && scrot.args.includes("0,295,642,505"));
|
||||
});
|
||||
|
||||
test("screenshot refuses window_id and malformed app_ref before probing the desktop", async (t) => {
|
||||
x11Env(t);
|
||||
const desk = fakeDesktop();
|
||||
const backend = create({ exec: { run: desk.run, have } });
|
||||
await assert.rejects(backend.screenshot({ window_id: 1 }), (err) => err.code === "unsupported_selector");
|
||||
await assert.rejects(backend.screenshot({ app_ref: { name: "mousepad", pid: 1 } }), (err) => err.code === "unsupported_selector");
|
||||
await assert.rejects(backend.screenshot({ app_ref: null }), (err) => err.code === "unsupported_selector");
|
||||
assert.equal(desk.calls.length, 0, "refusals must precede every desktop probe");
|
||||
});
|
||||
|
||||
test("x11grab arguments trim to even sizes and carry the region offset", () => {
|
||||
const args = x11GrabArgs({ display: ":99", rect: [10, 20, 641, 479], fps: 15, file: "/tmp/out.mp4", durationSec: 3 });
|
||||
assert.deepEqual(args.slice(args.indexOf("-video_size"), args.indexOf("-video_size") + 2), ["-video_size", "640x478"]);
|
||||
assert.equal(args[args.indexOf("-i") + 1], ":99+10,20");
|
||||
assert.deepEqual(args.slice(args.indexOf("-t"), args.indexOf("-t") + 2), ["-t", "3"]);
|
||||
assert.equal(args.at(-1), "/tmp/out.mp4");
|
||||
assert.throws(() => x11GrabArgs({ display: ":99", rect: [0, 0, 1, 1], fps: 15, file: "/tmp/x.mp4" }), (err) => err.code === "bad_args");
|
||||
});
|
||||
|
||||
test("recording without an X11 session never starts ffmpeg", async (t) => {
|
||||
const saved = Object.fromEntries(["DISPLAY", "WAYLAND_DISPLAY", "XDG_SESSION_TYPE"].map((k) => [k, process.env[k]]));
|
||||
for (const k of Object.keys(saved)) delete process.env[k];
|
||||
t.after(() => { for (const [k, v] of Object.entries(saved)) if (v !== undefined) process.env[k] = v; });
|
||||
const desk = fakeDesktop();
|
||||
const backend = create({ exec: { run: desk.run, have } });
|
||||
await assert.rejects(backend.recordingStart({}), (err) => err.code === "no_session");
|
||||
assert.equal(desk.calls.length, 0);
|
||||
});
|
||||
|
||||
test("list_sessions names the app with keyboard focus and reports no held input", async (t) => {
|
||||
x11Env(t);
|
||||
const backend = create({ exec: { run: fakeDesktop().run, have } });
|
||||
const { sessions, count, via } = await backend.list_sessions();
|
||||
assert.equal(via, "direct");
|
||||
assert.equal(count, 1);
|
||||
assert.equal(sessions[0].target.name, "mousepad");
|
||||
assert.equal(sessions[0].inputHeld, false);
|
||||
});
|
||||
|
||||
test("kill_app refuses ambiguity and never terminates its own process chain", async (t) => {
|
||||
x11Env(t);
|
||||
const backend = create({ exec: { run: fakeDesktop().run, have } });
|
||||
await assert.rejects(backend.kill_app({ pid: process.pid }), (err) => err.code === "bad_args");
|
||||
await assert.rejects(backend.kill_app({ pid: 999999999 }), (err) => err.code === "app_not_found");
|
||||
await assert.rejects(backend.kill_app({ bundle_id: "com.example.app" }), (err) => err.code === "unsupported_selector");
|
||||
});
|
||||
|
||||
test("kill_app refuses a name that matches two processes and ends the one that is named by pid", { skip: process.platform !== "linux" }, async (t) => {
|
||||
x11Env(t);
|
||||
// Two copies of sleep under the same unique name: two processes, one kernel name.
|
||||
const name = `cwamb-${process.pid.toString(36)}`;
|
||||
const children = [];
|
||||
for (const sub of ["a", "b"]) {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), `cu-amb-${sub}-`));
|
||||
t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
|
||||
const bin = path.join(dir, name);
|
||||
fs.copyFileSync("/bin/sleep", bin);
|
||||
fs.chmodSync(bin, 0o755);
|
||||
const child = spawn(bin, ["60"], { stdio: "ignore" });
|
||||
children.push(child);
|
||||
t.after(() => { try { child.kill("SIGKILL"); } catch {} });
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 300));
|
||||
const backend = create({ exec: { run: async () => ({ code: 1, stdout: "", stderr: "", timedOut: false, signal: null }), have } });
|
||||
await assert.rejects(backend.kill_app({ name }), (err) => err.code === "ambiguous_application" && err.pids.length === 2);
|
||||
const out = await backend.kill_app({ pid: children[0].pid });
|
||||
assert.equal(out.killed, true);
|
||||
// The second process was not touched.
|
||||
assert.equal(children[1].exitCode, null);
|
||||
});
|
||||
|
||||
test("kill_app ends a process that ignores its window and reports the method", { skip: process.platform !== "linux" }, async (t) => {
|
||||
x11Env(t);
|
||||
// A copy of sleep under a unique name, so its kernel name is unambiguous.
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "cu-kill-"));
|
||||
t.after(() => fs.rmSync(dir, { recursive: true, force: true }));
|
||||
const name = `cwkt-${process.pid.toString(36)}`;
|
||||
const bin = path.join(dir, name);
|
||||
fs.copyFileSync("/bin/sleep", bin);
|
||||
fs.chmodSync(bin, 0o755);
|
||||
const child = spawn(bin, ["60"], { stdio: "ignore" });
|
||||
t.after(() => { try { child.kill("SIGKILL"); } catch {} });
|
||||
await new Promise((r) => setTimeout(r, 200));
|
||||
const backend = create({ exec: { run: async () => ({ code: 1, stdout: "", stderr: "", timedOut: false, signal: null }), have } });
|
||||
const out = await backend.kill_app({ pid: child.pid });
|
||||
assert.equal(out.killed, true);
|
||||
assert.equal(out.method, "sigterm");
|
||||
});
|
||||
|
||||
test("a browser that exits at start is reported with its stderr; the root sandbox case names the fixes", () => {
|
||||
const root = browserStartFailure({ stderr: "ERROR: Running as root without --no-sandbox is not supported.", exit: { code: 1 } }, "/state/profile");
|
||||
assert.equal(root.code, "browser_unavailable");
|
||||
assert.match(root.message, /non-root user/);
|
||||
assert.match(root.message, /CODEWHALE_CU_BROWSER_APP/);
|
||||
assert.doesNotMatch(root.message, /--no-sandbox"?\s*flag/i, "the message must not offer to disable the sandbox");
|
||||
const other = browserStartFailure({ stderr: "cannot open profile", exit: { code: 2, signal: null } }, "/state/profile");
|
||||
assert.match(other.message, /exited before it opened its debugging endpoint \(code 2\)/);
|
||||
assert.match(other.message, /cannot open profile/);
|
||||
const silent = browserStartFailure(undefined, "/state/profile");
|
||||
assert.match(silent.message, /never came up/);
|
||||
});
|
||||
@@ -29,7 +29,10 @@ for(const platform of ['linux','harmonyos']) {
|
||||
assert.equal(calls.length,0,'refusal must precede even capability probes');
|
||||
}
|
||||
for(const tool of ['list_windows','screenshot']) {
|
||||
for(const app_ref of [...invalidRefs,{name:'Fixture'}]) await rejected(tool,{app_ref});
|
||||
// Linux selects windows by exact app name, so {name:'Fixture'} is a
|
||||
// valid selector there and only malformed selectors refuse. HarmonyOS
|
||||
// rejects every explicit app selector.
|
||||
for(const app_ref of platform==='harmonyos'?[...invalidRefs,{name:'Fixture'}]:invalidRefs) await rejected(tool,{app_ref});
|
||||
for(const window_id of [null,0,1,'0']) await rejected(tool,{window_id});
|
||||
}
|
||||
for(const app_ref of platform==='harmonyos'?[...invalidRefs,{name:'Fixture'}]:invalidRefs) {
|
||||
|
||||
Reference in new issue
Block a user